From ad21c75d4a5ded8efdb3bb3420d6304098a67e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 28 Aug 2024 13:22:18 -0700 Subject: [PATCH 01/21] core: Add rapidyaml parser support --- Makefile | 2 + common/vars.mk | 20 +- core/Makefile | 8 +- core/deps.edn | 1 + core/project.clj | 1 + core/src/yamlscript/parser.clj | 57 +- libyamlscript/Makefile | 7 +- rapidyaml/Makefile | 67 ++ rapidyaml/native/.gitignore | 4 + rapidyaml/native/CMakeLists.txt | 40 ++ rapidyaml/native/Makefile | 71 ++ rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 115 +++ rapidyaml/native/org_rapidyaml_Rapidyaml.h | 45 ++ rapidyaml/native/rapidyaml_edn.cpp | 108 +++ rapidyaml/native/rapidyaml_edn.hpp | 71 ++ rapidyaml/native/rapidyaml_edn_handler.cpp | 83 +++ rapidyaml/native/rapidyaml_edn_handler.hpp | 673 ++++++++++++++++++ rapidyaml/native/rapidyaml_test.cpp | 305 ++++++++ rapidyaml/pom.xml | 115 +++ rapidyaml/rapidyaml.mk | 5 + rapidyaml/src/README.md | 33 + .../main/java/org/rapidyaml/Rapidyaml.java | 61 ++ .../rapidyaml/YamlParseErrorException.java | 16 + rapidyaml/src/site/site.xml | 26 + .../java/org/rapidyaml/RapidyamlTest.java | 163 +++++ util/RYS | 13 + ys/Makefile | 7 +- ys/config/reflect-config.json | 1 + ys/config/resource-config.json | 6 + 29 files changed, 2098 insertions(+), 26 deletions(-) create mode 100644 rapidyaml/Makefile create mode 100644 rapidyaml/native/.gitignore create mode 100644 rapidyaml/native/CMakeLists.txt create mode 100644 rapidyaml/native/Makefile create mode 100644 rapidyaml/native/org_rapidyaml_Rapidyaml.cpp create mode 100644 rapidyaml/native/org_rapidyaml_Rapidyaml.h create mode 100644 rapidyaml/native/rapidyaml_edn.cpp create mode 100644 rapidyaml/native/rapidyaml_edn.hpp create mode 100644 rapidyaml/native/rapidyaml_edn_handler.cpp create mode 100644 rapidyaml/native/rapidyaml_edn_handler.hpp create mode 100644 rapidyaml/native/rapidyaml_test.cpp create mode 100644 rapidyaml/pom.xml create mode 100644 rapidyaml/rapidyaml.mk create mode 100644 rapidyaml/src/README.md create mode 100644 rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java create mode 100644 rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java create mode 100644 rapidyaml/src/site/site.xml create mode 100644 rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java create mode 100755 util/RYS create mode 120000 ys/config/reflect-config.json create mode 100644 ys/config/resource-config.json diff --git a/Makefile b/Makefile index b60d83b57..a62b0fbc5 100644 --- a/Makefile +++ b/Makefile @@ -22,12 +22,14 @@ BINDINGS := \ rust \ DIRS := \ + rapidyaml \ core \ libyamlscript \ $(BINDINGS) \ ys \ BUILD_DIRS := \ + rapidyaml \ libyamlscript \ go \ nodejs \ diff --git a/common/vars.mk b/common/vars.mk index 6c551f7d3..6aa80d5d2 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -83,9 +83,8 @@ CURL := $(shell command -v curl) TIME := time -p LIBYAMLSCRIPT_DIR := $(ROOT)/libyamlscript/lib -LIBRARY_PATH := $(LIBYAMLSCRIPT_DIR) -export $(DY)LD_LIBRARY_PATH := $(LIBRARY_PATH) -export LD_LIBRARY_PATH := $(LIBRARY_PATH) +LIBRARY_PATH := $(LIBYAMLSCRIPT_DIR):$(ROOT)/rapidyaml/native +export $(DY)LD_LIBRARY_PATH := $(LIBRARY_PATH):$(ROOT)/rapidyaml/native LIBYAMLSCRIPT_SO_NAME := $(LIBYAMLSCRIPT_DIR)/libyamlscript LIBYAMLSCRIPT_SO_FQNP := $(LIBYAMLSCRIPT_SO_NAME).$(SO).$(YS_VERSION) LIBYAMLSCRIPT_SO_BASE := $(LIBYAMLSCRIPT_DIR)/libyamlscript.$(SO) @@ -221,6 +220,21 @@ RELEASE_LYS_NAME := libyamlscript-$(YS_VERSION)-$(GRAALVM_ARCH) RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz +#------------------------------------------------------------------------------ +# RapidYAML variables: +#------------------------------------------------------------------------------ +RAPIDYAML := $(ROOT)/rapidyaml + +RAPIDYAML_VERSION := 0.7.2 +RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar +RAPIDYAML_SO := \ + $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_INSTALLED_DIR := \ + $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)/ +RAPIDYAML_INSTALLED := \ + $(RAPIDYAML_INSTALLED_DIR)/rapidyaml-$(RAPIDYAML_VERSION).jar + + #------------------------------------------------------------------------------ default:: diff --git a/core/Makefile b/core/Makefile index a6d28f544..c4591746f 100644 --- a/core/Makefile +++ b/core/Makefile @@ -6,11 +6,17 @@ include $(COMMON)/docker.mk export PATH := $(ROOT)/core/bin:$(PATH) #------------------------------------------------------------------------------ -build:: +build:: $(RAPIDYAML_SO) $(RAPIDYAML_INSTALLED) install test:: $(LEIN) build $< $@ +$(RAPIDYAML_SO): + $(MAKE) -C $(ROOT)/rapidyaml build + +$(RAPIDYAML_INSTALLED): + $(MAKE) -C $(ROOT)/rapidyaml install + Dockerfile:: $(COMMON) Makefile cat \ $> yaml-string - (.parseString parser) - (map ys-event) - (remove nil?) - rest) + events (if (System/getenv "YS_PARSER_RAPIDYAML") + (parse-rapidyaml yaml-string) + (parse-snakeyaml yaml-string)) [first-event & rest-events] events first-event-tag (:! first-event) first-event (if (and has-code-mode-shebang @@ -56,6 +57,26 @@ events (cons first-event rest-events)] (remove nil? events))) +(declare snake-event) + +;; TODO - Set bigger buffer size in scanner class +(defn parse-snakeyaml [yaml-string] + (let [parser (new Parse (.build (LoadSettings/builder)))] + (->> yaml-string + (.parseString parser) + (map snake-event) + (remove nil?) + rest))) + +(defn parse-rapidyaml [yaml-string] + (let [rapid-parser (new Rapidyaml)] + (->> yaml-string + ( #(.parseYS ^Rapidyaml rapid-parser %1)) + #_(#(do (println %1) %1)) + (#(str/replace %1 #"^\(\n\{:\+\ \"\+DOC\"\}" "(")) + #_(#(do (println %1) %1)) + read-string))) + (defn parse-test-case [yaml-string] (->> yaml-string parse @@ -132,16 +153,16 @@ (let [obj (event-obj event)] (assoc obj :* (str (. event getAlias))))) -(defmulti ys-event class) -(defmethod ys-event DocumentStartEvent [event] (doc-start event)) -(defmethod ys-event DocumentEndEvent [event] (doc-end event)) -(defmethod ys-event MappingStartEvent [event] (map-start event)) -(defmethod ys-event MappingEndEvent [event] (map-end event)) -(defmethod ys-event SequenceStartEvent [event] (seq-start event)) -(defmethod ys-event SequenceEndEvent [event] (seq-end event)) -(defmethod ys-event ScalarEvent [event] (scalar-val event)) -(defmethod ys-event AliasEvent [event] (alias-val event)) -(defmethod ys-event :default [_] nil) +(defmulti snake-event class) +(defmethod snake-event DocumentStartEvent [event] (doc-start event)) +(defmethod snake-event DocumentEndEvent [event] (doc-end event)) +(defmethod snake-event MappingStartEvent [event] (map-start event)) +(defmethod snake-event MappingEndEvent [event] (map-end event)) +(defmethod snake-event SequenceStartEvent [event] (seq-start event)) +(defmethod snake-event SequenceEndEvent [event] (seq-end event)) +(defmethod snake-event ScalarEvent [event] (scalar-val event)) +(defmethod snake-event AliasEvent [event] (alias-val event)) +(defmethod snake-event :default [_] nil) (comment ) diff --git a/libyamlscript/Makefile b/libyamlscript/Makefile index da62f4181..2e5a22988 100644 --- a/libyamlscript/Makefile +++ b/libyamlscript/Makefile @@ -28,7 +28,7 @@ endif #------------------------------------------------------------------------------ build:: $(BUILD_TARGETS) -jar: $(LIBYAMLSCRIPT_JAR_PATH) +jar: $(LIBYAMLSCRIPT_JAR_PATH) $(RAPIDYAML_INSTALLED) install:: $(BUILD_TARGETS) mkdir -p $(PREFIX)/include/ @@ -48,7 +48,7 @@ test:: $(LIBYAMLSCRIPT_SO_FQNP) repl-deps:: $(LIBYAMLSCRIPT_JAR_PATH) -$(LIBYAMLSCRIPT_SO_FQNP): $(LIBYAMLSCRIPT_JAR_PATH) +$(LIBYAMLSCRIPT_SO_FQNP): $(LIBYAMLSCRIPT_JAR_PATH) $(RAPIDYAML_INSTALLED) $(JAVA_INSTALLED) ifneq (true,$(LIBZ)) $(error *** The 'libz.$(SO)' library is required by native-image but not installed) endif @@ -77,6 +77,9 @@ $(LIBYAMLSCRIPT_JAR_PATH): $(LEIN) $(JAVA_INSTALLED) $(YAMLSCRIPT_CORE_INSTALLED $< uberjar endif +$(RAPIDYAML_INSTALLED): + $(MAKE) -C $(RAPIDYAML) $@ + Dockerfile:: $(COMMON) Makefile cat \ $ yamlscript" + HOMEPAGE_URL "https://github.com/biojppm/rapidyaml -> https://github.com/yaml/yamlscript" + LANGUAGES CXX) + +find_package(JNI REQUIRED) + +option(YS2EDN_TIMED "add timings to sections" OFF) + +if(UNIX) + set(CMAKE_SHARED_LIBRARY_SUFFIX .so) +endif() + +add_library(rapidyaml + org_rapidyaml_Rapidyaml.h + org_rapidyaml_Rapidyaml.cpp + rapidyaml_all.hpp + rapidyaml_edn_handler.hpp + rapidyaml_edn_handler.cpp + rapidyaml_edn.hpp + rapidyaml_edn.cpp +) +target_include_directories(rapidyaml PUBLIC + ${CMAKE_CURRENT_LIST_DIR} + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/test +) +target_compile_definitions(rapidyaml PUBLIC + RYML_WITH_TAB_TOKENS + RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS + RYML_SINGLE_HEADER + $<$:YS2EDN_TIMED> +) +set_target_properties(rapidyaml PROPERTIES CXX_STANDARD 17) + +# target_link_libraries(rapidyaml PUBLIC JNI::JNI) +target_include_directories(rapidyaml PUBLIC ${JNI_INCLUDE_DIRS}) + +add_executable(rapidyaml-test rapidyaml_test.cpp) +target_link_libraries(rapidyaml-test rapidyaml) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile new file mode 100644 index 000000000..876509c10 --- /dev/null +++ b/rapidyaml/native/Makefile @@ -0,0 +1,71 @@ +include ../../common/base.mk +include $(COMMON)/clojure.mk +include $(COMMON)/java.mk +include $(COMMON)/python.mk +include ../rapidyaml.mk + +RAPIDYAML_H := org_rapidyaml_Rapidyaml.h +RAPIDYAML_HPP := rapidyaml_all.hpp +RAPIDYAML_AMALGAMATE_OPTS := --stl + +RAPIDYAML_SO_DEPS := \ + Makefile \ + CMakeLists.txt \ + $(JAVA_HOME) \ + $(RAPIDYAML_H) \ + $(RAPIDYAML_HPP) \ + $(wildcard ./*pp) \ + +# TODO maybe change to static library! +# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni +SHARED := 1 +TIMED := 0 + +CMK_FLAGS := +CMK_FLAGS += -D YS2EDN_TIMED=$(TIMED) +CMK_FLAGS += -D BUILD_SHARED_LIBS=$(SHARED) + + +#------------------------------------------------------------------------------ +default:: + +build:: $(RAPIDYAML_SO) + +test:: build + cmake --build rapidyaml-build \ + --parallel --target rapidyaml-test + ./rapidyaml-build/rapidyaml-test + +rapidyaml: + mkdir -p $@ + git -C $@ init -q . + git -C $@ remote add origin $(RAPIDYAML_REPO) + git -C $@ fetch origin $(RAPIDYAML_TAG) + git -C $@ reset --hard FETCH_HEAD + git -C $@ submodule update --init --recursive + +clean:: + $(RM) librapidyaml.* + $(RM) $(RAPIDYAML_HPP) + $(RM) -r rapidyaml-build + $(RM) -r rapidyaml-install + +realclean:: clean + $(RM) -r rapidyaml + + +#------------------------------------------------------------------------------ +$(RAPIDYAML_SO): $(RAPIDYAML_SO_DEPS) + cmake -S . -B rapidyaml-build \ + $(CMK_FLAGS) + cmake --build rapidyaml-build \ + --parallel --verbose --target rapidyaml + mv rapidyaml-build/*.so $@ + ln -fs $@ librapidyaml.so + +$(RAPIDYAML_H): $(RAPIDYAML_JAVA) + javac -h . $^ + +$(RAPIDYAML_HPP): rapidyaml + $(PYTHON) rapidyaml/tools/amalgamate.py \ + $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp new file mode 100644 index 000000000..4e57b87d8 --- /dev/null +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -0,0 +1,115 @@ +#include +#include "./rapidyaml_edn.hpp" +#include + +#ifndef _Included_org_rapidyaml_Rapidyaml +#define _Included_org_rapidyaml_Rapidyaml +#ifdef __cplusplus +extern "C" { +#endif + +void throw_java_exception(JNIEnv * env, + const char* type, + const char* msg) +{ + jclass clazz = env->FindClass(type); + if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown + env->ThrowNew(clazz, msg); +} + +void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg) +{ + // see https://stackoverflow.com/questions/55013243/jni-custom-exceptions-with-more-than-one-parameter + jclass clazz = env->FindClass("org/rapidyaml/YamlParseErrorException"); + if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown + { + jstring jmsg = env->NewStringUTF(msg); + jint joffset = (jint)offset; + jint jline = (jint)line; + jint jcol = (jint)column; + // see https://www.rgagnon.com/javadetails/java-0286.html + // about the proper signature. + // we want (int, int, int, String): + const char * const signature = "(IIILjava/lang/String;)V"; + jmethodID ctor = env->GetMethodID(clazz, "", signature); + jobject jexc = env->NewObject(clazz, ctor, joffset, jline, jcol, jmsg); + env->Throw((jthrowable)jexc); // https://stackoverflow.com/questions/2455668/jni-cast-between-jobect-and-jthrowable + } +} + + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_init + * Signature: ()Ljava/lang/Object; + */ +JNIEXPORT jlong JNICALL +Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) +{ + Ryml2Edn *obj = ys2edn_init(); + return (jlong)obj; +} + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_destroy + * Signature: (Ljava/lang/Object;)V + */ +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) +{ + ys2edn_destroy((Ryml2Edn*)obj); +} + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_parse + * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I + */ +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jbyteArray src, jint src_len, + jbyteArray dst, jint dst_len) +{ + jboolean src_is_copy, dst_is_copy; + jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); + jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); + const char *filename = env->GetStringUTFChars(jfilename, 0); + int rc = 0; + try + { + rc = ys2edn_parse((Ryml2Edn*)obj, filename, + (char*)src_, src_len, + (char*)dst_, dst_len); + } + catch (Ryml2EdnParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + env->ReleaseByteArrayElements(src, src_, 0); + env->ReleaseByteArrayElements(dst, dst_, 0); + env->ReleaseStringUTFChars(jfilename, filename); + return rc; +} + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_retry_get + * Signature: (Ljava/lang/Object;[BI)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, + jlong obj, + jbyteArray dst, jint dst_len) +{ + jboolean src_is_copy, dst_is_copy; + jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); + int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); + env->ReleaseByteArrayElements(dst, dst_, 0); + return rc; +} + +#ifdef __cplusplus +} +#endif +#endif diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h new file mode 100644 index 000000000..c73497636 --- /dev/null +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_rapidyaml_Rapidyaml */ + +#ifndef _Included_org_rapidyaml_Rapidyaml +#define _Included_org_rapidyaml_Rapidyaml +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init + (JNIEnv *, jobject); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy + (JNIEnv *, jobject, jlong); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_parse + * Signature: (JLjava/lang/String;[BI[BI)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_retry_get + * Signature: (J[BI)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/rapidyaml/native/rapidyaml_edn.cpp b/rapidyaml/native/rapidyaml_edn.cpp new file mode 100644 index 000000000..8f5bcfca9 --- /dev/null +++ b/rapidyaml/native/rapidyaml_edn.cpp @@ -0,0 +1,108 @@ +#define RYML_SINGLE_HDR_DEFINE_NOW +#include +#include "rapidyaml_edn.hpp" +#include + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +using namespace ryml; + +#ifndef YS2EDN_TIMED +#define TIMED_SECTION(name) +#else +#include +#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) +struct timed_section +{ + using myclock = std::chrono::steady_clock; + using fmsecs = std::chrono::duration; + csubstr name; + myclock::time_point start; + fmsecs since() const { return myclock::now() - start; } + timed_section(csubstr n) : name(n), start(myclock::now()) {} + ~timed_section() + { + fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); + } +}; +#endif + + +#if defined(__cplusplus) +extern "C" { +#endif +// see +// https://stackoverflow.com/questions/230689/best-way-to-throw-exceptions-in-jni-code +// https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method + +namespace { +C4_NORETURN void ys2edn_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) +{ + Ryml2EdnParseError exc; + exc.location = location; + exc.msg.assign(msg, msg_len); + throw exc; +} +} // anon namespace + +RYML_EXPORT Ryml2Edn *ys2edn_init() +{ + TIMED_SECTION("ys2edn_init"); + Callbacks cb = {}; + cb.m_error = &ys2edn_parse_error; + set_callbacks(cb); + Ryml2Edn *ryml2edn = _RYML_CB_ALLOC(get_callbacks(), Ryml2Edn, 1); + _RYML_CB_CHECK(get_callbacks(), ryml2edn != nullptr); + new ((void*)ryml2edn) Ryml2Edn(); + return ryml2edn; +} + +RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn) +{ + TIMED_SECTION("ys2edn_destroy"); + ryml2edn->~Ryml2Edn(); +} + +RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, + const char *filename, + char *ys, size_type ys_size, + char *edn, size_type edn_size) +{ + TIMED_SECTION("ys2edn_parse"); + csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; + substr ys_(ys, (size_t)ys_size); + { + TIMED_SECTION("reset"); + ryml2edn->reset(); + } + { + TIMED_SECTION("parse_in_place"); + ryml2edn->m_parser.parse_in_place_ev(filename_, ys_); + } + return ys2edn_retry_get(ryml2edn, edn, edn_size); +} + +RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, + char *edn, size_type edn_size) +{ + TIMED_SECTION("ys2edn_retry_get"); + csubstr result = to_csubstr(ryml2edn->m_sink.result); + size_type required_size = 1 + (size_type)result.len; + if(required_size <= edn_size) + { + memcpy(edn, result.str, (size_t)result.len); + edn[result.len] = '\0'; + } + else if(edn_size > 0) + { + edn[0] = '\0'; + } + return required_size; +} + +#if defined(__cplusplus) +} +#endif diff --git a/rapidyaml/native/rapidyaml_edn.hpp b/rapidyaml/native/rapidyaml_edn.hpp new file mode 100644 index 000000000..69ca2f077 --- /dev/null +++ b/rapidyaml/native/rapidyaml_edn.hpp @@ -0,0 +1,71 @@ +#pragma once +#ifndef RAPIDYAML_EVENTS_H +#define RAPIDYAML_EVENTS_H + +#include +#include +#include "rapidyaml_edn_handler.hpp" + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +#if defined(__cplusplus) +extern "C" { +#endif + +using size_type = int; + +struct RYML_EXPORT Ryml2Edn +{ + c4::yml::EventHandlerEdn::EventSink m_sink; + c4::yml::EventHandlerEdn m_handler; + c4::yml::ParseEngine m_parser; + Ryml2Edn() + : m_sink() + , m_handler(&m_sink) + , m_parser(&m_handler) + { + } + void reset() + { + m_sink.reset(); + m_handler.reset(); + } +}; + +struct RYML_EXPORT Ryml2EdnParseError : public std::exception +{ + c4::yml::Location location; + std::string msg; + const char* what() const noexcept override { return msg.c_str(); } +}; + + +//----------------------------------------------------------------------------- + +/** Initialize the resources */ +RYML_EXPORT Ryml2Edn *ys2edn_init(); + +/** Destroy the resources */ +RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn); + +/** Parse YAML, and return corresponding EDN. Return the number of + * characters needed for edn. Check if the returned size is larger + * than edn_size. If it is, call ys_retry_get() can be called + * afterwards to extract the EDN. */ +RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, + const char *filename, + char *ys, size_type ys_size, + char *edn, size_type edn_size); + +/** Get the edn from the previous call to ys2edn_parse(). */ +RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, + char *edn, size_type edn_size); + +#if defined(__cplusplus) +} +#endif + +#endif /* RAPIDYAML_EVENTS_H */ diff --git a/rapidyaml/native/rapidyaml_edn_handler.cpp b/rapidyaml/native/rapidyaml_edn_handler.cpp new file mode 100644 index 000000000..635f44601 --- /dev/null +++ b/rapidyaml/native/rapidyaml_edn_handler.cpp @@ -0,0 +1,83 @@ +#ifndef RYML_SINGLE_HEADER +#include +#include +#include +#endif +#include "./rapidyaml_edn_handler.hpp" + + +namespace c4 { +namespace yml { + +// instantiate the template +template class ParseEngine; + + +void EventHandlerEdn::EventSink::append_escaped(csubstr val) +{ + #define _c4flush_use_instead(repl, skip) \ + do { \ + this->append(val.range(prev, i)); \ + this->append(repl); \ + prev = i + skip; \ + } \ + while(0) + uint8_t const* C4_RESTRICT s = reinterpret_cast(val.str); + size_t prev = 0; + for(size_t i = 0; i < val.len; ++i) + { + switch(s[i]) + { + case UINT8_C(0x0a): // \n + _c4flush_use_instead("\\n", 1); break; + case UINT8_C(0x5c): // '\\' + _c4flush_use_instead("\\\\", 1); break; + case UINT8_C(0x22): // \" + _c4flush_use_instead("\\\"", 1); break; + case UINT8_C(0x09): // \t + _c4flush_use_instead("\\t", 1); break; + case UINT8_C(0x0d): // \r + _c4flush_use_instead("\\r", 1); break; + case UINT8_C(0x00): // \0 + _c4flush_use_instead("\\0", 1); break; + case UINT8_C(0x0c): // \f (form feed) + _c4flush_use_instead("\\f", 1); break; + case UINT8_C(0x08): // \b (backspace) + _c4flush_use_instead("\\b", 1); break; + case UINT8_C(0x07): // \a (bell) + _c4flush_use_instead("\\a", 1); break; + case UINT8_C(0x0b): // \v (vertical tab) + _c4flush_use_instead("\\v", 1); break; + case UINT8_C(0x1b): // \e (escape) + _c4flush_use_instead("\\e", 1); break; + case UINT8_C(0xc2): + if(i+1 < val.len) + { + const uint8_t np1 = s[i+1]; + if(np1 == UINT8_C(0xa0)) + _c4flush_use_instead("\\_", 2); + else if(np1 == UINT8_C(0x85)) + _c4flush_use_instead("\\N", 2); + } + break; + case UINT8_C(0xe2): + if(i+2 < val.len) + { + if(s[i+1] == UINT8_C(0x80)) + { + if(s[i+2] == UINT8_C(0xa8)) + _c4flush_use_instead("\\L", 3); + else if(s[i+2] == UINT8_C(0xa9)) + _c4flush_use_instead("\\P", 3); + } + } + break; + } + } + // flush the rest + this->append(val.sub(prev)); + #undef _c4flush_use_instead +} + +} // namespace yml +} // namespace c4 diff --git a/rapidyaml/native/rapidyaml_edn_handler.hpp b/rapidyaml/native/rapidyaml_edn_handler.hpp new file mode 100644 index 000000000..973c52f57 --- /dev/null +++ b/rapidyaml/native/rapidyaml_edn_handler.hpp @@ -0,0 +1,673 @@ +#ifndef _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ +#define _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ + +#ifdef RYML_SINGLE_HEADER +#include +#else +#ifndef _C4_YML_EVENT_HANDLER_STACK_HPP_ +#include "c4/yml/event_handler_stack.hpp" +#endif +#ifndef _C4_YML_STD_STRING_HPP_ +#include "c4/yml/std/string.hpp" +#endif +#ifndef _C4_YML_DETAIL_PRINT_HPP_ +#include "c4/yml/detail/print.hpp" +#endif +#endif + +#include + +C4_SUPPRESS_WARNING_GCC_CLANG_PUSH +C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") +C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") + + +namespace c4 { +namespace yml { + + +/** @addtogroup doc_event_handlers + * @{ */ + + +struct EventHandlerEdnState : public ParserState +{ + NodeData ev_data; +}; + + +struct EventHandlerEdn : public EventHandlerStack +{ + + /** @name types + * @{ */ + + // our internal state must inherit from parser state + using state = EventHandlerEdnState; + + struct EventSink + { + std::string result; + void reset() noexcept { result.clear(); } + void append(csubstr s) noexcept { result.append(s.str, s.len); } + void append(char c) noexcept { result += c; } + void insert(csubstr s, size_t pos) noexcept { result.insert(pos, s.str, s.len); } + void insert(char c, size_t pos) noexcept { result.insert(pos, 1, c); } + csubstr get() const { return csubstr(&result[0], result.size()); } + substr get() { return substr(&result[0], result.size()); } + size_t find_last(csubstr s) const { return result.rfind(s.str, std::string::npos, s.len); } + void append_escaped(csubstr val); + }; + + /** @} */ + +public: + + /** @cond dev */ + EventSink *C4_RESTRICT m_sink; + std::vector m_val_buffers; + char m_key_tag_buf[256]; + char m_val_tag_buf[256]; + std::string m_arena; + bool m_first_doc; + + // undefined at the end + #define _enable_(bits) _enable__() + #define _disable_(bits) _disable__() + #define _has_any_(bits) _has_any__() + /** @endcond */ + +public: + + /** @name construction and resetting + * @{ */ + + EventHandlerEdn() : EventHandlerStack(), m_sink(), m_val_buffers(), m_first_doc() {} + EventHandlerEdn(Callbacks const& cb) : EventHandlerStack(cb), m_sink(), m_val_buffers(), m_first_doc() {} + EventHandlerEdn(EventSink *sink, Callbacks const& cb) : EventHandlerStack(cb), m_sink(sink), m_val_buffers(), m_first_doc() + { + reset(); + } + EventHandlerEdn(EventSink *sink) : EventHandlerEdn(sink, get_callbacks()) {} + + void reset() + { + _stack_reset_root(); + m_curr->flags |= RUNK|RTOP; + m_val_buffers.resize((size_t)m_stack.size()); + m_arena.clear(); + m_first_doc = true; + } + + /** @} */ + +public: + + /** @name parse events + * @{ */ + + void start_parse(const char* filename, detail::pfn_relocate_arena relocate_arena, void *relocate_arena_data) + { + this->_stack_start_parse(filename, relocate_arena, relocate_arena_data); + } + + void finish_parse() + { + this->_stack_finish_parse(); + } + + void cancel_parse() + { + while(m_stack.size() > 1) + _pop(); + _buf_flush_(); + } + + /** @} */ + +public: + + /** @name YAML stream events */ + /** @{ */ + + void begin_stream() + { + _send_("(\n"); + } + + void end_stream() + { + _send_(")\n"); + _buf_flush_(); + } + + /** @} */ + +public: + + /** @name YAML document events */ + /** @{ */ + + /** implicit doc start (without ---) */ + void begin_doc() + { + _c4dbgp("begin_doc"); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + _enable_(DOC); + } + m_first_doc = false; + } + /** implicit doc end (without ...) */ + void end_doc() + { + _c4dbgp("end_doc"); + _send_("{:+ \"-DOC\"}\n"); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** explicit doc start, with --- */ + void begin_doc_expl() + { + _c4dbgp("begin_doc_expl"); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + } + if (m_first_doc) + m_first_doc = false; + else + _send_("{:+ \"+DOC\"}\n"); + _enable_(DOC); + } + /** explicit doc end, with ... */ + void end_doc_expl() + { + _c4dbgp("end_doc_expl"); + _send_("{:+ \"-DOC\"}\n"); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** @} */ + +public: + + /** @name YAML map functions */ + /** @{ */ + + void begin_map_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_map_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_map_val_flow() + { + _send_("{:+ \"+MAP\""); + _send_val_props_(); + _send_(", :flow true}\n"); + _mark_parent_with_children_(); + _enable_(MAP|FLOW_SL); + _push(); + } + void begin_map_val_block() + { + _send_("{:+ \"+MAP\""); + _send_val_props_(); + _send_("}\n"); + _mark_parent_with_children_(); + _enable_(MAP|BLOCK); + _push(); + } + + void end_map() + { + _pop(); + _send_("{:+ \"-MAP\"}\n"); + } + + /** @} */ + +public: + + /** @name YAML seq events */ + /** @{ */ + + void begin_seq_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_seq_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_seq_val_flow() + { + _send_("{:+ \"+SEQ\""); + _send_val_props_(); + _send_(", :flow true}\n"); + _mark_parent_with_children_(); + _enable_(SEQ|FLOW_SL); + _push(); + } + void begin_seq_val_block() + { + _send_("{:+ \"+SEQ\""); + _send_val_props_(); + _send_("}\n"); + _mark_parent_with_children_(); + _enable_(SEQ|BLOCK); + _push(); + } + + void end_seq() + { + _pop(); + _send_("{:+ \"-SEQ\"}\n"); + } + + /** @} */ + +public: + + /** @name YAML structure events */ + /** @{ */ + + void add_sibling() + { + _RYML_CB_ASSERT(m_stack.m_callbacks, m_parent); + _buf_flush_to_(m_curr->level, m_parent->level); + m_curr->ev_data = {}; + } + + /** set the previous val as the first key of a new map, with flow style. + * + * See the documentation for @ref doc_event_handlers, which has + * important notes about this event. + */ + void actually_val_is_first_key_of_new_map_flow() + { + // ensure we have a temporary buffer to save the current val + const id_type tmp = m_curr->level + id_type(2); + _buf_ensure_(tmp + id_type(2)); + // save the current val to the temporary buffer + _buf_flush_to_(m_curr->level, tmp); + // create the map. + // this will push a new level, and tmp is one further + begin_map_val_flow(); + _RYML_CB_ASSERT(m_stack.m_callbacks, tmp != m_curr->level); + // now move the saved val as the first key + _buf_flush_to_(tmp, m_curr->level); + } + + void actually_val_is_first_key_of_new_map_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + /** @} */ + +public: + + /** @name YAML scalar events */ + /** @{ */ + + + C4_ALWAYS_INLINE void set_key_scalar_plain(csubstr scalar) + { + _send_key_scalar_(scalar, '='); + _enable_(KEY|KEY_PLAIN); + } + C4_ALWAYS_INLINE void set_val_scalar_plain(csubstr scalar) + { + _send_val_scalar_(scalar, '='); + _enable_(VAL|VAL_PLAIN); + } + + + C4_ALWAYS_INLINE void set_key_scalar_dquoted(csubstr scalar) + { + _send_key_scalar_(scalar, '$'); + _enable_(KEY|KEY_DQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_dquoted(csubstr scalar) + { + _send_val_scalar_(scalar, '$'); + _enable_(VAL|VAL_DQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_squoted(csubstr scalar) + { + _send_key_scalar_(scalar, '\''); + _enable_(KEY|KEY_SQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_squoted(csubstr scalar) + { + _send_val_scalar_(scalar, '\''); + _enable_(VAL|VAL_SQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_literal(csubstr scalar) + { + _send_key_scalar_(scalar, '|'); + _enable_(KEY|KEY_LITERAL); + } + C4_ALWAYS_INLINE void set_val_scalar_literal(csubstr scalar) + { + _send_val_scalar_(scalar, '|'); + _enable_(VAL|VAL_LITERAL); + } + + + C4_ALWAYS_INLINE void set_key_scalar_folded(csubstr scalar) + { + _send_key_scalar_(scalar, '>'); + _enable_(KEY|KEY_FOLDED); + } + C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) + { + _send_val_scalar_(scalar, '>'); + _enable_(VAL|VAL_FOLDED); + } + + + C4_ALWAYS_INLINE void mark_key_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + C4_ALWAYS_INLINE void mark_val_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + + /** @} */ + +public: + + /** @name YAML anchor/reference events */ + /** @{ */ + + void set_key_anchor(csubstr anchor) + { + _enable_(KEYANCH); + m_curr->ev_data.m_key.anchor = anchor; + } + void set_val_anchor(csubstr anchor) + { + _enable_(VALANCH); + m_curr->ev_data.m_val.anchor = anchor; + } + + void set_key_ref(csubstr ref) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); + _enable_(KEY|KEYREF); + _send_("{:+ \"=ALI\" :* \""); + _send_(ref.sub(1)); + _send_("\"}\n"); + } + void set_val_ref(csubstr ref) + { + _enable_(VAL|VALREF); + _send_("{:+ \"=ALI\" :* \""); + _send_(ref.sub(1)); + _send_("\"}\n"); + } + + /** @} */ + +public: + + /** @name YAML tag events */ + /** @{ */ + + void set_key_tag(csubstr tag) + { + _enable_(KEYTAG); + m_curr->ev_data.m_key.tag = _transform_directive(tag, m_key_tag_buf); + } + void set_val_tag(csubstr tag) + { + _enable_(VALTAG); + m_curr->ev_data.m_val.tag = _transform_directive(tag, m_val_tag_buf); + } + + /** @} */ + +public: + + /** @name YAML directive events */ + /** @{ */ + + void add_directive(csubstr directive) + { + _RYML_CB_ERR(m_stack.m_callbacks, "tag directives not supported"); + } + + /** @} */ + +public: + + /** @name YAML arena events */ + /** @{ */ + + substr alloc_arena(size_t len) + { + const size_t sz = m_arena.size(); + csubstr prev = to_csubstr(m_arena); + m_arena.resize(sz + len); + substr out = to_substr(m_arena).sub(sz); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + _stack_relocate_to_new_arena(prev, curr); + return out; + } + + substr alloc_arena(size_t len, substr *relocated) + { + csubstr prev = to_csubstr(m_arena); + if(!prev.is_super(*relocated)) + return alloc_arena(len); + substr out = alloc_arena(len); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); + return out; + } + + /** @} */ + +public: + + /** @cond dev */ + + /** push a new parent, add a child to the new parent, and set the + * child as the current node */ + void _push() + { + _stack_push(); + _buf_ensure_(m_stack.size() + id_type(1)); + _buf_().reset(); + m_curr->ev_data = {}; + } + + /** end the current scope */ + void _pop() + { + _buf_flush_to_(m_curr->level, m_parent->level); + _stack_pop(); + } + + template C4_ALWAYS_INLINE void _enable__() noexcept + { + m_curr->ev_data.m_type.type = static_cast(m_curr->ev_data.m_type.type | bits); + } + template C4_ALWAYS_INLINE void _disable__() noexcept + { + m_curr->ev_data.m_type.type = static_cast(m_curr->ev_data.m_type.type & (~bits)); + } + template C4_ALWAYS_INLINE bool _has_any__() const noexcept + { + return (m_curr->ev_data.m_type.type & bits) != 0; + } + + void _mark_parent_with_children_() + { + if(m_parent) + m_parent->has_children = true; + } + + EventSink& _buf_() noexcept + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)m_curr->level < m_val_buffers.size()); + return m_val_buffers[(size_t)m_curr->level]; + } + + EventSink& _buf_(id_type level) noexcept + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)level < m_val_buffers.size()); + return m_val_buffers[(size_t)level]; + } + + EventSink const& _buf_(id_type level) const noexcept + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)level < m_val_buffers.size()); + return m_val_buffers[(size_t)level]; + } + + static void _buf_flush_to_(EventSink &C4_RESTRICT src, EventSink &C4_RESTRICT dst) noexcept + { + dst.append(src.get()); + src.reset(); + } + + void _buf_flush_to_(id_type level_src, id_type level_dst) noexcept + { + auto &src = _buf_(level_src); + auto &dst = _buf_(level_dst); + _buf_flush_to_(src, dst); + } + + void _buf_flush_() noexcept + { + _buf_flush_to_(_buf_(), *m_sink); + } + + void _buf_ensure_(id_type size_needed) noexcept + { + if((size_t)size_needed > m_val_buffers.size()) + m_val_buffers.resize((size_t)size_needed); + } + + C4_ALWAYS_INLINE void _send_(csubstr s) noexcept { _buf_().append(s); } + C4_ALWAYS_INLINE void _send_(char c) noexcept { _buf_().append(c); } + + void _send_key_scalar_(csubstr scalar, char scalar_type_code) + { + _send_("{:+ \"=VAL\""); + _send_key_props_(); + _send_(", :"); + _send_(scalar_type_code); + _send_(" \""); + _buf_().append_escaped(scalar); + _send_("\"}\n"); + } + void _send_val_scalar_(csubstr scalar, char scalar_type_code) + { + _send_("{:+ \"=VAL\""); + _send_val_props_(); + _send_(", :"); + _send_(scalar_type_code); + _send_(" \""); + _buf_().append_escaped(scalar); + _send_("\"}\n"); + } + + void _send_key_props_() + { + if(_has_any_(KEYANCH|KEYREF)) + { + _send_(", :& \""); + _send_(m_curr->ev_data.m_key.anchor); + _send_('\"'); + } + if(_has_any_(KEYTAG)) + { + _send_(", :! \""); + _send_tag_(m_curr->ev_data.m_key.tag); + _send_('\"'); + } + m_curr->ev_data.m_key = {}; + _disable_(KEYANCH|KEYREF|KEYTAG); + } + void _send_val_props_() + { + if(_has_any_(VALANCH|VALREF)) + { + _send_(", :& \""); + _send_(m_curr->ev_data.m_val.anchor); + _send_('\"'); + } + if(m_curr->ev_data.m_type.type & VALTAG) + { + _send_(", :! \""); + _send_tag_(m_curr->ev_data.m_val.tag); + _send_('\"'); + } + m_curr->ev_data.m_val = {}; + _disable_(VALANCH|VALREF|VALTAG); + } + void _send_tag_(csubstr tag) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, !tag.empty()); + if(tag.begins_with('!')) + tag = tag.sub(1); + _send_(tag); + } + + csubstr _transform_directive(csubstr tag, substr output) + { + if(tag.begins_with('!')) + { + if(is_custom_tag(tag)) + { + _RYML_CB_ERR_(m_stack.m_callbacks, "tag not found", m_curr->pos); + } + } + csubstr result = normalize_tag_long(tag, output); + _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); + _RYML_CB_CHECK(m_stack.m_callbacks, result.str); + return result; + } +#undef _enable_ +#undef _disable_ +#undef _has_any_ + + /** @endcond */ +}; + +/** @} */ + +} // namespace yml +} // namespace c4 + +C4_SUPPRESS_WARNING_GCC_POP + +#endif /* _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ */ diff --git a/rapidyaml/native/rapidyaml_test.cpp b/rapidyaml/native/rapidyaml_test.cpp new file mode 100644 index 000000000..7c2e15f72 --- /dev/null +++ b/rapidyaml/native/rapidyaml_test.cpp @@ -0,0 +1,305 @@ +#include + +using c4::csubstr; +using c4::substr; + + +struct Ys2EdnScoped +{ + Ryml2Edn *ryml2edn; + Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} + ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } +}; + + +struct ResultScoped +{ + char *edn = nullptr; + ~ResultScoped() { if(edn) ys2edn_free(edn); } +}; + +struct TestResult +{ + uint32_t num_assertions; + uint32_t num_tests; + uint32_t num_failed_assertions; + uint32_t num_failed_tests; + operator bool() const { return num_failed_tests == 0; } + void add(TestResult const& that) + { + num_tests += 1 + that.num_tests; + num_assertions += that.num_assertions; + num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; + num_failed_assertions += that.num_failed_assertions; + } +}; + +struct TestCase +{ + csubstr ys; + csubstr edn; + bool testeq(csubstr actual) const + { + const bool status = (actual == edn); + if(!status) + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n" + "expected:~~~%.*s~~~\n" + "actual:~~~%.*s~~~\n", + (int)ys.len, ys.str, + (int)edn.len, edn.str, + (int)actual.len, actual.str); + return status; + } + + #define _runtest(name, ...) \ + do { \ + printf("[RUN ] %s ... \n", #name); \ + TestResult tr_ = name(__VA_ARGS__); \ + tr.add(tr_); \ + printf("[%s] %s\n", tr_?"OK ":"FAIL", #name); \ + } while(0) + #define CHECK(cond) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s", __FILE__, __LINE__, #cond); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + + TestResult test(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + _runtest(test_1_alloc, ); + _runtest(test_2_large_enough, ); + _runtest(test_3_too_small, ); + _runtest(test_4_nullptr, ); + _runtest(test_1_alloc_reuse, ryml2edn); + _runtest(test_2_large_enough_reuse, ryml2edn); + _runtest(test_3_too_small_reuse, ryml2edn); + _runtest(test_4_nullptr_reuse, ryml2edn); + return tr; + } + + // 1. simplest (but wasteful) first: alloc a new string + TestResult test_1_alloc_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + ResultScoped result; // frees when destroying + result.edn = ys2edn_alloc(ryml2edn, "ysfilename", + input.str, (size_type)input.len); + CHECK(testeq(c4::to_csubstr(result.edn))); + return tr; + } + TestResult test_1_alloc() const + { + Ys2EdnScoped lib; + return test_1_alloc_reuse(lib.ryml2edn); + } + + // 2. happy path: large-enough destination string + TestResult test_2_large_enough_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output; + output.resize(2 * edn.len); + size_type reqsize = ys2edn(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_2_large_enough() const + { + Ys2EdnScoped lib; + return test_2_large_enough_reuse(lib.ryml2edn); + } + + // less-happy path: destination string not large enough + TestResult test_3_too_small_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output = "?"; + size_type reqsize = ys2edn(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + CHECK(output != "?"); + output.resize(reqsize); + size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); + CHECK(getsize == reqsize); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_3_too_small() const + { + Ys2EdnScoped lib; + return test_3_too_small_reuse(lib.ryml2edn); + } + + // safe calling with nullptr + TestResult test_4_nullptr_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + size_type reqsize = ys2edn(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + return tr; + } + TestResult test_4_nullptr() const + { + Ys2EdnScoped lib; + return test_4_nullptr_reuse(lib.ryml2edn); + } +}; + + +const TestCase test_cases[] = { + // case ------------------------------ + {"say: 2 + 2", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "say"} +{:+ "=VAL", := "2 + 2"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {"a: 1", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "1"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {"𝄞: ✅", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "𝄞"} +{:+ "=VAL", := "✅"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {"[a, b, c]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "=VAL", := "c"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {"[a: b]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "-MAP"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {R"(--- !yamlscript/v0 +foo: ! +- {x: y} +- [x, y] +- foo +- 'foo' +- "foo" +- | + foo +- > + foo +- [1, 2, true, false, null] +- &anchor-1 !tag-1 foobar +--- +another: doc +)", + R"(( +{:+ "+MAP", :! "yamlscript/v0"} +{:+ "=VAL", := "foo"} +{:+ "+SEQ", :! ""} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-MAP"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-SEQ"} +{:+ "=VAL", := "foo"} +{:+ "=VAL", :' "foo"} +{:+ "=VAL", :$ "foo"} +{:+ "=VAL", :| "foo\n"} +{:+ "=VAL", :> "foo\n"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "1"} +{:+ "=VAL", := "2"} +{:+ "=VAL", := "true"} +{:+ "=VAL", := "false"} +{:+ "=VAL", := "null"} +{:+ "-SEQ"} +{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} +{:+ "-SEQ"} +{:+ "-MAP"} +{:+ "-DOC"} +{:+ "+DOC"} +{:+ "+MAP"} +{:+ "=VAL", := "another"} +{:+ "=VAL", := "doc"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)"} +}; + + +int main() +{ + Ys2EdnScoped ys2edn; + TestResult total = {}; + size_t failed_cases = {}; + size_t num_cases = C4_COUNTOF(test_cases); + for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) + { + printf("-----------------------------------------\n" + "case %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); + const TestResult tr = test_cases[i].test(ys2edn.ryml2edn); + total.add(tr); + failed_cases += (!tr); + printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + } + printf("%u/%u assertions failed\n", total.num_failed_assertions, total.num_assertions); + printf("%u/%u tests failed\n", total.num_failed_tests, total.num_tests); + printf("%zu/%zu cases failed\n", failed_cases, num_cases); + return total ? 0 : -1; +} diff --git a/rapidyaml/pom.xml b/rapidyaml/pom.xml new file mode 100644 index 000000000..3efb8c7c4 --- /dev/null +++ b/rapidyaml/pom.xml @@ -0,0 +1,115 @@ + + + + + rapidyaml + + 0.7.2 + + rapidyaml + + org.rapidyaml + + 4.0.0 + + + rapidyaml is a C++ library to parse and emit YAML, and do it fast. + + + https://rapidyaml.org + + + + MIT License + http://www.opensource.org/licenses/mit-license.php + repo + + + + + + clojars + Clojars repository + https://clojars.org/repo + + + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 3.8.1 + + + org.json + json + 20240205 + + + + + + + + + maven-clean-plugin + 3.1.0 + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + + + + + + maven-project-info-reports-plugin + + + + diff --git a/rapidyaml/rapidyaml.mk b/rapidyaml/rapidyaml.mk new file mode 100644 index 000000000..73e7bb913 --- /dev/null +++ b/rapidyaml/rapidyaml.mk @@ -0,0 +1,5 @@ +RAPIDYAML_TAG ?= v0.7.2 +RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_JAVA := \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java diff --git a/rapidyaml/src/README.md b/rapidyaml/src/README.md new file mode 100644 index 000000000..d920cfa6b --- /dev/null +++ b/rapidyaml/src/README.md @@ -0,0 +1,33 @@ +## Notes on JNI vs JNA +, packaging, loading, etc + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > JNI is almost always the fasted ffi option. I've heard good things about JNR performance. JNA is usually one of the slower ffi options, but can be sped up with direct mapping, https://github.com/java-native-access/jna/blob/master/www/DirectMapping.md. + + +## Notes on how to profile + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > Another tool to try to find the bottleneck is https://github.com/clojure-goes-fast/clj-async-profiler. The flamegraph might show some obvious performance issue (assuming the issue is on the jvm side). + + > Depending on how long a parse takes, I would recommend something like https://github.com/hugoduncan/criterium. time is not a good way to benchmark code unless it's a very slow function call. I would use the profiler to try and figure out where the bottleneck is. + + +## JNI examples + +- [full JNI example](https://github.com/mkowsiak/jnicookbook/tree/master/recipes/recipeNo031) +- [full JNI example with other non-JNI shared libraries](https://github.com/mkowsiak/jnicookbook/tree/master/recipes/recipeNo035) +- [another example linking with more libraries](https://www.dynamsoft.com/codepool/package-jni-shared-library-jar-file.html) + +- https://stackoverflow.com/questions/1611357/how-to-make-a-jar-file-that-includes-dll-files#comment1483970_1611367 + +## Notes on JNI - how to call c++ code from java + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > I'm surprised trying to pass a mutable byte buffer doesn't cause more issues. I think the recommended way to pass a byte buffer to native is with http://java-native-access.github.io/jna/5.13.0/javadoc/com/sun/jna/Memory.html and you can get the string with .getString. + + > > But I was thinking whether it is possible/practical/advisable (in terms of speed) to build the Clojure dictionary directly in the C++ code. Currently the C++ code is providing an EDN markup string that is later parsed in Clojure. Assuming a large dictionary of say ~40k entries, would it be possible to call native clojure/java JNI functions to build the final structure instead of creating the intermediate EDN? Would that be a gain? Would the ~40k calls to JNI end up costing too much? + > This isn't something I've tried before, so take it with a grain of salt, but you have at least a few options with different tradeoffs. All the collection types in clojure are based on protocols/interfaces, so it would be possible to just return a pointer, with no copying, and wrap it proxy that implements all the relevant interfaces for maps/lists/etc. When JVM code asks for a value from a map or element from list, you produce the JVM value for numbers/strings or you return another proxy pointer if it's a collection. You might still have to make a copy to return a string value. If you're returning large values that you expect will only be partially read or read only once, then lazily producing jvm values might be a win. If you expect the large value to be completely read multiple times, then it could be potentially faster to just convert the full data structure to a JVM value. There's also intermediate options where you do some of the work upfront, and do some of the work lazily. Granularity will also affect memory usage. You probably don't want some scenario where someone parses a giant blob and keeps only a small part, but still has to hold the giant value in memory until the small part gets reclaimed.I don't have a good answer for you here. My intuition is that your approach of building the final data structure in c++ is probably a good idea, but I don't really have the experience to say for sure.This type of question might get a better answer in #data-science. I think they similar issues with dealing with large datasets that are partially processed in native code. They've also built deep integrations with python via https://github.com/clj-python/libpython-clj where I think they've run into similar problems. diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java new file mode 100644 index 000000000..ccf7d4370 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -0,0 +1,61 @@ +package org.rapidyaml; + +import org.rapidyaml.YamlParseErrorException; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; + +/** + * Interface with the shared librapidyaml library + * + */ +public class Rapidyaml { + public static String RAPIDYAML_VERSION = "0.7.2"; + + private native long ys2edn_init(); + private native void ys2edn_destroy(long ryml2edn); + private native int ys2edn_parse(long ryml2edn, String filename, + byte[] ys, int ys_length, + byte[] edn, int edn_length); + private native int ys2edn_retry_get( + long ryml2edn, byte[] edn, int edn_size + ); + + private final long ryml2edn; + + public Rapidyaml() { + String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; + System.loadLibrary(library_name); + this.ryml2edn = this.ys2edn_init(); + } + + // Likely bad idea to implement finalize: + // + // https://stackoverflow.com/questions/158174/why-would-you-ever-implement-finalize + // + protected void finalize() throws Throwable { + try { + this.ys2edn_destroy(this.ryml2edn); + } + finally { + super.finalize(); + } + } + + public String parseYS(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + String filename = "yamlscript"; // fixme + byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); + int edn_size = 10 * src.length; + byte[] edn = new byte[edn_size]; + int required_size = ys2edn_parse(this.ryml2edn, filename, src, src.length, edn, edn_size); + if(required_size > edn_size) { + edn_size = required_size; + edn = new byte[edn_size]; + required_size = ys2edn_retry_get(this.ryml2edn, edn, edn_size); + if(required_size != edn_size) { + throw new RuntimeException("inconsistent size"); + } + } + String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); + return ret; + } +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java b/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java new file mode 100644 index 000000000..d5d4730df --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java @@ -0,0 +1,16 @@ +package org.rapidyaml; + +// https://www.baeldung.com/java-new-custom-exception +public class YamlParseErrorException extends Exception +{ + public final int offset; + public final int line; + public final int column; + public YamlParseErrorException(int offset_, int line_, int column_, String msg) + { + super(msg); + offset = offset_; + line = line_; + column = column_; + } +} diff --git a/rapidyaml/src/site/site.xml b/rapidyaml/src/site/site.xml new file mode 100644 index 000000000..27de8ef17 --- /dev/null +++ b/rapidyaml/src/site/site.xml @@ -0,0 +1,26 @@ + + + + + rapidyaml + https://maven.apache.org/images/apache-maven-project.png + https://www.apache.org/ + + + + https://maven.apache.org/images/maven-logo-black-on-white.png + https://maven.apache.org/ + + + + org.apache.maven.skins + maven-fluido-skin + 1.7 + + + + + + + diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java new file mode 100644 index 000000000..48e9eaf83 --- /dev/null +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -0,0 +1,163 @@ +package org.rapidyaml; + +import org.rapidyaml.*; +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class RapidyamlTest extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public RapidyamlTest(String testName) + { + super(testName); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite(RapidyamlTest.class); + } + + private void testEdn_(String ys, String expected) + { + Rapidyaml rapidyaml = new Rapidyaml(); + try { + String actual = rapidyaml.parseYS(ys); + try { + assertEquals(expected.length(), actual.length()); + assertEquals(expected, actual); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + catch (YamlParseErrorException e) { + fail("parse error:\n" + e.getMessage()); + } + } + + public void testPlainMap() + { + testEdn_( + "a: 1" + , + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"a\"}\n" + + "{:+ \"=VAL\", := \"1\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + } + + public void testUtf8() + { + testEdn_( + "𝄞: ✅" + , + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"𝄞\"}\n" + + "{:+ \"=VAL\", := \"✅\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + } + + public void testLargeCase() + { + testEdn_( + "foo: !\n" + + "- {x: y}\n" + + "- [x, y]\n" + + "- foo\n" + + "- 'foo'\n" + + "- \"foo\"\n" + + "- |\n" + + " foo\n" + + "- >\n" + + " foo\n" + + "- [1, 2, true, false, null]\n" + + "- &anchor-1 !tag-1 foobar\n" + + "---\n" + + "another: doc\n" + , + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"foo\"}\n" + + "{:+ \"+SEQ\", :! \"\"}\n" + + "{:+ \"+MAP\", :flow true}\n" + + "{:+ \"=VAL\", := \"x\"}\n" + + "{:+ \"=VAL\", := \"y\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"+SEQ\", :flow true}\n" + + "{:+ \"=VAL\", := \"x\"}\n" + + "{:+ \"=VAL\", := \"y\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"=VAL\", := \"foo\"}\n" + + "{:+ \"=VAL\", :' \"foo\"}\n" + + "{:+ \"=VAL\", :$ \"foo\"}\n" + + "{:+ \"=VAL\", :| \"foo\\n\"}\n" + + "{:+ \"=VAL\", :> \"foo\\n\"}\n" + + "{:+ \"+SEQ\", :flow true}\n" + + "{:+ \"=VAL\", := \"1\"}\n" + + "{:+ \"=VAL\", := \"2\"}\n" + + "{:+ \"=VAL\", := \"true\"}\n" + + "{:+ \"=VAL\", := \"false\"}\n" + + "{:+ \"=VAL\", := \"null\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"=VAL\", :& \"anchor-1\", :! \"tag-1\", := \"foobar\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + "{:+ \"+DOC\"}\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"another\"}\n" + + "{:+ \"=VAL\", := \"doc\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + } + + public void testFailure() + { + Rapidyaml rapidyaml = new Rapidyaml(); + String ys = ": : : :"; + boolean gotit = false; + try { + rapidyaml.parseYS(ys); + } + catch(YamlParseErrorException e) { + gotit = true; + assertEquals(2, e.offset); + assertEquals(1, e.line); + assertEquals(3, e.column); + assertTrue(e.getMessage() != null); + assertFalse(e.getMessage().isEmpty()); + } + catch(RuntimeException e) { + fail("wrong exception type"); + } + catch(Exception e) { + fail("wrong exception type"); + } + assertTrue(gotit); + } +} diff --git a/util/RYS b/util/RYS new file mode 100755 index 000000000..1d53e01a0 --- /dev/null +++ b/util/RYS @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +( + root=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd -P) + root_ry=$root/rapidyaml + make --no-print-directory -C "$root_ry" build + export YS_PARSER_RAPIDYAML=1 + export LD_LIBRARY_PATH=$root_ry/native + bin=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd -P) + exec "$bin/YS" "$@" +) diff --git a/ys/Makefile b/ys/Makefile index 8044f4cbf..d80f4a20b 100644 --- a/ys/Makefile +++ b/ys/Makefile @@ -54,7 +54,7 @@ install: build install -m 755 $(YAMLSCRIPT_CLI_BIN_BASH) \ $(PREFIX)/bin/ -jar: $(YAMLSCRIPT_CLI_JAR_PATH) +jar: $(YAMLSCRIPT_CLI_JAR_PATH) $(RAPIDYAML_INSTALLED) @: test: test-unit @@ -70,7 +70,7 @@ test-run: $(BPAN_LOCAL) build $(BPAN_LOCAL): git clone --depth=1 $(BPAN_REPO_URL) $@ -$(YAMLSCRIPT_CLI_BIN): $(YAMLSCRIPT_CLI_JAR_PATH) +$(YAMLSCRIPT_CLI_BIN): $(YAMLSCRIPT_CLI_JAR_PATH) $(RAPIDYAML_INSTALLED) ifndef YS_BIN_DIR ifneq (true,$(LIBZ)) $(error *** \ @@ -106,6 +106,9 @@ $(YAMLSCRIPT_CLI_JAR_PATH): $(LEIN) $(YAMLSCRIPT_CORE_INSTALLED) $(YAMLSCRIPT_CL $< uberjar endif +$(RAPIDYAML_INSTALLED): + $(MAKE) -C $(RAPIDYAML) $@ + Dockerfile:: $(COMMON) Makefile cat \ $ Date: Thu, 29 Aug 2024 09:41:49 +0100 Subject: [PATCH 02/21] rapidyaml: Enable C++ tests, clean native/Makefile --- rapidyaml/Makefile | 2 + rapidyaml/native/CMakeLists.txt | 5 ++ rapidyaml/native/Makefile | 22 ++++---- rapidyaml/native/rapidyaml_edn.hpp | 6 +-- rapidyaml/native/rapidyaml_test.cpp | 83 +++++++++++------------------ 5 files changed, 51 insertions(+), 67 deletions(-) diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index 5d455f5c6..ce5fb92a1 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -31,9 +31,11 @@ jar: $(RAPIDYAML_JAR) install:: $(RAPIDYAML_INSTALLED) test:: build $(JAVA_INSTALLED) + $(MAKE) -C native $@ mvn $@ test-x: build $(JAVA_INSTALLED) + $(MAKE) -C native $@ mvn -X -e test clean:: diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 2bdd95182..877eeb4fd 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -38,3 +38,8 @@ target_include_directories(rapidyaml PUBLIC ${JNI_INCLUDE_DIRS}) add_executable(rapidyaml-test rapidyaml_test.cpp) target_link_libraries(rapidyaml-test rapidyaml) +add_custom_target(rapidyaml-test-run + DEPENDS rapidyaml-test + COMMAND $ + COMMENT "running C++ tests" +) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index 876509c10..d1767be59 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -25,6 +25,10 @@ CMK_FLAGS := CMK_FLAGS += -D YS2EDN_TIMED=$(TIMED) CMK_FLAGS += -D BUILD_SHARED_LIBS=$(SHARED) +BDIR := rapidyaml-build + +CMK_BUILD := cmake --build $(BDIR) --parallel --verbose + #------------------------------------------------------------------------------ default:: @@ -32,9 +36,8 @@ default:: build:: $(RAPIDYAML_SO) test:: build - cmake --build rapidyaml-build \ - --parallel --target rapidyaml-test - ./rapidyaml-build/rapidyaml-test + $(CMK_BUILD) --target rapidyaml-test-run + ./$(BDIR)/rapidyaml-test rapidyaml: mkdir -p $@ @@ -47,7 +50,7 @@ rapidyaml: clean:: $(RM) librapidyaml.* $(RM) $(RAPIDYAML_HPP) - $(RM) -r rapidyaml-build + $(RM) -r $(BDIR) $(RM) -r rapidyaml-install realclean:: clean @@ -56,16 +59,13 @@ realclean:: clean #------------------------------------------------------------------------------ $(RAPIDYAML_SO): $(RAPIDYAML_SO_DEPS) - cmake -S . -B rapidyaml-build \ - $(CMK_FLAGS) - cmake --build rapidyaml-build \ - --parallel --verbose --target rapidyaml - mv rapidyaml-build/*.so $@ + cmake -S . -B $(BDIR) $(CMK_FLAGS) + $(CMK_BUILD) --target rapidyaml + mv $(BDIR)/*.so $@ ln -fs $@ librapidyaml.so $(RAPIDYAML_H): $(RAPIDYAML_JAVA) javac -h . $^ $(RAPIDYAML_HPP): rapidyaml - $(PYTHON) rapidyaml/tools/amalgamate.py \ - $(RAPIDYAML_AMALGAMATE_OPTS) > $@ + $(PYTHON) rapidyaml/tools/amalgamate.py $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/native/rapidyaml_edn.hpp b/rapidyaml/native/rapidyaml_edn.hpp index 69ca2f077..28ab0d370 100644 --- a/rapidyaml/native/rapidyaml_edn.hpp +++ b/rapidyaml/native/rapidyaml_edn.hpp @@ -52,9 +52,9 @@ RYML_EXPORT Ryml2Edn *ys2edn_init(); RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn); /** Parse YAML, and return corresponding EDN. Return the number of - * characters needed for edn. Check if the returned size is larger - * than edn_size. If it is, call ys_retry_get() can be called - * afterwards to extract the EDN. */ + * characters needed for edn. The caller must check if the returned + * size is larger than edn_size. If it is, call ys_retry_get() can be + * called afterwards to extract the EDN. */ RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, const char *filename, char *ys, size_type ys_size, diff --git a/rapidyaml/native/rapidyaml_test.cpp b/rapidyaml/native/rapidyaml_test.cpp index 7c2e15f72..50ca878de 100644 --- a/rapidyaml/native/rapidyaml_test.cpp +++ b/rapidyaml/native/rapidyaml_test.cpp @@ -12,12 +12,6 @@ struct Ys2EdnScoped }; -struct ResultScoped -{ - char *edn = nullptr; - ~ResultScoped() { if(edn) ys2edn_free(edn); } -}; - struct TestResult { uint32_t num_assertions; @@ -34,6 +28,7 @@ struct TestResult } }; + struct TestCase { csubstr ys; @@ -55,10 +50,10 @@ struct TestCase #define _runtest(name, ...) \ do { \ - printf("[RUN ] %s ... \n", #name); \ + printf("[ RUN ] %s ... \n", #name); \ TestResult tr_ = name(__VA_ARGS__); \ tr.add(tr_); \ - printf("[%s] %s\n", tr_?"OK ":"FAIL", #name); \ + printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ } while(0) #define CHECK(cond) \ do { \ @@ -73,68 +68,48 @@ struct TestCase TestResult test(Ryml2Edn *ryml2edn) const { TestResult tr = {}; - _runtest(test_1_alloc, ); - _runtest(test_2_large_enough, ); - _runtest(test_3_too_small, ); - _runtest(test_4_nullptr, ); - _runtest(test_1_alloc_reuse, ryml2edn); - _runtest(test_2_large_enough_reuse, ryml2edn); - _runtest(test_3_too_small_reuse, ryml2edn); - _runtest(test_4_nullptr_reuse, ryml2edn); + _runtest(test_large_enough, ); + _runtest(test_too_small, ); + _runtest(test_nullptr, ); + _runtest(test_large_enough_reuse, ryml2edn); + _runtest(test_too_small_reuse, ryml2edn); + _runtest(test_nullptr_reuse, ryml2edn); return tr; } - // 1. simplest (but wasteful) first: alloc a new string - TestResult test_1_alloc_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - ResultScoped result; // frees when destroying - result.edn = ys2edn_alloc(ryml2edn, "ysfilename", - input.str, (size_type)input.len); - CHECK(testeq(c4::to_csubstr(result.edn))); - return tr; - } - TestResult test_1_alloc() const - { - Ys2EdnScoped lib; - return test_1_alloc_reuse(lib.ryml2edn); - } - - // 2. happy path: large-enough destination string - TestResult test_2_large_enough_reuse(Ryml2Edn *ryml2edn) const + // happy path: large-enough destination string + TestResult test_large_enough_reuse(Ryml2Edn *ryml2edn) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); std::string output; output.resize(2 * edn.len); - size_type reqsize = ys2edn(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - &output[0], (size_type)output.size()); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); CHECK(reqsize == edn.len+1); CHECK(reqsize != 0); output.resize(reqsize - 1u); CHECK(testeq(c4::to_csubstr(output))); return tr; } - TestResult test_2_large_enough() const + TestResult test_large_enough() const { Ys2EdnScoped lib; - return test_2_large_enough_reuse(lib.ryml2edn); + return test_large_enough_reuse(lib.ryml2edn); } // less-happy path: destination string not large enough - TestResult test_3_too_small_reuse(Ryml2Edn *ryml2edn) const + TestResult test_too_small_reuse(Ryml2Edn *ryml2edn) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); std::string output = "?"; - size_type reqsize = ys2edn(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); CHECK(reqsize == edn.len+1); CHECK(reqsize != 0); CHECK(output != "?"); @@ -145,33 +120,35 @@ struct TestCase CHECK(testeq(c4::to_csubstr(output))); return tr; } - TestResult test_3_too_small() const + TestResult test_too_small() const { Ys2EdnScoped lib; - return test_3_too_small_reuse(lib.ryml2edn); + return test_too_small_reuse(lib.ryml2edn); } // safe calling with nullptr - TestResult test_4_nullptr_reuse(Ryml2Edn *ryml2edn) const + TestResult test_nullptr_reuse(Ryml2Edn *ryml2edn) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); - size_type reqsize = ys2edn(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - nullptr, 0); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); CHECK(reqsize == edn.len+1); CHECK(reqsize != 0); return tr; } - TestResult test_4_nullptr() const + TestResult test_nullptr() const { Ys2EdnScoped lib; - return test_4_nullptr_reuse(lib.ryml2edn); + return test_nullptr_reuse(lib.ryml2edn); } }; +//----------------------------------------------------------------------------- + const TestCase test_cases[] = { // case ------------------------------ {"say: 2 + 2", From 07538c86b5b3f5762161129e485aa952dfec059c Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 29 Aug 2024 11:33:15 +0100 Subject: [PATCH 03/21] rapidyaml: Reserve memory --- rapidyaml/native/rapidyaml_edn.cpp | 3 ++- rapidyaml/native/rapidyaml_edn_handler.hpp | 25 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/rapidyaml/native/rapidyaml_edn.cpp b/rapidyaml/native/rapidyaml_edn.cpp index 8f5bcfca9..59082f16f 100644 --- a/rapidyaml/native/rapidyaml_edn.cpp +++ b/rapidyaml/native/rapidyaml_edn.cpp @@ -75,8 +75,9 @@ RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); { - TIMED_SECTION("reset"); + TIMED_SECTION("reset + reserve"); ryml2edn->reset(); + ryml2edn->m_handler.reserve(ys_size > edn_size ? 3 * ys_size : edn_size, 256u); } { TIMED_SECTION("parse_in_place"); diff --git a/rapidyaml/native/rapidyaml_edn_handler.hpp b/rapidyaml/native/rapidyaml_edn_handler.hpp index 973c52f57..a50ee026e 100644 --- a/rapidyaml/native/rapidyaml_edn_handler.hpp +++ b/rapidyaml/native/rapidyaml_edn_handler.hpp @@ -82,13 +82,15 @@ struct EventHandlerEdn : public EventHandlerStackresult.reserve(edn_size); + for(size_t i = 0; i < m_val_buffers.size(); ++i) + { + int sz = edn_size / (int(1) << (uint32_t)i); + sz = sz >= 128 ? sz : 128; + m_val_buffers[i].result.reserve((size_t)sz); + } + m_arena.reserve(arena_size); + } + /** @} */ public: From 9a0db4582aad2b8888507b6a9d34522df6c2eaa4 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 5 Jan 2025 16:18:35 +0000 Subject: [PATCH 04/21] rapidyaml: Sysclean for jpmag --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index a62b0fbc5..5194d66b8 100644 --- a/Makefile +++ b/Makefile @@ -295,6 +295,9 @@ sysclean: realclean ifeq (ingy,$(USER)) $(RM) -r $(HOME)/.m2 endif +ifeq (jpmag,$(USER)) + $(RM) -r $(HOME)/.m2 +endif $(DOCKER_BUILD): docker-build: $(DOCKER_BUILD) From f9c5bbfff2c75bddb08ec033d51d189718d3bef9 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 5 Jan 2025 20:15:48 +0000 Subject: [PATCH 05/21] rapidyaml: Compile both shared and static --- common/vars.mk | 13 +++++++++++-- rapidyaml/Makefile | 8 +++++--- rapidyaml/native/Makefile | 41 ++++++++++++++++++++------------------- rapidyaml/rapidyaml.mk | 5 ----- 4 files changed, 37 insertions(+), 30 deletions(-) delete mode 100644 rapidyaml/rapidyaml.mk diff --git a/common/vars.mk b/common/vars.mk index 6aa80d5d2..04d7747ea 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -48,12 +48,15 @@ ifneq (,$(findstring linux,$(ostype))) GCC := gcc -std=gnu99 -fPIC -shared SO := so DY := + DOTLIB := a else ifneq (,$(findstring darwin,$(ostype))) IS_MACOS := true GCC := gcc -dynamiclib SO := dylib DY := DY + DOTLIB := a else + DOTLIB := lib $(error Unsupported OSTYPE: $(ostype)) endif @@ -226,9 +229,15 @@ RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz RAPIDYAML := $(ROOT)/rapidyaml RAPIDYAML_VERSION := 0.7.2 +RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) +RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_TIMED := 0 +RAPIDYAML_JAVA := \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java +RAPIDYAML_SO := $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_LIB := $(ROOT)/rapidyaml/native/librapidyaml.$(DOTLIB) RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar -RAPIDYAML_SO := \ - $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) RAPIDYAML_INSTALLED_DIR := \ $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)/ RAPIDYAML_INSTALLED := \ diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index ce5fb92a1..c0d7649a2 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -1,7 +1,6 @@ include ../common/base.mk include $(COMMON)/clojure.mk include $(COMMON)/java.mk -include ./rapidyaml.mk RAPIDYAML_JAR_DEPS := \ $(wildcard \ @@ -19,11 +18,12 @@ RAPIDYAML_CLASS := \ #------------------------------------------------------------------------------ default:: echo $(RAPIDYAML_SO) + echo $(RAPIDYAML_LIB) x: YS -ce 'foo: var' -build:: $(RAPIDYAML_SO) +build:: $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @: jar: $(RAPIDYAML_JAR) @@ -39,7 +39,7 @@ test-x: build $(JAVA_INSTALLED) mvn -X -e test clean:: - $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) + $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) $(RM) -r reports/ target/ $(MAKE) -C native $@ @@ -53,6 +53,8 @@ sysclean:: #------------------------------------------------------------------------------ $(RAPIDYAML_SO): $(MAKE) -C native $@ +$(RAPIDYAML_LIB): + $(MAKE) -C native $@ $(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED) mvn install diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index d1767be59..a11f5ccd2 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -2,13 +2,12 @@ include ../../common/base.mk include $(COMMON)/clojure.mk include $(COMMON)/java.mk include $(COMMON)/python.mk -include ../rapidyaml.mk RAPIDYAML_H := org_rapidyaml_Rapidyaml.h RAPIDYAML_HPP := rapidyaml_all.hpp RAPIDYAML_AMALGAMATE_OPTS := --stl -RAPIDYAML_SO_DEPS := \ +RAPIDYAML_DEPS := \ Makefile \ CMakeLists.txt \ $(JAVA_HOME) \ @@ -16,28 +15,19 @@ RAPIDYAML_SO_DEPS := \ $(RAPIDYAML_HPP) \ $(wildcard ./*pp) \ -# TODO maybe change to static library! -# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni -SHARED := 1 -TIMED := 0 - CMK_FLAGS := -CMK_FLAGS += -D YS2EDN_TIMED=$(TIMED) -CMK_FLAGS += -D BUILD_SHARED_LIBS=$(SHARED) +CMK_FLAGS += -D YS2EDN_TIMED=$(RAPIDYAML_TIMED) BDIR := rapidyaml-build -CMK_BUILD := cmake --build $(BDIR) --parallel --verbose +CMK_BUILD := #------------------------------------------------------------------------------ default:: -build:: $(RAPIDYAML_SO) - -test:: build - $(CMK_BUILD) --target rapidyaml-test-run - ./$(BDIR)/rapidyaml-test +build:: build-static build-shared +test: test-static test-shared rapidyaml: mkdir -p $@ @@ -58,14 +48,25 @@ realclean:: clean #------------------------------------------------------------------------------ -$(RAPIDYAML_SO): $(RAPIDYAML_SO_DEPS) - cmake -S . -B $(BDIR) $(CMK_FLAGS) - $(CMK_BUILD) --target rapidyaml - mv $(BDIR)/*.so $@ +build-static: $(RAPIDYAML_LIB) +test-static: build-static + cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml-test-run +$(RAPIDYAML_LIB): $(RAPIDYAML_DEPS) + cmake -S . -B $(BDIR)-static $(CMK_FLAGS) -D BUILD_SHARED_LIBS=OFF + cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml + cp -fv $(BDIR)-static/*.a $@ + +build-shared: $(RAPIDYAML_SO) +test-shared: build-shared + cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml-test-run +$(RAPIDYAML_SO): $(RAPIDYAML_DEPS) + cmake -S . -B $(BDIR)-shared $(CMK_FLAGS) -D BUILD_SHARED_LIBS=ON + cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml + cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so $(RAPIDYAML_H): $(RAPIDYAML_JAVA) - javac -h . $^ + javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work $(RAPIDYAML_HPP): rapidyaml $(PYTHON) rapidyaml/tools/amalgamate.py $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/rapidyaml.mk b/rapidyaml/rapidyaml.mk deleted file mode 100644 index 73e7bb913..000000000 --- a/rapidyaml/rapidyaml.mk +++ /dev/null @@ -1,5 +0,0 @@ -RAPIDYAML_TAG ?= v0.7.2 -RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml -RAPIDYAML_JAVA := \ - $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ - $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java From 4030e8b8aa843d9ff13125c4730ce40fb4e008a3 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 15 Sep 2024 13:36:58 +0100 Subject: [PATCH 06/21] rapidyaml: Add parsing of ys to integer-based events Adds a parser producing integer events, and integers signifying strings by indexing into the parsed YS string. Each event is provided as an integer bitmask. The event bits are defined in both the C++ side (ysparse_evt_handler.hpp) and the Java side (org.rapidyaml.Evt), and they need to stay consistent on both ends. When a string is associated with an event, it is provided as an integer offset and length. For example, the YAML `say: 2 + 2` produces the following sequence of integers: ```c++ BSTR, BDOC, VAL|BMAP|BLCK, KEY|SCLR|PLAI, 0, 3, // "say" VAL|SCLR|PLAI, 5, 5, // "2 + 2" EMAP, EDOC, ESTR, ``` Note that the scalar events, ie "say" and "2 + 2", are followed each by two extra integers encoding the offset and length of the scalar's string. These two extra integers are present whenever the event has any of the bits SCLR, ALIA, ANCH or TAG. For ease of use, there is a bitmask HAS_STR, which enables quick testing by a simple `flags & HAS_STR`. Also, where a string requires filtering, the parser filters it in-place in the input string, and returns the extra integers pertaining to the resulting filtered string. The existing EDN-producing parser was not removed, and the EVT-producing parser was added both on C++ and on the Java JNI bridge. Tests were enlarged to cover the new event parsing, both for C++ and Java. To benefit from this approach, the YS side must implement a mechanism to convert the int event sequence into its internal data-structure, using the symbols in the class org.rapidyaml.Evt. Other changes: - rapidyaml->ysparse: start renaming the C++ library providing the JNI. ysparse is a more accurate description of what the library does. - Directly use only the required rapidyaml source files instead of amalgamating into a single header. - Makefile: - Add target to generate the JNI header - Improve target dependencies --- common/vars.mk | 2 + rapidyaml/Makefile | 17 +- rapidyaml/native/.gitignore | 5 +- rapidyaml/native/CMakeLists.txt | 68 +- rapidyaml/native/Makefile | 29 +- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 58 +- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 24 + rapidyaml/native/rapidyaml_test.cpp | 282 -------- .../{rapidyaml_edn.cpp => ysparse_edn.cpp} | 7 +- .../{rapidyaml_edn.hpp => ysparse_edn.hpp} | 15 +- ...dn_handler.cpp => ysparse_edn_handler.cpp} | 17 +- ...dn_handler.hpp => ysparse_edn_handler.hpp} | 126 ++-- rapidyaml/native/ysparse_evt.cpp | 90 +++ rapidyaml/native/ysparse_evt.hpp | 108 +++ rapidyaml/native/ysparse_evt_handler.cpp | 12 + rapidyaml/native/ysparse_evt_handler.hpp | 680 ++++++++++++++++++ rapidyaml/native/ysparse_test.cpp | 675 +++++++++++++++++ .../src/main/java/org/rapidyaml/Evt.java | 37 + .../main/java/org/rapidyaml/Rapidyaml.java | 19 +- .../java/org/rapidyaml/RapidyamlTest.java | 245 ++++++- 20 files changed, 2081 insertions(+), 435 deletions(-) delete mode 100644 rapidyaml/native/rapidyaml_test.cpp rename rapidyaml/native/{rapidyaml_edn.cpp => ysparse_edn.cpp} (96%) rename rapidyaml/native/{rapidyaml_edn.hpp => ysparse_edn.hpp} (83%) rename rapidyaml/native/{rapidyaml_edn_handler.cpp => ysparse_edn_handler.cpp} (94%) rename rapidyaml/native/{rapidyaml_edn_handler.hpp => ysparse_edn_handler.hpp} (85%) create mode 100644 rapidyaml/native/ysparse_evt.cpp create mode 100644 rapidyaml/native/ysparse_evt.hpp create mode 100644 rapidyaml/native/ysparse_evt_handler.cpp create mode 100644 rapidyaml/native/ysparse_evt_handler.hpp create mode 100644 rapidyaml/native/ysparse_test.cpp create mode 100644 rapidyaml/src/main/java/org/rapidyaml/Evt.java diff --git a/common/vars.mk b/common/vars.mk index 04d7747ea..acccf60af 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -231,6 +231,8 @@ RAPIDYAML := $(ROOT)/rapidyaml RAPIDYAML_VERSION := 0.7.2 RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_BUILD_TYPE := Release +RAPIDYAML_DBG := 0 RAPIDYAML_TIMED := 0 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index c0d7649a2..e29ba8c02 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -10,9 +10,7 @@ RAPIDYAML_JAR_DEPS := \ pom.xml \ ) -RAPIDYAML_CLASS := \ - src/main/java/org/rapidyaml/Rapidyaml.class \ - src/main/java/org/rapidyaml/YamlParseErrorException.class +RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class) #------------------------------------------------------------------------------ @@ -23,7 +21,7 @@ default:: x: YS -ce 'foo: var' -build:: $(RAPIDYAML_SO) $(RAPIDYAML_LIB) +build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @: jar: $(RAPIDYAML_JAR) @@ -51,7 +49,7 @@ sysclean:: #------------------------------------------------------------------------------ -$(RAPIDYAML_SO): +$(RAPIDYAML_SO): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ $(RAPIDYAML_LIB): $(MAKE) -C native $@ @@ -63,9 +61,12 @@ $(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED) jar -cvf $@ $< $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) - # this doesn't work: - #javac $< - # ... but this does: + @# this doesn't work: + @#javac $< + @# ... but this does: javac $(RAPIDYAML_JAVA) +$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) + $(MAKE) -C native $@ + crl: $(RAPIDYAML_CLASS) diff --git a/rapidyaml/native/.gitignore b/rapidyaml/native/.gitignore index 226243a92..8e53ad039 100644 --- a/rapidyaml/native/.gitignore +++ b/rapidyaml/native/.gitignore @@ -1,4 +1,5 @@ /rapidyaml/ -/rapidyaml-build/ -/rapidyaml_all.hpp +/_build/ /librapidyaml.* +/.cache +/compile_commands.json diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 877eeb4fd..61fd11e3c 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -1,45 +1,69 @@ cmake_minimum_required(VERSION 3.12) -project(rapidyaml - DESCRIPTION "rapidyaml -> yamlscript" +project(ysparse + DESCRIPTION "ysparse: rapidyaml -> yamlscript" HOMEPAGE_URL "https://github.com/biojppm/rapidyaml -> https://github.com/yaml/yamlscript" LANGUAGES CXX) find_package(JNI REQUIRED) -option(YS2EDN_TIMED "add timings to sections" OFF) +option(YSPARSE_TIMED "add timings to sections" OFF) +option(YSPARSE_DBG "enable debug logs" OFF) if(UNIX) set(CMAKE_SHARED_LIBRARY_SUFFIX .so) endif() -add_library(rapidyaml +set(libname rapidyaml) # TODO rename to ysparse + +add_library(${libname} + # + # JNI bridge org_rapidyaml_Rapidyaml.h org_rapidyaml_Rapidyaml.cpp - rapidyaml_all.hpp - rapidyaml_edn_handler.hpp - rapidyaml_edn_handler.cpp - rapidyaml_edn.hpp - rapidyaml_edn.cpp + # + # ysparse files + ysparse_edn_handler.hpp + ysparse_edn_handler.cpp + ysparse_edn.hpp + ysparse_edn.cpp + ysparse_evt_handler.hpp + ysparse_evt_handler.cpp + ysparse_evt.hpp + ysparse_evt.cpp + # + # files required from rapidyaml + rapidyaml/src/c4/yml/common.cpp + rapidyaml/src/c4/yml/node_type.cpp + rapidyaml/src/c4/yml/parse.cpp + rapidyaml/src/c4/yml/tree.cpp + rapidyaml/src/c4/yml/tag.cpp + rapidyaml/src/c4/yml/reference_resolver.cpp + # + # files required from rapidyaml/ext/c4core + rapidyaml/ext/c4core/src/c4/base64.cpp + rapidyaml/ext/c4core/src/c4/error.cpp + rapidyaml/ext/c4core/src/c4/language.cpp + rapidyaml/ext/c4core/src/c4/utf.cpp ) -target_include_directories(rapidyaml PUBLIC +target_include_directories(${libname} PUBLIC ${CMAKE_CURRENT_LIST_DIR} - ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/test + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/ext/c4core/src ) -target_compile_definitions(rapidyaml PUBLIC +target_compile_definitions(${libname} PUBLIC RYML_WITH_TAB_TOKENS RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS - RYML_SINGLE_HEADER - $<$:YS2EDN_TIMED> + $<$:YSPARSE_TIMED> + $<$:RYML_DBG> ) -set_target_properties(rapidyaml PROPERTIES CXX_STANDARD 17) +set_target_properties(${libname} PROPERTIES CXX_STANDARD 17) -# target_link_libraries(rapidyaml PUBLIC JNI::JNI) -target_include_directories(rapidyaml PUBLIC ${JNI_INCLUDE_DIRS}) +target_include_directories(${libname} PUBLIC ${JNI_INCLUDE_DIRS}) -add_executable(rapidyaml-test rapidyaml_test.cpp) -target_link_libraries(rapidyaml-test rapidyaml) -add_custom_target(rapidyaml-test-run - DEPENDS rapidyaml-test - COMMAND $ +add_executable(${libname}-test ysparse_test.cpp) +target_link_libraries(${libname}-test ${libname}) +add_custom_target(${libname}-test-run + DEPENDS ${libname}-test + COMMAND $ COMMENT "running C++ tests" ) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index a11f5ccd2..91dd05ea8 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -3,24 +3,27 @@ include $(COMMON)/clojure.mk include $(COMMON)/java.mk include $(COMMON)/python.mk -RAPIDYAML_H := org_rapidyaml_Rapidyaml.h -RAPIDYAML_HPP := rapidyaml_all.hpp -RAPIDYAML_AMALGAMATE_OPTS := --stl +# TODO change to static library! +# https://www.graalvm.org/latest/reference-manual/native-image/guides/build-static-executables/ +# https://www.blog.akhil.cc/static-jni +# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni + +THIS_DIR := $(shell pwd) +BDIR := $(THIS_DIR)/_build/$(BUILD_TYPE)-shared$(SHARED)-timed$(TIMED)-dbg$(DBG) RAPIDYAML_DEPS := \ Makefile \ CMakeLists.txt \ $(JAVA_HOME) \ - $(RAPIDYAML_H) \ - $(RAPIDYAML_HPP) \ + $(RAPIDYAML_JNI_H) \ $(wildcard ./*pp) \ -CMK_FLAGS := -CMK_FLAGS += -D YS2EDN_TIMED=$(RAPIDYAML_TIMED) - -BDIR := rapidyaml-build - -CMK_BUILD := +CMK_ENV := +CMK_FLAGS := \ + -D CMAKE_BUILD_TYPE=$(RAPIDYAML_BUILD_TYPE) \ + -D YSPARSE_TIMED=$(RAPIDYAML_TIMED) \ + -D YSPARSE_DBG=$(RAPIDYAML_DBG) \ + -D CMAKE_EXPORT_COMPILE_COMMANDS=ON #------------------------------------------------------------------------------ @@ -39,7 +42,6 @@ rapidyaml: clean:: $(RM) librapidyaml.* - $(RM) $(RAPIDYAML_HPP) $(RM) -r $(BDIR) $(RM) -r rapidyaml-install @@ -67,6 +69,3 @@ $(RAPIDYAML_SO): $(RAPIDYAML_DEPS) $(RAPIDYAML_H): $(RAPIDYAML_JAVA) javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work - -$(RAPIDYAML_HPP): rapidyaml - $(PYTHON) rapidyaml/tools/amalgamate.py $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 4e57b87d8..63fa9efd1 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -1,5 +1,6 @@ #include -#include "./rapidyaml_edn.hpp" +#include "ysparse_edn.hpp" +#include "ysparse_evt.hpp" #include #ifndef _Included_org_rapidyaml_Rapidyaml @@ -50,6 +51,18 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) return (jlong)obj; } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) +{ + Ryml2Evt *obj = ys2evt_init(); + return (jlong)obj; +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_destroy @@ -61,6 +74,17 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) ys2edn_destroy((Ryml2Edn*)obj); } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_destroy + * Signature: (Ljava/lang/Object;)V + */ +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) +{ + ys2evt_destroy((Ryml2Evt*)obj); +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_parse @@ -93,6 +117,38 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, return rc; } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_parse + * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I + */ +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jbyteArray src, jint src_len, + jintArray dst, jint dst_len) +{ + jboolean src_is_copy, dst_is_copy; + jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); + int* dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + const char *filename = env->GetStringUTFChars(jfilename, 0); + int rc = 0; + try + { + rc = ys2evt_parse((Ryml2Evt*)obj, filename, + (char*)src_, src_len, + dst_, dst_len); + } + catch (Ryml2EvtParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + env->ReleaseByteArrayElements(src, src_, 0); + env->ReleaseIntArrayElements(dst, dst_, 0); + env->ReleaseStringUTFChars(jfilename, filename); + return rc; +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_retry_get diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index c73497636..daba52eff 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -39,6 +39,30 @@ JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get (JNIEnv *, jobject, jlong, jbyteArray, jint); +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init + (JNIEnv *, jobject); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy + (JNIEnv *, jobject, jlong); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_parse + * Signature: (JLjava/lang/String;[BI[II)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); + #ifdef __cplusplus } #endif diff --git a/rapidyaml/native/rapidyaml_test.cpp b/rapidyaml/native/rapidyaml_test.cpp deleted file mode 100644 index 50ca878de..000000000 --- a/rapidyaml/native/rapidyaml_test.cpp +++ /dev/null @@ -1,282 +0,0 @@ -#include - -using c4::csubstr; -using c4::substr; - - -struct Ys2EdnScoped -{ - Ryml2Edn *ryml2edn; - Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} - ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } -}; - - -struct TestResult -{ - uint32_t num_assertions; - uint32_t num_tests; - uint32_t num_failed_assertions; - uint32_t num_failed_tests; - operator bool() const { return num_failed_tests == 0; } - void add(TestResult const& that) - { - num_tests += 1 + that.num_tests; - num_assertions += that.num_assertions; - num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; - num_failed_assertions += that.num_failed_assertions; - } -}; - - -struct TestCase -{ - csubstr ys; - csubstr edn; - bool testeq(csubstr actual) const - { - const bool status = (actual == edn); - if(!status) - printf("------\n" - "FAIL:\n" - "input:~~~%.*s~~~\n" - "expected:~~~%.*s~~~\n" - "actual:~~~%.*s~~~\n", - (int)ys.len, ys.str, - (int)edn.len, edn.str, - (int)actual.len, actual.str); - return status; - } - - #define _runtest(name, ...) \ - do { \ - printf("[ RUN ] %s ... \n", #name); \ - TestResult tr_ = name(__VA_ARGS__); \ - tr.add(tr_); \ - printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ - } while(0) - #define CHECK(cond) \ - do { \ - bool pass = !!(cond); \ - ++tr.num_assertions; \ - if(!pass) { \ - printf("%s:%d: fail! %s", __FILE__, __LINE__, #cond); \ - ++tr.num_failed_assertions; \ - } \ - } while(0) - - TestResult test(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - _runtest(test_large_enough, ); - _runtest(test_too_small, ); - _runtest(test_nullptr, ); - _runtest(test_large_enough_reuse, ryml2edn); - _runtest(test_too_small_reuse, ryml2edn); - _runtest(test_nullptr_reuse, ryml2edn); - return tr; - } - - // happy path: large-enough destination string - TestResult test_large_enough_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output; - output.resize(2 * edn.len); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - &output[0], (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } - TestResult test_large_enough() const - { - Ys2EdnScoped lib; - return test_large_enough_reuse(lib.ryml2edn); - } - - // less-happy path: destination string not large enough - TestResult test_too_small_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output = "?"; - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - CHECK(output != "?"); - output.resize(reqsize); - size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); - CHECK(getsize == reqsize); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } - TestResult test_too_small() const - { - Ys2EdnScoped lib; - return test_too_small_reuse(lib.ryml2edn); - } - - // safe calling with nullptr - TestResult test_nullptr_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - nullptr, 0); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - return tr; - } - TestResult test_nullptr() const - { - Ys2EdnScoped lib; - return test_nullptr_reuse(lib.ryml2edn); - } -}; - - -//----------------------------------------------------------------------------- - -const TestCase test_cases[] = { - // case ------------------------------ - {"say: 2 + 2", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "say"} -{:+ "=VAL", := "2 + 2"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"a: 1", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "1"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"𝄞: ✅", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "𝄞"} -{:+ "=VAL", := "✅"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"[a, b, c]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "=VAL", := "c"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"[a: b]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "-MAP"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {R"(--- !yamlscript/v0 -foo: ! -- {x: y} -- [x, y] -- foo -- 'foo' -- "foo" -- | - foo -- > - foo -- [1, 2, true, false, null] -- &anchor-1 !tag-1 foobar ---- -another: doc -)", - R"(( -{:+ "+MAP", :! "yamlscript/v0"} -{:+ "=VAL", := "foo"} -{:+ "+SEQ", :! ""} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-MAP"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-SEQ"} -{:+ "=VAL", := "foo"} -{:+ "=VAL", :' "foo"} -{:+ "=VAL", :$ "foo"} -{:+ "=VAL", :| "foo\n"} -{:+ "=VAL", :> "foo\n"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "1"} -{:+ "=VAL", := "2"} -{:+ "=VAL", := "true"} -{:+ "=VAL", := "false"} -{:+ "=VAL", := "null"} -{:+ "-SEQ"} -{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} -{:+ "-SEQ"} -{:+ "-MAP"} -{:+ "-DOC"} -{:+ "+DOC"} -{:+ "+MAP"} -{:+ "=VAL", := "another"} -{:+ "=VAL", := "doc"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"} -}; - - -int main() -{ - Ys2EdnScoped ys2edn; - TestResult total = {}; - size_t failed_cases = {}; - size_t num_cases = C4_COUNTOF(test_cases); - for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) - { - printf("-----------------------------------------\n" - "case %zu/%zu ...\n" - "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); - const TestResult tr = test_cases[i].test(ys2edn.ryml2edn); - total.add(tr); - failed_cases += (!tr); - printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); - } - printf("%u/%u assertions failed\n", total.num_failed_assertions, total.num_assertions); - printf("%u/%u tests failed\n", total.num_failed_tests, total.num_tests); - printf("%zu/%zu cases failed\n", failed_cases, num_cases); - return total ? 0 : -1; -} diff --git a/rapidyaml/native/rapidyaml_edn.cpp b/rapidyaml/native/ysparse_edn.cpp similarity index 96% rename from rapidyaml/native/rapidyaml_edn.cpp rename to rapidyaml/native/ysparse_edn.cpp index 59082f16f..b074c4848 100644 --- a/rapidyaml/native/rapidyaml_edn.cpp +++ b/rapidyaml/native/ysparse_edn.cpp @@ -1,6 +1,4 @@ -#define RYML_SINGLE_HDR_DEFINE_NOW -#include -#include "rapidyaml_edn.hpp" +#include "ysparse_edn.hpp" #include namespace ryml { @@ -10,7 +8,7 @@ using namespace c4::yml; using namespace ryml; -#ifndef YS2EDN_TIMED +#ifndef YSPARSE_TIMED #define TIMED_SECTION(name) #else #include @@ -64,6 +62,7 @@ RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn) { TIMED_SECTION("ys2edn_destroy"); ryml2edn->~Ryml2Edn(); + _RYML_CB_FREE(get_callbacks(), ryml2edn, Ryml2Edn, 1); } RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, diff --git a/rapidyaml/native/rapidyaml_edn.hpp b/rapidyaml/native/ysparse_edn.hpp similarity index 83% rename from rapidyaml/native/rapidyaml_edn.hpp rename to rapidyaml/native/ysparse_edn.hpp index 28ab0d370..5a5878967 100644 --- a/rapidyaml/native/rapidyaml_edn.hpp +++ b/rapidyaml/native/ysparse_edn.hpp @@ -1,10 +1,9 @@ #pragma once -#ifndef RAPIDYAML_EVENTS_H -#define RAPIDYAML_EVENTS_H +#ifndef YSPARSE_EDN_HPP_ +#define YSPARSE_EDN_HPP_ #include -#include -#include "rapidyaml_edn_handler.hpp" +#include "ysparse_edn_handler.hpp" namespace ryml { using namespace c4; @@ -19,9 +18,9 @@ using size_type = int; struct RYML_EXPORT Ryml2Edn { - c4::yml::EventHandlerEdn::EventSink m_sink; - c4::yml::EventHandlerEdn m_handler; - c4::yml::ParseEngine m_parser; + ys::EventHandlerEdn::EventSink m_sink; + ys::EventHandlerEdn m_handler; + c4::yml::ParseEngine m_parser; Ryml2Edn() : m_sink() , m_handler(&m_sink) @@ -68,4 +67,4 @@ RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, } #endif -#endif /* RAPIDYAML_EVENTS_H */ +#endif /* YSPARSE_EDN_HPP_ */ diff --git a/rapidyaml/native/rapidyaml_edn_handler.cpp b/rapidyaml/native/ysparse_edn_handler.cpp similarity index 94% rename from rapidyaml/native/rapidyaml_edn_handler.cpp rename to rapidyaml/native/ysparse_edn_handler.cpp index 635f44601..ab295234e 100644 --- a/rapidyaml/native/rapidyaml_edn_handler.cpp +++ b/rapidyaml/native/ysparse_edn_handler.cpp @@ -1,18 +1,17 @@ -#ifndef RYML_SINGLE_HEADER +#include "ysparse_edn_handler.hpp" #include -#include #include -#endif -#include "./rapidyaml_edn_handler.hpp" +// instantiate the template namespace c4 { namespace yml { - -// instantiate the template -template class ParseEngine; +template class ParseEngine; +} // namespace yml +} // namespace c4 +namespace ys { void EventHandlerEdn::EventSink::append_escaped(csubstr val) { #define _c4flush_use_instead(repl, skip) \ @@ -78,6 +77,4 @@ void EventHandlerEdn::EventSink::append_escaped(csubstr val) this->append(val.sub(prev)); #undef _c4flush_use_instead } - -} // namespace yml -} // namespace c4 +} // namespace ys diff --git a/rapidyaml/native/rapidyaml_edn_handler.hpp b/rapidyaml/native/ysparse_edn_handler.hpp similarity index 85% rename from rapidyaml/native/rapidyaml_edn_handler.hpp rename to rapidyaml/native/ysparse_edn_handler.hpp index a50ee026e..5497fc609 100644 --- a/rapidyaml/native/rapidyaml_edn_handler.hpp +++ b/rapidyaml/native/ysparse_edn_handler.hpp @@ -1,19 +1,12 @@ -#ifndef _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ -#define _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ - -#ifdef RYML_SINGLE_HEADER -#include -#else -#ifndef _C4_YML_EVENT_HANDLER_STACK_HPP_ -#include "c4/yml/event_handler_stack.hpp" -#endif -#ifndef _C4_YML_STD_STRING_HPP_ -#include "c4/yml/std/string.hpp" -#endif -#ifndef _C4_YML_DETAIL_PRINT_HPP_ -#include "c4/yml/detail/print.hpp" -#endif -#endif +#ifndef _YSPARSE_EDN_HANDLER_HPP_ +#define _YSPARSE_EDN_HANDLER_HPP_ + +#include +#include +#include +#include +#include +#include #include @@ -22,21 +15,26 @@ C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") -namespace c4 { -namespace yml { - - -/** @addtogroup doc_event_handlers - * @{ */ +namespace ys { +using c4::csubstr; +using c4::substr; +using c4::to_substr; +using c4::to_csubstr; +using c4::yml::id_type; +using c4::yml::NodeType_e; +using c4::yml::type_bits; +#ifdef RYML_DBG +using c4::_dbg_printf; +#endif -struct EventHandlerEdnState : public ParserState +struct EventHandlerEdnState : public c4::yml::ParserState { - NodeData ev_data; + c4::yml::NodeData ev_data; }; -struct EventHandlerEdn : public EventHandlerStack +struct EventHandlerEdn : public c4::yml::EventHandlerStack { /** @name types @@ -82,20 +80,20 @@ struct EventHandlerEdn : public EventHandlerStackflags |= RUNK|RTOP; + m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; m_val_buffers.resize((size_t)m_stack.size()); m_arena.clear(); m_first_doc = true; @@ -123,7 +121,7 @@ struct EventHandlerEdn : public EventHandlerStack_stack_start_parse(filename, relocate_arena, relocate_arena_data); } @@ -173,7 +171,7 @@ struct EventHandlerEdn : public EventHandlerStack'); - _enable_(KEY|KEY_FOLDED); + _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); } C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) { _send_val_scalar_(scalar, '>'); - _enable_(VAL|VAL_FOLDED); + _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); } @@ -423,26 +421,26 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.anchor = anchor; } void set_val_anchor(csubstr anchor) { - _enable_(VALANCH); + _enable_(c4::yml::VALANCH); m_curr->ev_data.m_val.anchor = anchor; } void set_key_ref(csubstr ref) { _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); - _enable_(KEY|KEYREF); + _enable_(c4::yml::KEY|c4::yml::KEYREF); _send_("{:+ \"=ALI\" :* \""); _send_(ref.sub(1)); _send_("\"}\n"); } void set_val_ref(csubstr ref) { - _enable_(VAL|VALREF); + _enable_(c4::yml::VAL|c4::yml::VALREF); _send_("{:+ \"=ALI\" :* \""); _send_(ref.sub(1)); _send_("\"}\n"); @@ -457,12 +455,12 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.tag = _transform_directive(tag, m_key_tag_buf); } void set_val_tag(csubstr tag) { - _enable_(VALTAG); + _enable_(c4::yml::VALTAG); m_curr->ev_data.m_val.tag = _transform_directive(tag, m_val_tag_buf); } @@ -513,8 +511,6 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.anchor); _send_('\"'); } - if(_has_any_(KEYTAG)) + if(_has_any_(c4::yml::KEYTAG)) { _send_(", :! \""); _send_tag_(m_curr->ev_data.m_key.tag); _send_('\"'); } m_curr->ev_data.m_key = {}; - _disable_(KEYANCH|KEYREF|KEYTAG); + _disable_(c4::yml::KEYANCH|c4::yml::KEYREF|c4::yml::KEYTAG); } void _send_val_props_() { - if(_has_any_(VALANCH|VALREF)) + if(_has_any_(c4::yml::VALANCH|c4::yml::VALREF)) { _send_(", :& \""); _send_(m_curr->ev_data.m_val.anchor); _send_('\"'); } - if(m_curr->ev_data.m_type.type & VALTAG) + if(m_curr->ev_data.m_type.type & c4::yml::VALTAG) { _send_(", :! \""); _send_tag_(m_curr->ev_data.m_val.tag); _send_('\"'); } m_curr->ev_data.m_val = {}; - _disable_(VALANCH|VALREF|VALTAG); + _disable_(c4::yml::VALANCH|c4::yml::VALREF|c4::yml::VALTAG); } void _send_tag_(csubstr tag) { @@ -663,12 +659,12 @@ struct EventHandlerEdn : public EventHandlerStackpos); } } - csubstr result = normalize_tag_long(tag, output); + csubstr result = c4::yml::normalize_tag_long(tag, output); _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); _RYML_CB_CHECK(m_stack.m_callbacks, result.str); return result; @@ -677,14 +673,10 @@ struct EventHandlerEdn : public EventHandlerStack + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +using namespace ryml; + +#ifndef YS2PARSE_TIMED +#define TIMED_SECTION(name) +#else +#include +#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) +struct timed_section +{ + using myclock = std::chrono::steady_clock; + using fmsecs = std::chrono::duration; + csubstr name; + myclock::time_point start; + fmsecs since() const { return myclock::now() - start; } + timed_section(csubstr n) : name(n), start(myclock::now()) {} + ~timed_section() + { + fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); + } +}; +#endif + + +#if defined(__cplusplus) +extern "C" { +#endif +// see +// https://stackoverflow.com/questions/230689/best-way-to-throw-exceptions-in-jni-code +// https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method + +namespace { +C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) +{ + Ryml2EvtParseError exc; + exc.location = location; + exc.msg.assign(msg, msg_len); + throw exc; +} +} // anon namespace + +RYML_EXPORT Ryml2Evt *ys2evt_init() +{ + TIMED_SECTION("ys2evt_init"); + Callbacks cb = {}; + cb.m_error = &ys2evt_parse_error; + set_callbacks(cb); + Ryml2Evt *ryml2evt = _RYML_CB_ALLOC(get_callbacks(), Ryml2Evt, 1); + _RYML_CB_CHECK(get_callbacks(), ryml2evt != nullptr); + new ((void*)ryml2evt) Ryml2Evt(); + return ryml2evt; +} + +RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt) +{ + TIMED_SECTION("ys2evt_destroy"); + ryml2evt->~Ryml2Evt(); + _RYML_CB_FREE(get_callbacks(), ryml2evt, Ryml2Evt, 1); +} + +RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *events, size_type evt_size) +{ + TIMED_SECTION("ys2evt_parse"); + csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; + substr ys_(ys, (size_t)ys_size); + { + TIMED_SECTION("reset + reserve"); + ryml2evt->reset(ys_, events, evt_size); + ryml2evt->m_handler.reserve(256u); + } + { + TIMED_SECTION("parse_in_place"); + ryml2evt->m_parser.parse_in_place_ev(filename_, ys_); + } + return (size_type)ryml2evt->m_handler.m_evt_curr; +} + +#if defined(__cplusplus) +} +#endif diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp new file mode 100644 index 000000000..7712f489f --- /dev/null +++ b/rapidyaml/native/ysparse_evt.hpp @@ -0,0 +1,108 @@ +#pragma once +#ifndef YSPARSE_EVT_HPP_ +#define YSPARSE_EVT_HPP_ + +#include +#include "ysparse_evt_handler.hpp" + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +#if defined(__cplusplus) +extern "C" { +#endif + +using size_type = int; + +struct RYML_EXPORT Ryml2Evt +{ + ys::EventHandlerEvt m_handler; + c4::yml::ParseEngine m_parser; + Ryml2Evt() + : m_handler() + , m_parser(&m_handler) + { + RYML_CHECK(m_parser.options().scalar_filtering()); + } + void reset(c4::csubstr src, evt::DataType *evt, int32_t evt_size) + { + m_handler.reset(src, evt, evt_size); + } +}; + +struct RYML_EXPORT Ryml2EvtParseError : public std::exception +{ + c4::yml::Location location; + std::string msg; + const char* what() const noexcept override { return msg.c_str(); } +}; + + +//----------------------------------------------------------------------------- + +/** Initialize the resources */ +RYML_EXPORT Ryml2Evt *ys2evt_init(); + +/** Destroy the resources */ +RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt); + +/** Parse YAML in the string `ys` of size `ys_size`, and write the + * result into the array of (integer) events `evt` of size + * `evt_size`. Each event is encoded as a mask of evt::EventFlags + * (note that it uses the integer evt::DataType as the underlying + * type), and when an event has an associated string, it is followed + * in the array by two extra values, which encode the offset and the + * length of the string in the `ys` string. The `ys` string is mutated + * during parsing. + * + * @return the size needed for `evt`. The caller must check if the + * returned size is larger than `evt_size`. If so, this means that + * `evt` could not accomodate all events produced from `ys`, and is + * incomplete. The caller must then (1) resize `evt` to at least the + * return value, (2) re-copy the original YS into `ys` and (3) call + * again this function, passing in the resized `evt` and the fresh + * copy in `ys`. + * + * @note nothing is written beyond `evt_size`. This means that when + * `evt_size` is 0, then `evt` can be null. This function can be + * safely called for any valid pair of `evt` and `evt_size`, and will + * always return the same required size. + * + * For example, the YAML `say: 2 + 2` produces the following sequence of + * 12 integers: + * + * ```c++ + * BSTR, + * BDOC, + * VAL|BMAP|BLCK, + * KEY|SCLR|PLAI, 0, 3, // "say" + * VAL|SCLR|PLAI, 5, 5, // "2 + 2" + * EMAP, + * EDOC, + * ESTR, + * ``` + * + * Note that the scalar events, ie "say" and "2 + 2", are followed + * each by two extra integers encoding the offset and length of the + * scalar's string. These two extra integers are present whenever the + * event has any of the bits `SCLR`, `ALIA`, `ANCH` or `TAG`. For ease + * of use, there is a bitmask `HAS_STR`, which enables quick testing + * by a simple `flags & HAS_STR`. Refer to evt::EventFlags for the + * full list of flags and their meaning. + * + * Also, where a string requires filtering, the parser filters it + * in-place in the input string, and the extra integers will pertain + * to the resulting filtered string. + */ +RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *evt, size_type evt_size); + +#if defined(__cplusplus) +} +#endif + +#endif /* YSPARSE_EVT_HPP_ */ diff --git a/rapidyaml/native/ysparse_evt_handler.cpp b/rapidyaml/native/ysparse_evt_handler.cpp new file mode 100644 index 000000000..24e5fd662 --- /dev/null +++ b/rapidyaml/native/ysparse_evt_handler.cpp @@ -0,0 +1,12 @@ +#include "ysparse_evt_handler.hpp" +#include +#include + +namespace c4 { +namespace yml { + +// instantiate the template +template class ParseEngine; + +} // namespace yml +} // namespace c4 diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp new file mode 100644 index 000000000..f00d7fe58 --- /dev/null +++ b/rapidyaml/native/ysparse_evt_handler.hpp @@ -0,0 +1,680 @@ +#ifndef _YSPARSE_EVT_HANDLER_HPP_ +#define _YSPARSE_EVT_HANDLER_HPP_ + +#include +#include +#include +#include +#include +#include + +C4_SUPPRESS_WARNING_GCC_CLANG_PUSH +C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") +C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") + +namespace evt { +using DataType = int32_t; +typedef enum : DataType { + // --------------------- + // structure flags + KEY_ = 1 << 0, // as key + VAL_ = 1 << 1, // as value + SCLR = 1 << 2, // =VAL + BSEQ = 1 << 3, // +SEQ + ESEQ = 1 << 4, // -SEQ + BMAP = 1 << 5, // +MAP + EMAP = 1 << 6, // -MAP + ALIA = 1 << 7, // ref + ANCH = 1 << 8, // anchor + TAG_ = 1 << 9, // tag + // --------------------- + // style flags + PLAI = 1 << 10, // : (plain scalar) + SQUO = 1 << 11, // ' (single-quoted scalar) + DQUO = 1 << 12, // " (double-quoted scalar) + LITL = 1 << 13, // | (block literal scalar) + FOLD = 1 << 14, // > (block folded scalar) + FLOW = 1 << 15, // flow container: [] for seqs or {} for maps + BLCK = 1 << 16, // block container + // --------------------- + // document flags + BDOC = 1 << 17, // +DOC + EDOC = 1 << 18, // -DOC + EXPL = 1 << 21, // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) + BSTR = 1 << 19, // +STR + ESTR = 1 << 20, // -STR + // --------------------- + // utility flags + LAST = EXPL, + MASK = (LAST << 1) - 1, + HAS_STR = SCLR|ALIA|ANCH|TAG_ // the event requires a string. the next two integers will provide respectively the string's offset and length +} EventFlags; +} // namespace evt + + +namespace ys { + +using c4::csubstr; +using c4::substr; +using c4::to_substr; +using c4::to_csubstr; +#ifdef RYML_DBG +using c4::_dbg_printf; +#endif + +struct EventHandlerEvtState : public c4::yml::ParserState +{ + c4::yml::type_bits evt_type; + int32_t evt_id; +}; + + +struct EventHandlerEvt : public c4::yml::EventHandlerStack +{ + + /** @name types + * @{ */ + + // our internal state must inherit from parser state + using state = EventHandlerEvtState; + + /** @} */ + +public: + + /** @cond dev */ + csubstr m_str; + evt::DataType * m_evt; + int32_t m_evt_curr; + int32_t m_evt_prev; + int32_t m_evt_size; + char m_key_tag_buf[256]; + char m_val_tag_buf[256]; + std::string m_arena; + + // undefined at the end + #define _enable_(bits) _enable__() + #define _disable_(bits) _disable__() + #define _has_any_(bits) _has_any__() + /** @endcond */ + +public: + + /** @name construction and resetting + * @{ */ + + EventHandlerEvt(c4::yml::Callbacks const& cb) + : EventHandlerStack(cb) + { + reset({}, nullptr, 0); + } + EventHandlerEvt() + : EventHandlerEvt(c4::yml::get_callbacks()) + { + } + + void reset(csubstr str, evt::DataType *dst, int32_t dst_size) + { + _stack_reset_root(); + m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; + m_curr->evt_type = {}; + m_curr->evt_id = 0; + m_arena.clear(); + m_str = str; + m_evt = dst; + m_evt_size = dst_size; + m_evt_curr = 0; + m_evt_prev = 0; + } + + void reserve(int arena_size) + { + m_arena.reserve(arena_size); + } + + /** @} */ + +public: + + /** @name parse events + * @{ */ + + void start_parse(const char* filename, c4::yml::detail::pfn_relocate_arena relocate_arena, void *relocate_arena_data) + { + this->_stack_start_parse(filename, relocate_arena, relocate_arena_data); + } + + void finish_parse() + { + this->_stack_finish_parse(); + } + + void cancel_parse() + { + while(m_stack.size() > 1) + _pop(); + } + + /** @} */ + +public: + + /** @name YAML stream events */ + /** @{ */ + + void begin_stream() + { + _send_flag_only_(evt::BSTR); + } + + void end_stream() + { + _send_flag_only_(evt::ESTR); + } + + /** @} */ + +public: + + /** @name YAML document events */ + /** @{ */ + + /** implicit doc start (without ---) */ + void begin_doc() + { + _c4dbgpf("{}/{}: begin_doc", m_evt_curr, m_evt_size); + _send_flag_only_(evt::BDOC); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + } + } + /** implicit doc end (without ...) */ + void end_doc() + { + _c4dbgpf("{}/{}: end_doc", m_evt_curr, m_evt_size); + _send_flag_only_(evt::EDOC); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** explicit doc start, with --- */ + void begin_doc_expl() + { + _c4dbgpf("{}/{}: begin_doc_expl", m_evt_curr, m_evt_size); + _send_flag_only_(evt::BDOC|evt::EXPL); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + } + } + /** explicit doc end, with ... */ + void end_doc_expl() + { + _c4dbgpf("{}/{}: end_doc_expl", m_evt_curr, m_evt_size); + _send_flag_only_(evt::EDOC|evt::EXPL); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** @} */ + +public: + + /** @name YAML map functions */ + /** @{ */ + + void begin_map_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_map_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_map_val_flow() + { + _c4dbgpf("{}/{}: bmap flow", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BMAP|evt::FLOW); + _mark_parent_with_children_(); + _enable_(c4::yml::MAP|c4::yml::FLOW_SL); + _push(); + } + void begin_map_val_block() + { + _c4dbgpf("{}/{}: bmap block", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BMAP|evt::BLCK); + _mark_parent_with_children_(); + _enable_(c4::yml::MAP|c4::yml::BLOCK); + _push(); + } + + void end_map() + { + _pop(); + _send_flag_only_(evt::EMAP); + } + + /** @} */ + +public: + + /** @name YAML seq events */ + /** @{ */ + + void begin_seq_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_seq_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_seq_val_flow() + { + _c4dbgpf("{}/{}: bseq flow", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BSEQ|evt::FLOW); + _mark_parent_with_children_(); + _enable_(c4::yml::SEQ|c4::yml::FLOW_SL); + _push(); + } + void begin_seq_val_block() + { + _c4dbgpf("{}/{}: bseq block", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BSEQ|evt::BLCK); + _mark_parent_with_children_(); + _enable_(c4::yml::SEQ|c4::yml::BLOCK); + _push(); + } + + void end_seq() + { + _pop(); + _send_flag_only_(evt::ESEQ); + } + + /** @} */ + +public: + + /** @name YAML structure events */ + /** @{ */ + + void add_sibling() + { + _RYML_CB_ASSERT(m_stack.m_callbacks, m_parent); + m_curr->evt_type = {}; + } + + /** set the previous val as the first key of a new map, with flow style. + * + * See the documentation for @ref doc_event_handlers, which has + * important notes about this event. + */ + void actually_val_is_first_key_of_new_map_flow() + { + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt_curr > 2); + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt_prev > 0); + _c4dbgpf("{}/{}: prev={} actually_val_is_first_key_of_new_map_flow", m_evt_curr, m_evt_size, m_evt_prev); + // BEFORE + // ... flag start len (free) + // | | + // prev curr + // AFTER + // ... flag flag start len (free) + // | | + // prev curr + if(m_evt_prev < m_evt_size) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (m_evt[m_evt_prev] & evt::HAS_STR) || m_evt_curr >= m_evt_size); + if(m_evt_curr < m_evt_size) + { + // watchout: it must be in this order! + m_evt[m_evt_curr ] = m_evt[m_evt_prev + 2]; + m_evt[m_evt_curr - 1] = m_evt[m_evt_prev + 1]; + m_evt[m_evt_curr - 2] = m_evt[m_evt_prev] | evt::KEY_; + m_evt[m_evt_curr - 2] &= ~evt::VAL_; + } + m_evt[m_evt_prev] = evt::BMAP|evt::FLOW|evt::VAL_; + } + m_curr->evt_id = m_evt_curr - 2; + ++m_evt_prev; + ++m_evt_curr; + _enable_(c4::yml::MAP|c4::yml::FLOW); + _push(); + } + + void actually_val_is_first_key_of_new_map_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + /** @} */ + +public: + + /** @name YAML scalar events */ + /** @{ */ + + + C4_ALWAYS_INLINE void set_key_scalar_plain(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_plain: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::PLAI); + _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN); + } + C4_ALWAYS_INLINE void set_val_scalar_plain(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_plain: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::PLAI); + _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN); + } + + + C4_ALWAYS_INLINE void set_key_scalar_dquoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_dquo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::DQUO); + _enable_(c4::yml::KEY|c4::yml::KEY_DQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_dquoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_dquo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::DQUO); + _enable_(c4::yml::VAL|c4::yml::VAL_DQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_squoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_squo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::SQUO); + _enable_(c4::yml::KEY|c4::yml::KEY_SQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_squoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_squo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::SQUO); + _enable_(c4::yml::VAL|c4::yml::VAL_SQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_literal(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_literal: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::LITL); + _enable_(c4::yml::KEY|c4::yml::KEY_LITERAL); + } + C4_ALWAYS_INLINE void set_val_scalar_literal(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_literal: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::LITL); + _enable_(c4::yml::VAL|c4::yml::VAL_LITERAL); + } + + + C4_ALWAYS_INLINE void set_key_scalar_folded(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_folded: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::FOLD); + _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); + } + C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_folded: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::FOLD); + _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); + } + + + C4_ALWAYS_INLINE void mark_key_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + C4_ALWAYS_INLINE void mark_val_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + + /** @} */ + +public: + +#define _add_scalar_(i, scalar) \ + _c4dbgpf("{}/{}: scalar!", i, m_evt_size); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, scalar.is_sub(m_str)); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt[i] & evt::HAS_STR); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, i + 2 < m_evt_size); \ + m_evt[i + 1] = (evt::DataType)(scalar.str - m_str.str); \ + m_evt[i + 2] = (evt::DataType)scalar.len + + /** @name YAML anchor/reference events */ + /** @{ */ + + void set_key_anchor(csubstr anchor) + { + _c4dbgpf("{}/{}: set_key_anchor", m_evt_curr, m_evt_size); + _enable_(c4::yml::KEYANCH); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::KEY_|evt::ANCH; + _add_scalar_(m_evt_curr, anchor); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + void set_val_anchor(csubstr anchor) + { + _c4dbgpf("{}/{}: set_val_anchor", m_evt_curr, m_evt_size); + _enable_(c4::yml::VALANCH); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::VAL_|evt::ANCH; + _add_scalar_(m_evt_curr, anchor); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + void set_key_ref(csubstr ref) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); + _enable_(c4::yml::KEY|c4::yml::KEYREF); + _send_key_scalar_(ref, evt::KEY_|evt::ALIA); + } + void set_val_ref(csubstr ref) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); + _enable_(c4::yml::VAL|c4::yml::VALREF); + _send_val_scalar_(ref, evt::VAL_|evt::ALIA); + } + + /** @} */ + +public: + + /** @name YAML tag events */ + /** @{ */ + + void set_key_tag(csubstr tag) + { + _enable_(c4::yml::KEYTAG); + csubstr ttag = _transform_directive(tag, m_key_tag_buf); + _RYML_CB_ASSERT(m_stack.m_callbacks, !ttag.empty()); + if(ttag.begins_with('!')) + ttag = ttag.sub(1); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::KEY_|evt::TAG_; + _add_scalar_(m_evt_curr, ttag); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + void set_val_tag(csubstr tag) + { + _enable_(c4::yml::VALTAG); + csubstr ttag = _transform_directive(tag, m_val_tag_buf); + _RYML_CB_ASSERT(m_stack.m_callbacks, !ttag.empty()); + if(ttag.begins_with('!')) + ttag = ttag.sub(1); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::VAL_|evt::TAG_; + _add_scalar_(m_evt_curr, ttag); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + /** @} */ + +public: + + /** @name YAML directive events */ + /** @{ */ + + void add_directive(csubstr directive) + { + _RYML_CB_ERR(m_stack.m_callbacks, "tag directives not supported"); + } + + /** @} */ + +public: + + /** @name YAML arena events */ + /** @{ */ + + substr alloc_arena(size_t len) + { + const size_t sz = m_arena.size(); + csubstr prev = to_csubstr(m_arena); + m_arena.resize(sz + len); + substr out = to_substr(m_arena).sub(sz); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + _stack_relocate_to_new_arena(prev, curr); + return out; + } + + substr alloc_arena(size_t len, substr *relocated) + { + csubstr prev = to_csubstr(m_arena); + if(!prev.is_super(*relocated)) + return alloc_arena(len); + substr out = alloc_arena(len); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); + return out; + } + + /** @} */ + +public: + + /** push a new parent, add a child to the new parent, and set the + * child as the current node */ + void _push() + { + _stack_push(); + m_curr->evt_type = {}; + } + + /** end the current scope */ + void _pop() + { + _stack_pop(); + } + + template C4_ALWAYS_INLINE void _enable__() noexcept + { + m_curr->evt_type |= bits; + } + template C4_ALWAYS_INLINE void _disable__() noexcept + { + m_curr->evt_type &= ~bits; + } + template C4_ALWAYS_INLINE bool _has_any__() const noexcept + { + return (m_curr->evt_type & bits) != c4::yml::type_bits(0); + } + + void _mark_parent_with_children_() + { + if(m_parent) + m_parent->has_children = true; + } + + C4_ALWAYS_INLINE void _send_flag_only_(evt::DataType flags) + { + _c4dbgpf("{}/{}: flag only", m_evt_curr, m_evt_size); + if(m_evt_curr < m_evt_size) + m_evt[m_evt_curr] = flags; + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + ++m_evt_curr; + } + + C4_ALWAYS_INLINE void _send_key_scalar_(csubstr scalar, evt::DataType flags) + { + _c4dbgpf("{}/{}: key scalar", m_evt_curr, m_evt_size); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::SCLR|evt::KEY_|flags; + _add_scalar_(m_evt_curr, scalar); + } + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + C4_ALWAYS_INLINE void _send_val_scalar_(csubstr scalar, evt::DataType flags) + { + _c4dbgpf("{}/{}: val scalar", m_evt_curr, m_evt_size); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::SCLR|evt::VAL_|flags; + _add_scalar_(m_evt_curr, scalar); + } + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + csubstr _transform_directive(csubstr tag, substr output) + { + if(tag.begins_with('!')) + { + if(c4::yml::is_custom_tag(tag)) + { + _RYML_CB_ERR_(m_stack.m_callbacks, "tag not found", m_curr->pos); + } + } + csubstr result = c4::yml::normalize_tag_long(tag, output); + _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); + _RYML_CB_CHECK(m_stack.m_callbacks, result.str); + return result; + } +#undef _enable_ +#undef _disable_ +#undef _has_any_ + +}; + +} // namespace ys + +C4_SUPPRESS_WARNING_GCC_POP + +#endif /* _C4_YML_EVENT_HANDLER_EVT_HPP_ */ diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp new file mode 100644 index 000000000..40a7e5a7a --- /dev/null +++ b/rapidyaml/native/ysparse_test.cpp @@ -0,0 +1,675 @@ +#include +#include +#include + +using c4::csubstr; +using c4::substr; + +namespace c4 +{ +template<> +c4::EnumSymbols const esyms() +{ + static constexpr typename c4::EnumSymbols::Sym syms[] = { + {evt::KEY_, "KEY_"}, + {evt::VAL_, "VAL_"}, + {evt::SCLR, "SCLR"}, + {evt::BSEQ, "BSEQ"}, + {evt::ESEQ, "ESEQ"}, + {evt::BMAP, "BMAP"}, + {evt::EMAP, "EMAP"}, + {evt::ALIA, "ALIA"}, + {evt::ANCH, "ANCH"}, + {evt::TAG_, "TAG_"}, + {evt::PLAI, "PLAI"}, + {evt::SQUO, "SQUO"}, + {evt::DQUO, "DQUO"}, + {evt::LITL, "LITL"}, + {evt::FOLD, "FOLD"}, + {evt::FLOW, "FLOW"}, + {evt::BLCK, "BLCK"}, + {evt::BDOC, "BDOC"}, + {evt::EDOC, "EDOC"}, + {evt::BSTR, "BSTR"}, + {evt::ESTR, "ESTR"}, + {evt::EXPL, "EXPL"}, + }; + return c4::EnumSymbols(syms); +} +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +struct Ys2EdnScoped +{ + Ryml2Edn *ryml2edn; + Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} + ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } +}; +struct Ys2EvtScoped +{ + Ryml2Evt *ryml2evt; + Ys2EvtScoped() : ryml2evt(ys2evt_init()) {} + ~Ys2EvtScoped() { if(ryml2evt) ys2evt_destroy(ryml2evt); } +}; + + +struct TestResult +{ + uint32_t num_assertions; + uint32_t num_tests; + uint32_t num_failed_assertions; + uint32_t num_failed_tests; + operator bool() const { return num_failed_tests == 0 && num_failed_assertions == 0; } + void add(TestResult const& that) + { + num_tests += 1 + that.num_tests; + num_assertions += that.num_assertions; + num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; + num_failed_assertions += that.num_failed_assertions; + } +}; + +// provide a structured input for the events, grouping the relevant +// data in a single structure +struct EvtWithScalar +{ + evt::DataType flags, str_start, str_len; + csubstr scalar; + bool needs_filter; + EvtWithScalar(evt::DataType t, evt::DataType start=0, evt::DataType len=0, csubstr sclr={}, bool needs_filter_=false) + { + flags = t; + str_start = start; + str_len = len; + scalar = sclr; + needs_filter = needs_filter_; + } + size_t required_size() const { return (flags & evt::HAS_STR) ? 3u : 1u; } +}; + +size_t expected_size(std::vector const& evt) +{ + size_t exp = 0; + for(EvtWithScalar const& e : evt) + exp += e.required_size(); + return exp; +} + +struct TestCase +{ + csubstr ys; + csubstr edn; + std::vector evt; + +public: + + #define _runtest(name, ...) \ + do { \ + printf("[ RUN ] %s ... \n", #name); \ + TestResult tr_ = name(__VA_ARGS__); \ + tr.add(tr_); \ + printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ + } while(0) + #define CHECK(cond) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s\n", __FILE__, __LINE__, #cond); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + #define CHECK_MSG(cond, fmt, ...) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s:" fmt "\n", __FILE__, __LINE__, #cond, ## __VA_ARGS__); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + + TestResult test(Ryml2Edn *ryml2edn, Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + _runtest(test_edn_large_enough, ); + _runtest(test_edn_too_small, ); + _runtest(test_edn_nullptr, ); + _runtest(test_edn_large_enough_reuse, ryml2edn); + _runtest(test_edn_too_small_reuse, ryml2edn); + _runtest(test_edn_nullptr_reuse, ryml2edn); + _runtest(test_evt_large_enough, ); + _runtest(test_evt_too_small, ); + _runtest(test_evt_nullptr, ); + _runtest(test_evt_large_enough_reuse, ryml2evt); + _runtest(test_evt_too_small_reuse, ryml2evt); + _runtest(test_evt_nullptr_reuse, ryml2evt); + return tr; + } + + // happy path: large-enough destination string + TestResult test_edn_large_enough_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output; + output.resize(2 * edn.len); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_evt_large_enough_reuse(Ryml2Evt *ryml2evt) const + { + if(evt.empty()) return {}; + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::vector output; + output.resize(2 * expected_size(evt)); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); + CHECK_MSG((size_t)reqsize == expected_size(evt), "%d vs %zu", reqsize, expected_size(evt)); + CHECK(reqsize != 0); + output.resize(reqsize); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_large_enough() const + { + Ys2EdnScoped lib; + return test_edn_large_enough_reuse(lib.ryml2edn); + } + TestResult test_evt_large_enough() const + { + Ys2EvtScoped lib; + return test_evt_large_enough_reuse(lib.ryml2evt); + } + + // less-happy path: destination string not large enough + TestResult test_edn_too_small_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output = "?"; + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + CHECK(output != "?"); + output.resize(reqsize); + size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); + CHECK(getsize == reqsize); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_evt_too_small_reuse(Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::vector output; + output.resize(expected_size(evt)); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize == expected_size(evt)); + CHECK(reqsize != 0); + output.resize(reqsize); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize2 == reqsize); + output.resize(reqsize2); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_too_small() const + { + Ys2EdnScoped lib; + return test_edn_too_small_reuse(lib.ryml2edn); + } + TestResult test_evt_too_small() const + { + Ys2EvtScoped lib; + return test_evt_too_small_reuse(lib.ryml2evt); + } + + // safe calling with nullptr + TestResult test_edn_nullptr_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + return tr; + } + TestResult test_evt_nullptr_reuse(Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + CHECK(reqsize == expected_size(evt)); + CHECK(reqsize != 0); + std::vector output; + output.resize(reqsize); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize2 == reqsize); + CHECK(reqsize2 == output.size()); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_nullptr() const + { + Ys2EdnScoped lib; + return test_edn_nullptr_reuse(lib.ryml2edn); + } + TestResult test_evt_nullptr() const + { + Ys2EvtScoped lib; + return test_evt_nullptr_reuse(lib.ryml2evt); + } + +public: + + bool testeq(csubstr actual) const + { + const bool status = (actual == edn); + if(!status) + printf("------\n" + "FAIL:\n" + "input:[%zu]~~~%.*s~~~\n" + "expected:[%zu]~~~%.*s~~~\n" + "actual:[%zu]~~~%.*s~~~\n", + ys.len, (int)ys.len, ys.str, + edn.len, (int)edn.len, edn.str, + actual.len, (int)actual.len, actual.str); + return status; + } + + bool testeq(std::vector const& actual, csubstr parsed_source) const + { + int status = true; + size_t num_events_expected = evt.size(); + size_t num_ints_expected = expected_size(evt); + if(actual.size() != num_ints_expected) + { + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n" + "expected size:~~~%zu~~~\n" + "actual size:~~~%zu~~~\n", + (int)ys.len, ys.str, + num_ints_expected, + actual.size()); + status = false; + } + for(size_t i = 0, ie = 0; ie < num_events_expected; ++ie) + { + if(i >= actual.size()) + { + printf("fail: bad actual size. i=%zu vs %zu=actual.size()=\n", i, actual.size()); + status = false; + break; + } + #define _testcmp(fmt, cmp, ...) \ + printf("status=%d cmp=%d evt=%zu i=%zu: " fmt "\n", status, (cmp), ie, i, ## __VA_ARGS__); \ + status &= (cmp) + char actualbuf[100]; + char expectedbuf[100]; + size_t reqsize_actual = c4::bm2str(actual[i] & evt::MASK, actualbuf, sizeof(actualbuf)); + size_t reqsize_expected = c4::bm2str(evt[ie].flags & evt::MASK, expectedbuf, sizeof(expectedbuf)); + C4_CHECK(reqsize_actual < sizeof(actualbuf)); + C4_CHECK(reqsize_expected < sizeof(expectedbuf)); + _testcmp("exp=%d(%s) vs act=%d(%s)", evt[ie].flags == actual[i], evt[ie].flags, expectedbuf, actual[i], actualbuf); + status &= (evt[ie].flags == actual[i]); + if((evt[ie].flags & evt::HAS_STR) && (actual[i] & evt::HAS_STR)) + { + _testcmp(" exp=%d vs act=%d", evt[ie].str_start == actual[i + 1], evt[ie].str_start, actual[i + 1]); + _testcmp(" exp=%d vs act=%d", evt[ie].str_len == actual[i + 2], evt[ie].str_len, actual[i + 2]); + bool safeactual = (i + 2 < actual.size()) && (actual[i + 1] < (int)parsed_source.len && actual[i + 1] + actual[i + 2] <= (int)parsed_source.len); + bool safeexpected = (evt[ie].str_start < (int)parsed_source.len && evt[ie].str_start + evt[ie].str_len <= (int)parsed_source.len); + _testcmp(" safeactual=%d", safeactual, safeactual); + _testcmp(" safeactual=%d safeexpected=%d", safeactual == safeexpected, safeactual, safeexpected); + if(safeactual && safeexpected) + { + csubstr evtstr = parsed_source.sub((size_t)evt[ie].str_start, (size_t)evt[ie].str_len); + csubstr actualstr = parsed_source.sub((size_t)actual[i + 1], (size_t)actual[i + 2]); + _testcmp(" ref=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", + evt[ie].scalar == actualstr, + evt[ie].scalar.len, (int)evt[ie].scalar.len, evt[ie].scalar.str, + actualstr.len, (int)actualstr.len, actualstr.str); + if( ! evt[ie].needs_filter) + { + _testcmp(" exp=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", + evtstr == actualstr, + evtstr.len, (int)evtstr.len, evtstr.str, + actualstr.len, (int)actualstr.len, actualstr.str); + } + } + } + i += (actual[i] & evt::HAS_STR) ? 3 : 1; + } + if(!status) + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n", + (int)ys.len, ys.str); + return status; + } +}; + + +//----------------------------------------------------------------------------- + +namespace { +// make the declarations shorter +#define tc(ys, edn, ...) {ys, edn, std::vector(__VA_ARGS__)} +#define e(...) EvtWithScalar{__VA_ARGS__} +using namespace evt; +inline constexpr bool needs_filter = true; +const TestCase test_cases[] = { + // case ------------------------------------------------- + tc("a: 1", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "1"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 1, "a"), + e(VAL_|SCLR|PLAI, 3, 1, "1"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("say: 2 + 2", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "say"} +{:+ "=VAL", := "2 + 2"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 3, "say"), + e(VAL_|SCLR|PLAI, 5, 5, "2 + 2"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("𝄞: ✅", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "𝄞"} +{:+ "=VAL", := "✅"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 4, "𝄞"), + e(VAL_|SCLR|PLAI, 6, 3, "✅"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("[a, b, c]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "=VAL", := "c"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI, 4, 1, "b"), + e(VAL_|SCLR|PLAI, 7, 1, "c"), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------ + tc("[a: b]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "-MAP"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|FLOW), + e(VAL_|BMAP|FLOW), + e(KEY_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI, 4, 1, "b"), + e(EMAP), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------ + tc(R"(--- !yamlscript/v0 +foo: ! +- {x: y} +- [x, y] +- foo +- 'foo' +- "foo" +- | + foo +- > + foo +- [1, 2, true, false, null] +- &anchor-1 !tag-1 foobar +--- +another: doc +)", + R"(( +{:+ "+MAP", :! "yamlscript/v0"} +{:+ "=VAL", := "foo"} +{:+ "+SEQ", :! ""} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-MAP"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-SEQ"} +{:+ "=VAL", := "foo"} +{:+ "=VAL", :' "foo"} +{:+ "=VAL", :$ "foo"} +{:+ "=VAL", :| "foo\n"} +{:+ "=VAL", :> "foo\n"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "1"} +{:+ "=VAL", := "2"} +{:+ "=VAL", := "true"} +{:+ "=VAL", := "false"} +{:+ "=VAL", := "null"} +{:+ "-SEQ"} +{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} +{:+ "-SEQ"} +{:+ "-MAP"} +{:+ "-DOC"} +{:+ "+DOC"} +{:+ "+MAP"} +{:+ "=VAL", := "another"} +{:+ "=VAL", := "doc"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC|EXPL), + e(VAL_|TAG_, 5, 13, "yamlscript/v0"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 19, 3, "foo"), + e(VAL_|TAG_, 25, 0, ""), + e(VAL_|BSEQ|BLCK), + e(VAL_|BMAP|FLOW), + e(KEY_|SCLR|PLAI, 29, 1, "x"), + e(VAL_|SCLR|PLAI, 32, 1, "y"), + e(EMAP), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 38, 1, "x"), + e(VAL_|SCLR|PLAI, 41, 1, "y"), + e(ESEQ), + e(VAL_|SCLR|PLAI, 46, 3, "foo"), + e(VAL_|SCLR|SQUO, 53, 3, "foo"), + e(VAL_|SCLR|DQUO, 61, 3, "foo"), + e(VAL_|SCLR|LITL, 70, 4, "foo\n", needs_filter), + e(VAL_|SCLR|FOLD, 80, 4, "foo\n", needs_filter), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 89, 1, "1"), + e(VAL_|SCLR|PLAI, 92, 1, "2"), + e(VAL_|SCLR|PLAI, 95, 4, "true"), + e(VAL_|SCLR|PLAI, 101, 5, "false"), + e(VAL_|SCLR|PLAI, 108, 4, "null"), + e(ESEQ), + e(VAL_|TAG_, 127, 5, "tag-1"), + e(VAL_|ANCH, 117, 8, "anchor-1"), + e(VAL_|SCLR|PLAI, 133, 6, "foobar"), + e(ESEQ), + e(EMAP), + e(EDOC), + e(BDOC|EXPL), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 144, 7, "another"), + e(VAL_|SCLR|PLAI, 153, 3, "doc"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"(plain: well + a + b + c +squo: 'single''quote' +dquo: "x\t\ny" +lit: | + X + Y + Z +fold: > + U + V + W +)", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "plain"} +{:+ "=VAL", := "well a b c"} +{:+ "=VAL", := "squo"} +{:+ "=VAL", :' "single'quote"} +{:+ "=VAL", := "dquo"} +{:+ "=VAL", :$ "x\t\ny"} +{:+ "=VAL", := "lit"} +{:+ "=VAL", :| "X\nY\nZ\n"} +{:+ "=VAL", := "fold"} +{:+ "=VAL", :> "U V W\n"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 5, "plain"), + e(VAL_|SCLR|PLAI, 7, 10, "well a b c"), + e(KEY_|SCLR|PLAI, 24, 4, "squo"), + e(VAL_|SCLR|SQUO, 31, 12, "single'quote", needs_filter), + e(KEY_|SCLR|PLAI, 46, 4, "dquo"), + e(VAL_|SCLR|DQUO, 53, 4, "x\t\ny", needs_filter), + e(KEY_|SCLR|PLAI, 61, 3, "lit"), + e(VAL_|SCLR|LITL, 68, 6, "X\nY\nZ\n", needs_filter), + e(KEY_|SCLR|PLAI, 89, 4, "fold"), + e(VAL_|SCLR|FOLD, 97, 6, "U V W\n", needs_filter), + e(EMAP), + e(EDOC), + e(ESTR), + }), +}; +} // namespace + +int main() +{ + Ys2EdnScoped ys2edn; + Ys2EvtScoped ys2evt; + TestResult total = {}; + size_t failed_cases = {}; + size_t num_cases = C4_COUNTOF(test_cases); + for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) + { + printf("-----------------------------------------\n" + "case %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); + const TestResult tr = test_cases[i].test(ys2edn.ryml2edn, ys2evt.ryml2evt); + total.add(tr); + failed_cases += (!tr); + printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + } + printf("assertions: %u/%u pass %u/%u fail\n", total.num_assertions - total.num_failed_assertions, total.num_assertions, total.num_failed_assertions, total.num_assertions); + printf("tests: %u/%u pass %u/%u fail\n", total.num_tests - total.num_failed_tests, total.num_tests, total.num_failed_tests, total.num_tests); + printf("cases: %zu/%zu pass %zu/%zu fail\n", num_cases-failed_cases, num_cases, failed_cases, num_cases); + if(total) + printf("TESTS SUCCEED!\n"); + return total ? 0 : -1; +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/Evt.java b/rapidyaml/src/main/java/org/rapidyaml/Evt.java new file mode 100644 index 000000000..4f06211f3 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/Evt.java @@ -0,0 +1,37 @@ +package org.rapidyaml; + +public class Evt { + // --------------------- + // structure flags + public static final int KEY_ = 1 << 0; // as key + public static final int VAL_ = 1 << 1; // as value + public static final int SCLR = 1 << 2; // =VAL + public static final int BSEQ = 1 << 3; // +SEQ + public static final int ESEQ = 1 << 4; // -SEQ + public static final int BMAP = 1 << 5; // +MAP + public static final int EMAP = 1 << 6; // -MAP + public static final int ALIA = 1 << 7; // ref + public static final int ANCH = 1 << 8; // anchor + public static final int TAG_ = 1 << 9; // tag + // --------------------- + // style flags + public static final int PLAI = 1 << 10; // : (plain scalar) + public static final int SQUO = 1 << 11; // ' (single-quoted scalar) + public static final int DQUO = 1 << 12; // " (double-quoted scalar) + public static final int LITL = 1 << 13; // | (block literal scalar) + public static final int FOLD = 1 << 14; // > (block folded scalar) + public static final int FLOW = 1 << 15; // flow container: [] for seqs or {} for maps + public static final int BLCK = 1 << 16; // block container + // --------------------- + // document flags + public static final int BDOC = 1 << 17; // +DOC + public static final int EDOC = 1 << 18; // -DOC + public static final int EXPL = 1 << 21; // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) + public static final int BSTR = 1 << 19; // +STR + public static final int ESTR = 1 << 20; // -STR + // --------------------- + // utility flags + public static final int LAST = ESTR; + public static final int MASK = ((LAST << 1) - 1); + public static final int HAS_STR = SCLR|ALIA|ANCH|TAG_; +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index ccf7d4370..c36eb768f 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -6,7 +6,6 @@ /** * Interface with the shared librapidyaml library - * */ public class Rapidyaml { public static String RAPIDYAML_VERSION = "0.7.2"; @@ -19,13 +18,20 @@ private native int ys2edn_parse(long ryml2edn, String filename, private native int ys2edn_retry_get( long ryml2edn, byte[] edn, int edn_size ); - private final long ryml2edn; + private native long ys2evt_init(); + private native void ys2evt_destroy(long ryml2evt); + private native int ys2evt_parse(long ryml2evt, String filename, + byte[] ys, int ys_length, + int[] evt, int evt_length); + private final long ryml2evt; + public Rapidyaml() { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); this.ryml2edn = this.ys2edn_init(); + this.ryml2evt = this.ys2evt_init(); } // Likely bad idea to implement finalize: @@ -35,13 +41,14 @@ public Rapidyaml() { protected void finalize() throws Throwable { try { this.ys2edn_destroy(this.ryml2edn); + this.ys2evt_destroy(this.ryml2evt); } finally { super.finalize(); } } - public String parseYS(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { String filename = "yamlscript"; // fixme byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); int edn_size = 10 * src.length; @@ -58,4 +65,10 @@ public String parseYS(String srcstr) throws RuntimeException, org.rapidyaml.Yaml String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); return ret; } + + public int parseYsToEvt(byte[] src, int[] evts) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + String filename = "yamlscript"; // fixme + int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); + return required_size; + } } diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index 48e9eaf83..db62cc2bc 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -1,6 +1,8 @@ package org.rapidyaml; import org.rapidyaml.*; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; @@ -32,10 +34,110 @@ private void testEdn_(String ys, String expected) { Rapidyaml rapidyaml = new Rapidyaml(); try { - String actual = rapidyaml.parseYS(ys); + String actual = rapidyaml.parseYsToEdn(ys); try { - assertEquals(expected.length(), actual.length()); assertEquals(expected, actual); + assertEquals(expected.length(), actual.length()); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + catch (YamlParseErrorException e) { + fail("parse error:\n" + e.getMessage()); + } + } + + // the result is an array of integers, but we use this to simplify + // running the tests + private class ExpectedEvent + { + int flags; + int str_start; + int str_len; + String str; + ExpectedEvent(int flags) + { + this.flags = flags; + this.str_start = 0; + this.str_len = 0; + this.str = ""; + } + ExpectedEvent(int flags, int str_start, int str_len, String str) + { + this.flags = flags; + this.str_start = str_start; + this.str_len = str_len; + this.str = str; + } + int required_size() { return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; } + }; + private static int required_size_(ExpectedEvent[] evts) + { + int sz = 0; + for(int i = 0; i < evts.length; ++i) { + sz += evts[i].required_size(); + } + return sz; + } + + private void testEvt_(String ys, ExpectedEvent[] expected) + { + Rapidyaml rapidyaml = new Rapidyaml(); + try { + int[] actual = new int[2 * required_size_(expected)]; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + int numEvts = rapidyaml.parseYsToEvt(src, actual); + assertTrue(numEvts < actual.length); + try { + int ia = 0; + int ie = 0; + int status = 1; + while(true) { + if((ia < numEvts) != (ie < expected.length)) { + status = 0; + break; + } + if(ia >= numEvts) + break; + if(ie >= expected.length) + break; + int cmp = 1; + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; + if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { + cmp &= (ia + 2 < numEvts) ? 1 : 0; + if(cmp != 0) { + cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; + cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + if(cmp != 0) { + cmp &= (actual[ia + 1] >= 0) ? 1 : 0; + cmp &= (actual[ia + 2] >= 0) ? 1 : 0; + cmp &= (actual[ia + 1] + actual[ia + 2] <= src.length) ? 1 : 0; + if(cmp != 0) { + String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); + cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + } + else { + System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); + } + } + } + } + System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); + status &= cmp; + ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; + ++ie; + } + if(required_size_(expected) != numEvts) + status = 0; + assertEquals(1, status); } catch (Exception e) { System.err.println("expected:"); @@ -52,9 +154,8 @@ private void testEdn_(String ys, String expected) public void testPlainMap() { - testEdn_( - "a: 1" - , + String ys = "a: 1"; + testEdn_(ys, "(\n" + "{:+ \"+MAP\"}\n" + "{:+ \"=VAL\", := \"a\"}\n" + @@ -63,13 +164,23 @@ public void testPlainMap() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 1, "a"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 3, 1, "1"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testUtf8() { - testEdn_( - "𝄞: ✅" - , + String ys = "𝄞: ✅"; + testEdn_(ys, "(\n" + "{:+ \"+MAP\"}\n" + "{:+ \"=VAL\", := \"𝄞\"}\n" + @@ -78,11 +189,22 @@ public void testUtf8() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "𝄞"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "✅"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testLargeCase() { - testEdn_( + String ys = "--- !yamlscript/v0\n" + "foo: !\n" + "- {x: y}\n" + "- [x, y]\n" + @@ -96,10 +218,10 @@ public void testLargeCase() "- [1, 2, true, false, null]\n" + "- &anchor-1 !tag-1 foobar\n" + "---\n" + - "another: doc\n" - , + "another: doc\n"; + testEdn_(ys, "(\n" + - "{:+ \"+MAP\"}\n" + + "{:+ \"+MAP\", :! \"yamlscript/v0\"}\n" + "{:+ \"=VAL\", := \"foo\"}\n" + "{:+ \"+SEQ\", :! \"\"}\n" + "{:+ \"+MAP\", :flow true}\n" + @@ -134,6 +256,103 @@ public void testLargeCase() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC|Evt.EXPL), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 5, 13, "yamlscript/v0"), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 25, 0, ""), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.FLOW), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 29, 1, "x"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 32, 1, "y"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 38, 1, "x"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 41, 1, "y"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 46, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 53, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 61, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 70, 4, "foo\n"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 80, 4, "foo\n"), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 89, 1, "1"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 92, 1, "2"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 95, 4, "true"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 101, 5, "false"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 108, 4, "null"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 127, 5, "tag-1"), + new ExpectedEvent(Evt.VAL_|Evt.ANCH, 117, 8, "anchor-1"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 133, 6, "foobar"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.BDOC|Evt.EXPL), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 144, 7, "another"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 153, 3, "doc"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testFilterCase() + { + String ys = "" + + "plain: well\n" + + " a\n" + + " b\n" + + " c\n" + + "squo: 'single''quote'\n" + + "dquo: \"x\\t\\ny\"\n" + + "lit: |\n" + + " X\n" + + " Y\n" + + " Z\n" + + "fold: >\n" + + " U\n" + + " V\n" + + " W\n"; + testEdn_(ys, + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"plain\"}\n" + + "{:+ \"=VAL\", := \"well a b c\"}\n" + + "{:+ \"=VAL\", := \"squo\"}\n" + + "{:+ \"=VAL\", :' \"single'quote\"}\n" + + "{:+ \"=VAL\", := \"dquo\"}\n" + + "{:+ \"=VAL\", :$ \"x\\t\\ny\"}\n" + + "{:+ \"=VAL\", := \"lit\"}\n" + + "{:+ \"=VAL\", :| \"X\\nY\\nZ\\n\"}\n" + + "{:+ \"=VAL\", := \"fold\"}\n" + + "{:+ \"=VAL\", :> \"U V W\\n\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n"); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 5, "plain"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 7, 10, "well a b c"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 24, 4, "squo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 31, 12, "single'quote"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 46, 4, "dquo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 53, 4, "x\t\ny"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 61, 3, "lit"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 68, 6, "X\nY\nZ\n"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 89, 4, "fold"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 97, 6, "U V W\n"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testFailure() @@ -142,7 +361,7 @@ public void testFailure() String ys = ": : : :"; boolean gotit = false; try { - rapidyaml.parseYS(ys); + rapidyaml.parseYsToEdn(ys); } catch(YamlParseErrorException e) { gotit = true; From e2da18d5d7fda9b0c624a421e840c6d8347fad1b Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 15 Feb 2025 15:23:42 +0000 Subject: [PATCH 07/21] rapidyaml: Update to 0.8.0 --- common/vars.mk | 2 +- core/deps.edn | 2 +- core/project.clj | 2 +- rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/vars.mk b/common/vars.mk index acccf60af..72fb175a7 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -228,7 +228,7 @@ RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz #------------------------------------------------------------------------------ RAPIDYAML := $(ROOT)/rapidyaml -RAPIDYAML_VERSION := 0.7.2 +RAPIDYAML_VERSION := 0.8.0 RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml RAPIDYAML_BUILD_TYPE := Release diff --git a/core/deps.edn b/core/deps.edn index 7b157e2d3..439d0b5bb 100644 --- a/core/deps.edn +++ b/core/deps.edn @@ -4,7 +4,7 @@ org.clojure/data.json {:mvn/version "2.4.0"}, clj-commons/clj-yaml {:mvn/version "1.0.27"}, org.flatland/ordered {:mvn/version "1.15.11"}, - org.rapidyaml/rapidyaml {:mvn/version "0.7.2"}, + org.rapidyaml/rapidyaml {:mvn/version "0.8.0"}, org.snakeyaml/snakeyaml-engine {:mvn/version "2.7"}, babashka/babashka.pods {:mvn/version "0.2.0"}, babashka/fs {:mvn/version "0.5.20"}, diff --git a/core/project.clj b/core/project.clj index 9c9a81579..764e6397b 100644 --- a/core/project.clj +++ b/core/project.clj @@ -21,7 +21,7 @@ [org.clojure/data.json "2.4.0"] [clj-commons/clj-yaml "1.0.27"] [org.flatland/ordered "1.15.11"] - [org.rapidyaml/rapidyaml "0.7.2"] + [org.rapidyaml/rapidyaml "0.8.0"] [org.snakeyaml/snakeyaml-engine "2.7"] [babashka/babashka.pods "0.2.0"] [babashka/fs "0.5.20"] diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index c36eb768f..26e10fb0c 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -8,7 +8,7 @@ * Interface with the shared librapidyaml library */ public class Rapidyaml { - public static String RAPIDYAML_VERSION = "0.7.2"; + public static String RAPIDYAML_VERSION = "0.8.0"; private native long ys2edn_init(); private native void ys2edn_destroy(long ryml2edn); From 8536dedaad5d7524d6bc283410d3050245a8c0d4 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 17 Feb 2025 18:38:35 +0000 Subject: [PATCH 08/21] rapidyaml: YS integration updates --- common/vars.mk | 11 +- rapidyaml/Makefile | 6 +- rapidyaml/native/CMakeLists.txt | 1 + rapidyaml/native/Makefile | 18 +- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 123 +++++++++---- rapidyaml/native/ysparse_common.hpp | 44 +++++ rapidyaml/native/ysparse_edn.cpp | 33 +--- rapidyaml/native/ysparse_edn.hpp | 16 +- rapidyaml/native/ysparse_edn_handler.hpp | 14 ++ rapidyaml/native/ysparse_evt.cpp | 33 +--- rapidyaml/native/ysparse_evt.hpp | 16 +- rapidyaml/native/ysparse_evt_handler.hpp | 48 ++++- rapidyaml/native/ysparse_test.cpp | 173 +++++++++++++++++- .../main/java/org/rapidyaml/Rapidyaml.java | 20 +- .../java/org/rapidyaml/cmp/CmpEdnEvt.java | 52 ++++++ .../main/java/org/rapidyaml/cmp/manifest.mf | 1 + .../src/main/java/org/rapidyaml/cmp/run.sh | 17 ++ .../java/org/rapidyaml/RapidyamlTest.java | 73 +++++++- 18 files changed, 543 insertions(+), 156 deletions(-) create mode 100644 rapidyaml/native/ysparse_common.hpp create mode 100644 rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java create mode 100644 rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf create mode 100755 rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh diff --git a/common/vars.mk b/common/vars.mk index 72fb175a7..3c8ad8341 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -229,14 +229,17 @@ RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz RAPIDYAML := $(ROOT)/rapidyaml RAPIDYAML_VERSION := 0.8.0 -RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) +#RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) +RAPIDYAML_TAG ?= 8c37616378aefd376690a19459c31a56ce596b5e RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml -RAPIDYAML_BUILD_TYPE := Release -RAPIDYAML_DBG := 0 -RAPIDYAML_TIMED := 0 +RAPIDYAML_BUILD_TYPE ?= Release +RAPIDYAML_DBG ?= 0 +RAPIDYAML_TIMED ?= 0 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java +RAPIDYAML_JNI_H := $(ROOT)/rapidyaml/native/org_rapidyaml_Rapidyaml.h RAPIDYAML_SO := $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) RAPIDYAML_LIB := $(ROOT)/rapidyaml/native/librapidyaml.$(DOTLIB) RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index e29ba8c02..c01c44d09 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -47,11 +47,13 @@ realclean:: clean sysclean:: $(RM) -r $(MAVEN_REPOSITORY)/org/rapidyaml +jni: $(RAPIDYAML_JNI_H) + #------------------------------------------------------------------------------ $(RAPIDYAML_SO): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ -$(RAPIDYAML_LIB): +$(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ $(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED) @@ -68,5 +70,3 @@ $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) $(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) $(MAKE) -C native $@ - -crl: $(RAPIDYAML_CLASS) diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 61fd11e3c..4f8524e97 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -22,6 +22,7 @@ add_library(${libname} org_rapidyaml_Rapidyaml.cpp # # ysparse files + ysparse_common.hpp ysparse_edn_handler.hpp ysparse_edn_handler.cpp ysparse_edn.hpp diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index 91dd05ea8..f86b2e630 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -9,7 +9,7 @@ include $(COMMON)/python.mk # https://stackoverflow.com/questions/24493337/linking-static-library-with-jni THIS_DIR := $(shell pwd) -BDIR := $(THIS_DIR)/_build/$(BUILD_TYPE)-shared$(SHARED)-timed$(TIMED)-dbg$(DBG) +BDIR := $(THIS_DIR)/_build/$(RAPIDYAML_BUILD_TYPE)-shared$(RAPIDYAML_SHARED)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) RAPIDYAML_DEPS := \ Makefile \ @@ -18,7 +18,8 @@ RAPIDYAML_DEPS := \ $(RAPIDYAML_JNI_H) \ $(wildcard ./*pp) \ -CMK_ENV := +CMK_ENV ?= +CMK_FLAGS_EXTRA ?= CMK_FLAGS := \ -D CMAKE_BUILD_TYPE=$(RAPIDYAML_BUILD_TYPE) \ -D YSPARSE_TIMED=$(RAPIDYAML_TIMED) \ @@ -50,22 +51,25 @@ realclean:: clean #------------------------------------------------------------------------------ + +$(RAPIDYAML_LIB): rapidyaml build-static: $(RAPIDYAML_LIB) test-static: build-static - cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml-test-run + $(CMK_ENV) cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml-test-run $(RAPIDYAML_LIB): $(RAPIDYAML_DEPS) - cmake -S . -B $(BDIR)-static $(CMK_FLAGS) -D BUILD_SHARED_LIBS=OFF + $(CMK_ENV) cmake -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml cp -fv $(BDIR)-static/*.a $@ +$(RAPIDYAML_SO): rapidyaml build-shared: $(RAPIDYAML_SO) test-shared: build-shared cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml-test-run $(RAPIDYAML_SO): $(RAPIDYAML_DEPS) - cmake -S . -B $(BDIR)-shared $(CMK_FLAGS) -D BUILD_SHARED_LIBS=ON - cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml + $(CMK_ENV) cmake -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON + $(CMK_ENV) cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so -$(RAPIDYAML_H): $(RAPIDYAML_JAVA) +$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 63fa9efd1..48d3d8b87 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -85,6 +85,22 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) ys2evt_destroy((Ryml2Evt*)obj); } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_retry_get + * Signature: (Ljava/lang/Object;[BI)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, + jlong obj, + jbyteArray dst, jint dst_len) +{ + jboolean dst_is_copy; + jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); + int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); + env->ReleaseByteArrayElements(dst, dst_, 0); + return rc; +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_parse @@ -96,10 +112,27 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, jbyteArray src, jint src_len, jbyteArray dst, jint dst_len) { - jboolean src_is_copy, dst_is_copy; - jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); - jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); - const char *filename = env->GetStringUTFChars(jfilename, 0); + TIMED_SECTION("jni_ys2edn_parse", (size_type)src_len); + jbyte* src_ = nullptr; + jbyte* dst_ = nullptr; + jboolean src_is_copy = false; + jboolean dst_is_copy = false; + const char *filename = nullptr; + { + TIMED_SECTION("jni_ys2edn_parse/get_jni_data", (size_type)src_len); + { + TIMED_SECTION("jni_ys2edn_parse/GetByteArray(src)"); + src_ = env->GetByteArrayElements(src, &src_is_copy); + } + { + TIMED_SECTION("jni_ys2edn_parse/GetByteArray(dst)"); + dst_ = env->GetByteArrayElements(dst, &dst_is_copy); + } + { + TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); + filename = env->GetStringUTFChars(jfilename, 0); + } + } int rc = 0; try { @@ -107,13 +140,16 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, (char*)src_, src_len, (char*)dst_, dst_len); } - catch (Ryml2EdnParseError const& exc) + catch (YsParseError const& exc) { throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); } - env->ReleaseByteArrayElements(src, src_, 0); - env->ReleaseByteArrayElements(dst, dst_, 0); - env->ReleaseStringUTFChars(jfilename, filename); + { + TIMED_SECTION("jni_ys2edn_parse/release"); + env->ReleaseByteArrayElements(src, src_, 0); + env->ReleaseByteArrayElements(dst, dst_, 0); + env->ReleaseStringUTFChars(jfilename, filename); + } return rc; } @@ -128,10 +164,36 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, jbyteArray src, jint src_len, jintArray dst, jint dst_len) { - jboolean src_is_copy, dst_is_copy; - jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); - int* dst_ = env->GetIntArrayElements(dst, &dst_is_copy); - const char *filename = env->GetStringUTFChars(jfilename, 0); + TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + jbyte* src_ = nullptr; + int* dst_ = nullptr; + const char *filename = nullptr; + jboolean dst_is_copy = false; + jboolean src_is_copy = false; + { + TIMED_SECTION("jni_ys2evt_parse/get_jni_data", (size_type)src_len); + { + TIMED_SECTION("jni_ys2evt_parse/GetByteArray(src)"); + src_ = env->GetByteArrayElements(src, &src_is_copy); + } + { + // TODO this is __S__L__O__W__ + // + // the problem is with GetIntArrayElements(). we should + // use GetDirectBufferAddress(), but that requires a ByteBuffer->jobject + // instead of a int[]->jintArray + // + // see: + // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time + // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string + TIMED_SECTION("jni_ys2evt_parse/GetIntArray(dst)"); + dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + } + { + TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); + filename = env->GetStringUTFChars(jfilename, 0); + } + } int rc = 0; try { @@ -139,29 +201,26 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, (char*)src_, src_len, dst_, dst_len); } - catch (Ryml2EvtParseError const& exc) + catch (YsParseError const& exc) { throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); } - env->ReleaseByteArrayElements(src, src_, 0); - env->ReleaseIntArrayElements(dst, dst_, 0); - env->ReleaseStringUTFChars(jfilename, filename); - return rc; -} - -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_retry_get - * Signature: (Ljava/lang/Object;[BI)I - */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, - jlong obj, - jbyteArray dst, jint dst_len) -{ - jboolean src_is_copy, dst_is_copy; - jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); - int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); - env->ReleaseByteArrayElements(dst, dst_, 0); + { + TIMED_SECTION("jni_ys2evt_parse/release"); + { + TIMED_SECTION("jni_ys2evt_parse/ReleaseByteArray(src)"); + env->ReleaseByteArrayElements(src, src_, 0); + } + { + // TODO __S__L__O__W__ + TIMED_SECTION("jni_ys2evt_parse/ReleaseIntArray(dst)"); + env->ReleaseIntArrayElements(dst, dst_, 0); + } + { + TIMED_SECTION("jni_ys2evt_parse/ReleaseStringUTFChars()"); + env->ReleaseStringUTFChars(jfilename, filename); + } + } return rc; } diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp new file mode 100644 index 000000000..9026f951e --- /dev/null +++ b/rapidyaml/native/ysparse_common.hpp @@ -0,0 +1,44 @@ +#pragma once +#ifndef YSPARSE_COMMON_HPP_ +#define YSPARSE_COMMON_HPP_ + +#include + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +using size_type = int; + +struct YsParseError : public std::exception +{ + ryml::Location location; + std::string msg; + const char* what() const noexcept override { return msg.c_str(); } +}; + +#ifndef YSPARSE_TIMED +#define TIMED_SECTION(...) +#else +#include +#define TIMED_SECTION(...) timed_section C4_XCAT(ts, __LINE__)(__VA_ARGS__) +struct timed_section +{ + using myclock = std::chrono::steady_clock; + ryml::csubstr name; + size_type len; + myclock::time_point start; + timed_section(ryml::csubstr n, size_type len_=0) : name(n), len(len_), start(myclock::now()) {} + ~timed_section() + { + const std::chrono::duration t = myclock::now() - start; + fprintf(stderr, "%.6fms: %.*s", t.count(), (int)name.len, name.str); + if(len) + fprintf(stderr, " %.3fMB/s", (double)len / t.count() * 1.e-3); + fprintf(stderr, "\n"); + } +}; +#endif // YSPARSE_TIMED + +#endif // YSPARSE_COMMON_HPP_ diff --git a/rapidyaml/native/ysparse_edn.cpp b/rapidyaml/native/ysparse_edn.cpp index b074c4848..586c3a79e 100644 --- a/rapidyaml/native/ysparse_edn.cpp +++ b/rapidyaml/native/ysparse_edn.cpp @@ -1,34 +1,7 @@ #include "ysparse_edn.hpp" -#include - -namespace ryml { -using namespace c4; -using namespace c4::yml; -} // namespace ryml using namespace ryml; -#ifndef YSPARSE_TIMED -#define TIMED_SECTION(name) -#else -#include -#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) -struct timed_section -{ - using myclock = std::chrono::steady_clock; - using fmsecs = std::chrono::duration; - csubstr name; - myclock::time_point start; - fmsecs since() const { return myclock::now() - start; } - timed_section(csubstr n) : name(n), start(myclock::now()) {} - ~timed_section() - { - fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); - } -}; -#endif - - #if defined(__cplusplus) extern "C" { #endif @@ -39,7 +12,7 @@ extern "C" { namespace { C4_NORETURN void ys2edn_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) { - Ryml2EdnParseError exc; + YsParseError exc; exc.location = location; exc.msg.assign(msg, msg_len); throw exc; @@ -70,7 +43,7 @@ RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, char *ys, size_type ys_size, char *edn, size_type edn_size) { - TIMED_SECTION("ys2edn_parse"); + TIMED_SECTION("ys2edn_parse", ys_size); csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); { @@ -79,7 +52,7 @@ RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, ryml2edn->m_handler.reserve(ys_size > edn_size ? 3 * ys_size : edn_size, 256u); } { - TIMED_SECTION("parse_in_place"); + TIMED_SECTION("parse_in_place", ys_size); ryml2edn->m_parser.parse_in_place_ev(filename_, ys_); } return ys2edn_retry_get(ryml2edn, edn, edn_size); diff --git a/rapidyaml/native/ysparse_edn.hpp b/rapidyaml/native/ysparse_edn.hpp index 5a5878967..3dfe73606 100644 --- a/rapidyaml/native/ysparse_edn.hpp +++ b/rapidyaml/native/ysparse_edn.hpp @@ -2,20 +2,13 @@ #ifndef YSPARSE_EDN_HPP_ #define YSPARSE_EDN_HPP_ -#include #include "ysparse_edn_handler.hpp" - -namespace ryml { -using namespace c4; -using namespace c4::yml; -} // namespace ryml +#include "ysparse_common.hpp" #if defined(__cplusplus) extern "C" { #endif -using size_type = int; - struct RYML_EXPORT Ryml2Edn { ys::EventHandlerEdn::EventSink m_sink; @@ -34,13 +27,6 @@ struct RYML_EXPORT Ryml2Edn } }; -struct RYML_EXPORT Ryml2EdnParseError : public std::exception -{ - c4::yml::Location location; - std::string msg; - const char* what() const noexcept override { return msg.c_str(); } -}; - //----------------------------------------------------------------------------- diff --git a/rapidyaml/native/ysparse_edn_handler.hpp b/rapidyaml/native/ysparse_edn_handler.hpp index 5497fc609..89c36813f 100644 --- a/rapidyaml/native/ysparse_edn_handler.hpp +++ b/rapidyaml/native/ysparse_edn_handler.hpp @@ -343,6 +343,18 @@ struct EventHandlerEdn : public c4::yml::EventHandlerStack')) + result = result.offs(1, 1); _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); _RYML_CB_CHECK(m_stack.m_callbacks, result.str); return result; diff --git a/rapidyaml/native/ysparse_evt.cpp b/rapidyaml/native/ysparse_evt.cpp index f866aec75..9ef064b3f 100644 --- a/rapidyaml/native/ysparse_evt.cpp +++ b/rapidyaml/native/ysparse_evt.cpp @@ -1,34 +1,7 @@ #include "ysparse_evt.hpp" -#include - -namespace ryml { -using namespace c4; -using namespace c4::yml; -} // namespace ryml using namespace ryml; -#ifndef YS2PARSE_TIMED -#define TIMED_SECTION(name) -#else -#include -#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) -struct timed_section -{ - using myclock = std::chrono::steady_clock; - using fmsecs = std::chrono::duration; - csubstr name; - myclock::time_point start; - fmsecs since() const { return myclock::now() - start; } - timed_section(csubstr n) : name(n), start(myclock::now()) {} - ~timed_section() - { - fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); - } -}; -#endif - - #if defined(__cplusplus) extern "C" { #endif @@ -39,7 +12,7 @@ extern "C" { namespace { C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) { - Ryml2EvtParseError exc; + YsParseError exc; exc.location = location; exc.msg.assign(msg, msg_len); throw exc; @@ -70,7 +43,7 @@ RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, char *ys, size_type ys_size, evt::DataType *events, size_type evt_size) { - TIMED_SECTION("ys2evt_parse"); + TIMED_SECTION("ys2evt_parse", ys_size); csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); { @@ -79,7 +52,7 @@ RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, ryml2evt->m_handler.reserve(256u); } { - TIMED_SECTION("parse_in_place"); + TIMED_SECTION("parse_in_place", ys_size); ryml2evt->m_parser.parse_in_place_ev(filename_, ys_); } return (size_type)ryml2evt->m_handler.m_evt_curr; diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp index 7712f489f..cc231ae1a 100644 --- a/rapidyaml/native/ysparse_evt.hpp +++ b/rapidyaml/native/ysparse_evt.hpp @@ -2,20 +2,13 @@ #ifndef YSPARSE_EVT_HPP_ #define YSPARSE_EVT_HPP_ -#include #include "ysparse_evt_handler.hpp" - -namespace ryml { -using namespace c4; -using namespace c4::yml; -} // namespace ryml +#include "ysparse_common.hpp" #if defined(__cplusplus) extern "C" { #endif -using size_type = int; - struct RYML_EXPORT Ryml2Evt { ys::EventHandlerEvt m_handler; @@ -32,13 +25,6 @@ struct RYML_EXPORT Ryml2Evt } }; -struct RYML_EXPORT Ryml2EvtParseError : public std::exception -{ - c4::yml::Location location; - std::string msg; - const char* what() const noexcept override { return msg.c_str(); } -}; - //----------------------------------------------------------------------------- diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp index f00d7fe58..5892492d1 100644 --- a/rapidyaml/native/ysparse_evt_handler.hpp +++ b/rapidyaml/native/ysparse_evt_handler.hpp @@ -367,6 +367,27 @@ struct EventHandlerEvt : public c4::yml::EventHandlerStackevt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + csubstr _transform_directive(csubstr tag, substr output) { - if(tag.begins_with('!')) + if(tag.begins_with("!!")) + { + return tag; + } + else if(tag.begins_with('!')) { if(c4::yml::is_custom_tag(tag)) { diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 40a7e5a7a..5688d1ebf 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -57,6 +57,7 @@ struct Ys2EvtScoped }; +static bool showcmp = false; struct TestResult { uint32_t num_assertions; @@ -318,17 +319,18 @@ struct TestCase int status = true; size_t num_events_expected = evt.size(); size_t num_ints_expected = expected_size(evt); + bool same_size = true; if(actual.size() != num_ints_expected) { printf("------\n" - "FAIL:\n" + "FAIL: different size\n" "input:~~~%.*s~~~\n" "expected size:~~~%zu~~~\n" "actual size:~~~%zu~~~\n", (int)ys.len, ys.str, num_ints_expected, actual.size()); - status = false; + same_size = false; } for(size_t i = 0, ie = 0; ie < num_events_expected; ++ie) { @@ -339,7 +341,7 @@ struct TestCase break; } #define _testcmp(fmt, cmp, ...) \ - printf("status=%d cmp=%d evt=%zu i=%zu: " fmt "\n", status, (cmp), ie, i, ## __VA_ARGS__); \ + if(showcmp) { printf("status=%d cmp=%d evt=%zu i=%zu: " fmt "\n", status, (cmp), ie, i, ## __VA_ARGS__); } \ status &= (cmp) char actualbuf[100]; char expectedbuf[100]; @@ -381,7 +383,7 @@ struct TestCase "FAIL:\n" "input:~~~%.*s~~~\n", (int)ys.len, ys.str); - return status; + return status && same_size; } }; @@ -646,6 +648,169 @@ fold: > e(EDOC), e(ESTR), }), + // case ------------------------------------------------- + tc("- !!seq []", + R"(( +{:+ "+SEQ"} +{:+ "+SEQ", :! "tag:yaml.org,2002:seq", :flow true} +{:+ "-SEQ"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|BLCK), + e(VAL_|TAG_, 2, 5, "!!seq"), + e(VAL_|BSEQ|FLOW), + e(ESEQ), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"_(defn run(prompt session=nil): + when session: + write session _ :append true: |+ + Q: $(orig-prompt:trim) + A ($api-model): + $(answer:trim) +)_", + R"_(( +{:+ "+MAP"} +{:+ "=VAL", := "defn run(prompt session=nil)"} +{:+ "+MAP"} +{:+ "=VAL", := "when session"} +{:+ "+MAP"} +{:+ "=VAL", := "write session _ :append true"} +{:+ "=VAL", :| "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)_", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 28, "defn run(prompt session=nil)"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 32, 12, "when session"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 50, 28, "write session _ :append true"), + e(VAL_|SCLR|LITL, 83, 54, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n", needs_filter), + e(EMAP), + e(EMAP), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"_(#!/usr/bin/env ys-0 + +defn run(prompt session=nil): + session-text =: + when session && session:fs-e: + + answer =: + cond: + api-model =~ /^dall-e/: + openai-image(prompt).data.0.url + api-model.in?(anthropic-models): + anthropic(prompt):anthropic-message:format + api-model.in?(groq-models): + groq(prompt).choices.0.message.content:format + api-model.in?(openai-models): + openai-chat(prompt).choices.0.message.content:format + else: die() + + say: answer + + when session: + write session _ :append true: |+ + Q: $(orig-prompt:trim) + A ($api-model): + $(answer:trim) + +)_", + R"_(( +{:+ "+MAP"} +{:+ "=VAL", := "defn run(prompt session=nil)"} +{:+ "+MAP"} +{:+ "=VAL", := "session-text ="} +{:+ "+MAP"} +{:+ "=VAL", := "when session && session:fs-e"} +{:+ "=VAL", :: ""} +{:+ "-MAP"} +{:+ "=VAL", := "answer ="} +{:+ "+MAP"} +{:+ "=VAL", := "cond"} +{:+ "+MAP"} +{:+ "=VAL", := "api-model =~ /^dall-e/"} +{:+ "=VAL", := "openai-image(prompt).data.0.url"} +{:+ "=VAL", := "api-model.in?(anthropic-models)"} +{:+ "=VAL", := "anthropic(prompt):anthropic-message:format"} +{:+ "=VAL", := "api-model.in?(groq-models)"} +{:+ "=VAL", := "groq(prompt).choices.0.message.content:format"} +{:+ "=VAL", := "api-model.in?(openai-models)"} +{:+ "=VAL", := "openai-chat(prompt).choices.0.message.content:format"} +{:+ "=VAL", := "else"} +{:+ "=VAL", := "die()"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "=VAL", := "say"} +{:+ "=VAL", := "answer"} +{:+ "=VAL", := "when session"} +{:+ "+MAP"} +{:+ "=VAL", := "write session _ :append true"} +{:+ "=VAL", :| "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)_", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 21, 28, "defn run(prompt session=nil)"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 53, 14, "session-text ="), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 73, 28, "when session && session:fs-e"), + e(VAL_|SCLR|PLAI, 0, 0, ""), // note empty scalar pointing at the front + e(EMAP), + e(KEY_|SCLR|PLAI, 106, 8, "answer ="), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 120, 4, "cond"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 132, 22, "api-model =~ /^dall-e/"), + e(VAL_|SCLR|PLAI, 164, 31, "openai-image(prompt).data.0.url"), + e(KEY_|SCLR|PLAI, 202, 31, "api-model.in?(anthropic-models)"), + e(VAL_|SCLR|PLAI, 243, 42, "anthropic(prompt):anthropic-message:format"), + e(KEY_|SCLR|PLAI, 292, 26, "api-model.in?(groq-models)"), + e(VAL_|SCLR|PLAI, 328, 45, "groq(prompt).choices.0.message.content:format"), + e(KEY_|SCLR|PLAI, 380, 28, "api-model.in?(openai-models)"), + e(VAL_|SCLR|PLAI, 418, 52, "openai-chat(prompt).choices.0.message.content:format"), + e(KEY_|SCLR|PLAI, 477, 4, "else"), + e(VAL_|SCLR|PLAI, 483, 5, "die()"), + e(EMAP), + e(EMAP), + e(KEY_|SCLR|PLAI, 492, 3, "say"), + e(VAL_|SCLR|PLAI, 497, 6, "answer"), + e(KEY_|SCLR|PLAI, 507, 12, "when session"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 525, 28, "write session _ :append true"), + e(VAL_|SCLR|LITL, 558, 55, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n", needs_filter), + e(EMAP), + e(EMAP), + e(EMAP), + e(EDOC), + e(ESTR), + }), }; } // namespace diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 26e10fb0c..805cd9e08 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -27,7 +27,8 @@ private native int ys2evt_parse(long ryml2evt, String filename, int[] evt, int evt_length); private final long ryml2evt; - public Rapidyaml() { + public Rapidyaml() + { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); this.ryml2edn = this.ys2edn_init(); @@ -38,7 +39,8 @@ public Rapidyaml() { // // https://stackoverflow.com/questions/158174/why-would-you-ever-implement-finalize // - protected void finalize() throws Throwable { + protected void finalize() throws Throwable + { try { this.ys2edn_destroy(this.ryml2edn); this.ys2evt_destroy(this.ryml2evt); @@ -48,8 +50,11 @@ protected void finalize() throws Throwable { } } - public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + public String parseYsToEdn(String srcstr) + throws RuntimeException, org.rapidyaml.YamlParseErrorException + { String filename = "yamlscript"; // fixme +long t = System.nanoTime(); byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); int edn_size = 10 * src.length; byte[] edn = new byte[edn_size]; @@ -62,13 +67,20 @@ public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml throw new RuntimeException("inconsistent size"); } } +t = System.nanoTime() - t; +System.out.printf(" edn@java=%.6fms\n", (double)t / 1.e6); String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); return ret; } - public int parseYsToEvt(byte[] src, int[] evts) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + public int parseYsToEvt(byte[] src, int[] evts) + throws RuntimeException, org.rapidyaml.YamlParseErrorException + { String filename = "yamlscript"; // fixme +long t = System.nanoTime(); int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); +t = System.nanoTime() - t; +System.out.printf(" evt@java=%.6fms\n", (double)t / 1.e6); return required_size; } } diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java new file mode 100644 index 000000000..cf6654d81 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java @@ -0,0 +1,52 @@ +package cmp; + +import org.rapidyaml.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +// https://stackoverflow.com/questions/804466/how-do-i-create-executable-java-program +public class CmpEdnEvt +{ + public static void main(String[] args) throws Throwable + { + Rapidyaml rapidyaml = new Rapidyaml(); + int evtSize = 10000000; + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); + } + + public static void compareEdnEvt(int evtSize, Rapidyaml rapidyaml, String path) throws Throwable + { + String ys = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); + int[] evt = new int[evtSize]; + // + long t0 = System.nanoTime(); + byte[] ysBytes = ys.getBytes(StandardCharsets.UTF_8); + long tys2Bytes = System.nanoTime() - t0; + System.out.printf("-----\n"); + System.out.printf("%s\n", path); + System.out.printf(" ys.length=%d\n", ys.length()); + System.out.printf(" ys2bytes=%fms\n", (double)tys2Bytes / 1.e6); + // + System.out.printf(" edn...\n"); + t0 = System.nanoTime(); + String edn = rapidyaml.parseYsToEdn(ys); + long tEdn = System.nanoTime() - t0; + System.out.printf(" edn=%.6fms, length=%d -> %dB @%.3fMB/s\n", (double)tEdn / 1.e6, edn.length(), edn.length(), (double)ysBytes.length / tEdn * 1.e3); + // + System.out.printf(" evt...\n"); + t0 = System.nanoTime(); + int numEvts = rapidyaml.parseYsToEvt(ysBytes, evt); + long tEvt = System.nanoTime() - t0; + System.out.printf(" evt=%.6fms, length=%d -> %dB @%.3fMB/s\n", (double)tEvt / 1.e6, numEvts, 4*numEvts, (double)ysBytes.length / tEvt * 1.e3); + // + } +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf new file mode 100644 index 000000000..97704e895 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf @@ -0,0 +1 @@ +Main-class: cmp.CmpEdnEvt diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh new file mode 100755 index 000000000..d3a5e8f10 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -xe + +thisdir=$(dirname $0) +nativedir=$(cd $thisdir/../../../../../../native ; pwd) +rymldir=$(cd $nativedir/.. ; pwd) + +make -C $nativedir build RAPIDYAML_TIMED=1 +make -C $rymldir test RAPIDYAML_TIMED=1 + +cd $thisdir +jd=${jd:-/usr/lib/jvm/java-23-openjdk/bin} +$jd/javac -d . ../*.java +$jd/javac -d . -cp . CmpEdnEvt.java +$jd/jar -cmf manifest.mf CmpEdnEvt.jar cmp org +$jd/java -jar -Djava.library.path=$nativedir CmpEdnEvt.jar diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index db62cc2bc..bed13660e 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -1,11 +1,14 @@ package org.rapidyaml; import org.rapidyaml.*; -import java.nio.charset.StandardCharsets; -import java.nio.ByteBuffer; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; /** * Unit test for simple App. @@ -87,6 +90,7 @@ private static int required_size_(ExpectedEvent[] evts) private void testEvt_(String ys, ExpectedEvent[] expected) { + boolean dbglog = false; Rapidyaml rapidyaml = new Rapidyaml(); try { int[] actual = new int[2 * required_size_(expected)]; @@ -107,14 +111,16 @@ private void testEvt_(String ys, ExpectedEvent[] expected) if(ie >= expected.length) break; int cmp = 1; - System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + if(dbglog) + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { cmp &= (ia + 2 < numEvts) ? 1 : 0; if(cmp != 0) { cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; - System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + if(dbglog) + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); if(cmp != 0) { cmp &= (actual[ia + 1] >= 0) ? 1 : 0; cmp &= (actual[ia + 2] >= 0) ? 1 : 0; @@ -122,15 +128,18 @@ private void testEvt_(String ys, ExpectedEvent[] expected) if(cmp != 0) { String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; - System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + if(dbglog) + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); } else { - System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); + if(dbglog) + System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); } } } } - System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); + if(dbglog) + System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); status &= cmp; ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; ++ie; @@ -202,6 +211,56 @@ public void testUtf8() testEvt_(ys, expected); } + public void testTaggedInt() + { + String ys = "- !!int 42"; + testEdn_(ys, + "(\n" + + "{:+ \"+SEQ\"}\n" + + "{:+ \"=VAL\", :! \"tag:yaml.org,2002:int\", := \"42\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 2, 5, "!!int"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 8, 2, "42"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testTaggedSeq() + { + String ys = "- !!seq []"; + testEdn_(ys, + "(\n" + + "{:+ \"+SEQ\"}\n" + + "{:+ \"+SEQ\", :! \"tag:yaml.org,2002:seq\", :flow true}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 2, 5, "!!seq"), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); + } + public void testLargeCase() { String ys = "--- !yamlscript/v0\n" + From 9f13a167c2235b559377990c80e19080238c3b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Fri, 21 Feb 2025 14:00:13 -0500 Subject: [PATCH 09/21] rapidyaml: Integrate rapidyaml evt int parser.clj --- core/src/yamlscript/parser.clj | 115 +++++++++++++++--- core/test/yamlscript/parser_test.clj | 7 +- rapidyaml/native/ysparse_evt_handler.hpp | 69 ++++++----- rapidyaml/pom.xml | 2 +- .../src/main/java/org/rapidyaml/Evt.java | 69 ++++++----- .../main/java/org/rapidyaml/Rapidyaml.java | 16 ++- 6 files changed, 188 insertions(+), 90 deletions(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index ae9d390a5..1c799a78e 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -4,16 +4,15 @@ ;; The yamlscript.parser is responsible for parsing YAML into a sequence of ;; event objects. -;; TODO -;; - switch from snakeyaml to libfyaml (ffi) - (ns yamlscript.parser (:require [clojure.string :as str] [yamlscript.common]) (:import (java.util Optional) - (org.rapidyaml Rapidyaml) + (java.nio ByteBuffer) + (java.nio.charset StandardCharsets) + (org.rapidyaml Evt Rapidyaml) (org.snakeyaml.engine.v2.api LoadSettings) (org.snakeyaml.engine.v2.api.lowlevel Parse) (org.snakeyaml.engine.v2.exceptions Mark) @@ -31,20 +30,17 @@ SequenceEndEvent)) (:refer-clojure)) -(declare - parse-snakeyaml - parse-rapidyaml) +(declare parse-fn) (def shebang-ys #"^#!.*/env ys-0(?:\.d+\.\d+)?\n") (def shebang-bash #"^#!.*[/ ]bash\n+source +<\(") (defn parse "Parse a YAML string into a sequence of event objects." [yaml-string] - (let [has-code-mode-shebang (or (re-find shebang-ys yaml-string) + (let [has-code-mode-shebang (or + (re-find shebang-ys yaml-string) (re-find shebang-bash yaml-string)) - events (if (System/getenv "YS_PARSER_RAPIDYAML") - (parse-rapidyaml yaml-string) - (parse-snakeyaml yaml-string)) + events (parse-fn yaml-string) [first-event & rest-events] events first-event-tag (:! first-event) first-event (if (and has-code-mode-shebang @@ -68,19 +64,98 @@ (remove nil?) rest))) -(defn parse-rapidyaml [yaml-string] +(defn parse-rapidyaml-edn [yaml-string] (let [rapid-parser (new Rapidyaml)] (->> yaml-string - ( #(.parseYS ^Rapidyaml rapid-parser %1)) - #_(#(do (println %1) %1)) - (#(str/replace %1 #"^\(\n\{:\+\ \"\+DOC\"\}" "(")) - #_(#(do (println %1) %1)) + (#(.parseYsToEdn ^Rapidyaml rapid-parser %1)) read-string))) -(defn parse-test-case [yaml-string] - (->> yaml-string - parse - (remove (fn [ev] (= "DOC" (subs (:+ ev) 1)))))) +(defn event-type [mask] + (condp = (bit-and mask 2r11111111111) + Evt/BSTR nil + Evt/ESTR nil + Evt/BDOC "+DOC" + Evt/EDOC "-DOC" + Evt/BMAP "+MAP" + Evt/EMAP "-MAP" + Evt/BSEQ "+SEQ" + Evt/ESEQ "-SEQ" + Evt/SCLR "=VAL" + Evt/ALIA "=ALI" + nil)) + +(defmacro flag? [flag mask] + `(pos? (bit-and ~mask (. Evt ~flag)))) + +(defn get-skey [mask] + (condp = (bit-and mask 2r111110000000000000000) + Evt/PLAI := + Evt/SQUO :' + Evt/DQUO :$ + Evt/LITL :| + Evt/FOLD :> + nil)) + +(defn parse-rapidyaml-evt [^String yaml-string] + (rest + (let [parser ^Rapidyaml (new Rapidyaml) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array 5) + needed (.parseYsToEvt parser buffer masks) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array needed) + _ (.parseYsToEvt parser buffer masks) + get-str (fn [i] + (let [off (aget masks (inc i)) + len (aget masks (+ i 2))] + (reduce + (fn [slice i] (str slice (char (aget buffer i)))) + "" (range off (+ off len)))))] + + (loop [i 0, tag nil, anchor nil, events []] + (if (< i needed) + (let [mask (aget masks i) + type (event-type mask) + ; _ (WWW (Integer/toString mask 2) type) + sval (when (flag? HAS_STR mask) (get-str i)) + tag (if (flag? TAG_ mask) sval tag) + anchor (if (flag? ANCH mask) sval anchor) + event (when type + (let [event {:+ type} + event (if (flag? FLOW mask) + (assoc event :flow true) event) + event (if anchor (assoc event :& anchor) event) + event (if tag + (let [tag (str/replace tag + #"^!!" + "tag:yaml.org,2002:")] + (assoc event :! tag)) event) + event (if sval (assoc event + (get-skey mask) sval) event) + event (if (= type "=ALI") + {:+ "=ALI" :* sval} + event)] + event)) + events (if event (conj events event) events) + i (+ i (if sval 3 1))] + (if event + (recur i nil nil events) + (recur i tag anchor events))) + events))))) + +(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")] + (condp = parser-name + "" parse-snakeyaml + "snake" parse-snakeyaml + "ry1" parse-rapidyaml-edn + "ry2" parse-rapidyaml-evt + "ry-edn" parse-rapidyaml-edn + "ry-evt" parse-rapidyaml-evt + "ryml-edn" parse-rapidyaml-edn + "ryml-evt" parse-rapidyaml-evt + (die "Unknown YS_PARSER value: " parser-name)) + ; parse-rapidyaml-evt + parse-snakeyaml)) ;; ;; Functions to turn Java event objects into Clojure objects diff --git a/core/test/yamlscript/parser_test.clj b/core/test/yamlscript/parser_test.clj index 29420b7d8..ef04eec4b 100644 --- a/core/test/yamlscript/parser_test.clj +++ b/core/test/yamlscript/parser_test.clj @@ -8,6 +8,11 @@ [yamlscript.parser :as parser] [yamltest.core :as test])) +(defn parse-test-case [yaml-string] + (->> yaml-string + parser/parse + (remove (fn [ev] (= "DOC" (subs (:+ ev) 1)))))) + (test/load-yaml-test-files ["test/compiler-stack.yaml" "test/resolver.yaml" @@ -16,7 +21,7 @@ :test (fn [test] (->> test :yamlscript - parser/parse-test-case + parse-test-case (map pr-str) (map #(subs %1 4 (dec (count %1)))))) :want (fn [test] diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp index 5892492d1..a8f098b99 100644 --- a/rapidyaml/native/ysparse_evt_handler.hpp +++ b/rapidyaml/native/ysparse_evt_handler.hpp @@ -15,39 +15,46 @@ C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") namespace evt { using DataType = int32_t; typedef enum : DataType { - // --------------------- - // structure flags - KEY_ = 1 << 0, // as key - VAL_ = 1 << 1, // as value - SCLR = 1 << 2, // =VAL - BSEQ = 1 << 3, // +SEQ - ESEQ = 1 << 4, // -SEQ - BMAP = 1 << 5, // +MAP - EMAP = 1 << 6, // -MAP - ALIA = 1 << 7, // ref - ANCH = 1 << 8, // anchor - TAG_ = 1 << 9, // tag - // --------------------- - // style flags - PLAI = 1 << 10, // : (plain scalar) - SQUO = 1 << 11, // ' (single-quoted scalar) - DQUO = 1 << 12, // " (double-quoted scalar) - LITL = 1 << 13, // | (block literal scalar) - FOLD = 1 << 14, // > (block folded scalar) - FLOW = 1 << 15, // flow container: [] for seqs or {} for maps - BLCK = 1 << 16, // block container - // --------------------- - // document flags - BDOC = 1 << 17, // +DOC - EDOC = 1 << 18, // -DOC - EXPL = 1 << 21, // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) - BSTR = 1 << 19, // +STR - ESTR = 1 << 20, // -STR - // --------------------- - // utility flags + // Event types + BSTR = 1 << 0, // +STR + ESTR = 1 << 1, // -STR + BDOC = 1 << 2, // +DOC + EDOC = 1 << 3, // -DOC + BMAP = 1 << 4, // +MAP + EMAP = 1 << 5, // -MAP + BSEQ = 1 << 6, // +SEQ + ESEQ = 1 << 7, // -SEQ + SCLR = 1 << 8, // =VAL + ALIA = 1 << 9, // =ALI + + // Style flags + PLAI = 1 << 16, // : (plain scalar) + SQUO = 1 << 17, // ' (single-quoted scalar) + DQUO = 1 << 18, // " (double-quoted scalar) + LITL = 1 << 19, // | (block literal scalar) + FOLD = 1 << 20, // > (block folded scalar) + FLOW = 1 << 21, // flow container: + // [] for seqs or {} for maps + BLCK = 1 << 22, // block container + + // Modifiers + ANCH = 1 << 24, // anchor + TAG_ = 1 << 25, // tag + + // Structure flags + KEY_ = 1 << 26, // as key + VAL_ = 1 << 27, // as value + EXPL = 1 << 28, // --- (with BDOC) or + // ... (with EDOC) + // (may be fused with FLOW + // if needed) + + // Utility flags LAST = EXPL, MASK = (LAST << 1) - 1, - HAS_STR = SCLR|ALIA|ANCH|TAG_ // the event requires a string. the next two integers will provide respectively the string's offset and length + // the event requires a string. the next two integers will provide + // respectively the string's offset and length + HAS_STR = SCLR|ALIA|ANCH|TAG_ } EventFlags; } // namespace evt diff --git a/rapidyaml/pom.xml b/rapidyaml/pom.xml index 3efb8c7c4..54902b7fd 100644 --- a/rapidyaml/pom.xml +++ b/rapidyaml/pom.xml @@ -8,7 +8,7 @@ rapidyaml - 0.7.2 + 0.8.0 rapidyaml diff --git a/rapidyaml/src/main/java/org/rapidyaml/Evt.java b/rapidyaml/src/main/java/org/rapidyaml/Evt.java index 4f06211f3..5305610be 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Evt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Evt.java @@ -1,37 +1,44 @@ package org.rapidyaml; public class Evt { - // --------------------- - // structure flags - public static final int KEY_ = 1 << 0; // as key - public static final int VAL_ = 1 << 1; // as value - public static final int SCLR = 1 << 2; // =VAL - public static final int BSEQ = 1 << 3; // +SEQ - public static final int ESEQ = 1 << 4; // -SEQ - public static final int BMAP = 1 << 5; // +MAP - public static final int EMAP = 1 << 6; // -MAP - public static final int ALIA = 1 << 7; // ref - public static final int ANCH = 1 << 8; // anchor - public static final int TAG_ = 1 << 9; // tag - // --------------------- - // style flags - public static final int PLAI = 1 << 10; // : (plain scalar) - public static final int SQUO = 1 << 11; // ' (single-quoted scalar) - public static final int DQUO = 1 << 12; // " (double-quoted scalar) - public static final int LITL = 1 << 13; // | (block literal scalar) - public static final int FOLD = 1 << 14; // > (block folded scalar) - public static final int FLOW = 1 << 15; // flow container: [] for seqs or {} for maps - public static final int BLCK = 1 << 16; // block container - // --------------------- - // document flags - public static final int BDOC = 1 << 17; // +DOC - public static final int EDOC = 1 << 18; // -DOC - public static final int EXPL = 1 << 21; // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) - public static final int BSTR = 1 << 19; // +STR - public static final int ESTR = 1 << 20; // -STR - // --------------------- - // utility flags - public static final int LAST = ESTR; + // Event types + public static final int BSTR = 1 << 0; // +STR + public static final int ESTR = 1 << 1; // -STR + public static final int BDOC = 1 << 2; // +DOC + public static final int EDOC = 1 << 3; // -DOC + public static final int BMAP = 1 << 4; // +MAP + public static final int EMAP = 1 << 5; // -MAP + public static final int BSEQ = 1 << 6; // +SEQ + public static final int ESEQ = 1 << 7; // -SEQ + public static final int SCLR = 1 << 8; // =VAL + public static final int ALIA = 1 << 9; // =ALI + + // Style flags + public static final int PLAI = 1 << 16; // : (plain scalar) + public static final int SQUO = 1 << 17; // ' (single-quoted scalar) + public static final int DQUO = 1 << 18; // " (double-quoted scalar) + public static final int LITL = 1 << 19; // | (block literal scalar) + public static final int FOLD = 1 << 20; // > (block folded scalar) + + public static final int FLOW = 1 << 21; // flow container: + // [] for seqs or {} for maps + public static final int BLCK = 1 << 22; // block container + + // Modifiers + public static final int ANCH = 1 << 24; // anchor + public static final int TAG_ = 1 << 25; // tag + + // Structure flags + public static final int KEY_ = 1 << 26; // as key + public static final int VAL_ = 1 << 27; // as value + public static final int EXPL = 1 << 28; // --- (with BDOC) or + // ... (with EDOC) + // (may be fused with FLOW + // if needed) + + // Utility flags + public static final int LAST = EXPL; public static final int MASK = ((LAST << 1) - 1); public static final int HAS_STR = SCLR|ALIA|ANCH|TAG_; + } diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 805cd9e08..58d6cb729 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -54,7 +54,7 @@ public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { String filename = "yamlscript"; // fixme -long t = System.nanoTime(); + long t = System.nanoTime(); byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); int edn_size = 10 * src.length; byte[] edn = new byte[edn_size]; @@ -67,8 +67,10 @@ public String parseYsToEdn(String srcstr) throw new RuntimeException("inconsistent size"); } } -t = System.nanoTime() - t; -System.out.printf(" edn@java=%.6fms\n", (double)t / 1.e6); + if (System.getenv("YS_RYML_TIMER") != null) { + t = System.nanoTime() - t; + System.out.printf(" edn@java=%.6fms\n", (double)t / 1.e6); + } String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); return ret; } @@ -77,10 +79,12 @@ public int parseYsToEvt(byte[] src, int[] evts) throws RuntimeException, org.rapidyaml.YamlParseErrorException { String filename = "yamlscript"; // fixme -long t = System.nanoTime(); + long t = System.nanoTime(); int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); -t = System.nanoTime() - t; -System.out.printf(" evt@java=%.6fms\n", (double)t / 1.e6); + if (System.getenv("YS_RYML_TIMER") != null) { + t = System.nanoTime() - t; + System.out.printf(" evt@java=%.6fms\n", (double)t / 1.e6); + } return required_size; } } From 9506507959e9173cf20ff7de8f08b376d6eb0748 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 22 Feb 2025 19:44:37 +0000 Subject: [PATCH 10/21] cicd: Add a rapidyaml workflow --- .github/workflows/rapidyaml.yml | 109 ++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .github/workflows/rapidyaml.yml diff --git a/.github/workflows/rapidyaml.yml b/.github/workflows/rapidyaml.yml new file mode 100644 index 000000000..fd15daa1e --- /dev/null +++ b/.github/workflows/rapidyaml.yml @@ -0,0 +1,109 @@ +name: rapidyaml + +defaults: + run: + shell: bash -xeo pipefail {0} +'on': + workflow_dispatch: null + push: + branches: + - master + pull_request: + branches: + - master +env: + +jobs: + + # check that the spec'ed version of rapidyaml passes its own tests + ryml: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - bt: Debug + - bt: Release + env: + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: checkout rapidyaml + run: | + cd rapidyaml/native + make rapidyaml + - name: configure + run: | + cd rapidyaml/native/rapidyaml + cmake -B build -D CMAKE_BUILD_TYPE=${{matrix.bt}} -D RYML_BUILD_TESTS=ON + - name: build + run: | + cd rapidyaml/native/rapidyaml + cmake --build build --target ryml-test-build --parallel --verbose + - name: run tests + run: | + cd rapidyaml/native/rapidyaml + cmake --build build --target ryml-test-run + + # run the c++ tests + cpp: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + env: + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: run c++ tests + run: | + make -C rapidyaml/native test + + # run the java tests + java: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + env: + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - make -C rapidyaml| + cd rapidyaml + - name: run java tests + run: | + make -C rapidyaml test + + # run core and ys tests + ystest: + name: core/${{matrix.ysparser}} + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - ysparser: ryml-evt + #- ysparser: ryml-edn + env: + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: run core tests + run: | + . .profile + YS_PARSER=${{matrix.ysparser}} make test-core v=1 + - name: run ys tests + run: | + . .profile + YS_PARSER=${{matrix.ysparser}} make test-ys v=1 From 0e394c8f41e1dfb17b29519bfea5d11c134a3bad Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 22 Feb 2025 19:53:07 +0000 Subject: [PATCH 11/21] rapidyaml: Test parsing yamllm.ys --- rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java | 1 + rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java index cf6654d81..192ffc9a0 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java @@ -13,6 +13,7 @@ public static void main(String[] args) throws Throwable { Rapidyaml rapidyaml = new Rapidyaml(); int evtSize = 10000000; + compareEdnEvt(evtSize, rapidyaml, "./yamllm.ys"); compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh index d3a5e8f10..da951ab24 100755 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh @@ -9,7 +9,10 @@ rymldir=$(cd $nativedir/.. ; pwd) make -C $nativedir build RAPIDYAML_TIMED=1 make -C $rymldir test RAPIDYAML_TIMED=1 + cd $thisdir +wget https://raw.githubusercontent.com/yaml/yamllm/refs/heads/main/bin/yamllm.ys +ls -lFhp jd=${jd:-/usr/lib/jvm/java-23-openjdk/bin} $jd/javac -d . ../*.java $jd/javac -d . -cp . CmpEdnEvt.java From 3083ce59b28ca4166ef04caf657b649a6a92622b Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 23 Feb 2025 10:35:06 +0000 Subject: [PATCH 12/21] minor cleanup to tests --- rapidyaml/native/.gitignore | 1 + .../java/org/rapidyaml/RapidyamlTest.java | 219 ++++++++++-------- 2 files changed, 118 insertions(+), 102 deletions(-) diff --git a/rapidyaml/native/.gitignore b/rapidyaml/native/.gitignore index 8e53ad039..2a816d627 100644 --- a/rapidyaml/native/.gitignore +++ b/rapidyaml/native/.gitignore @@ -1,5 +1,6 @@ /rapidyaml/ /_build/ +/build/ /librapidyaml.* /.cache /compile_commands.json diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index bed13660e..357191860 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -77,7 +77,10 @@ private class ExpectedEvent this.str_len = str_len; this.str = str; } - int required_size() { return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; } + int required_size() + { + return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; + } }; private static int required_size_(ExpectedEvent[] evts) { @@ -90,7 +93,7 @@ private static int required_size_(ExpectedEvent[] evts) private void testEvt_(String ys, ExpectedEvent[] expected) { - boolean dbglog = false; + boolean dbglog = true; Rapidyaml rapidyaml = new Rapidyaml(); try { int[] actual = new int[2 * required_size_(expected)]; @@ -111,8 +114,8 @@ private void testEvt_(String ys, ExpectedEvent[] expected) if(ie >= expected.length) break; int cmp = 1; - if(dbglog) - System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + if(dbglog) + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { cmp &= (ia + 2 < numEvts) ? 1 : 0; @@ -120,7 +123,7 @@ private void testEvt_(String ys, ExpectedEvent[] expected) cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; if(dbglog) - System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); if(cmp != 0) { cmp &= (actual[ia + 1] >= 0) ? 1 : 0; cmp &= (actual[ia + 2] >= 0) ? 1 : 0; @@ -129,16 +132,16 @@ private void testEvt_(String ys, ExpectedEvent[] expected) String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; if(dbglog) - System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); } else { - if(dbglog) - System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); + if(dbglog) + System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); } } } } - if(dbglog) + if(dbglog) System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); status &= cmp; ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; @@ -161,6 +164,16 @@ private void testEvt_(String ys, ExpectedEvent[] expected) } } + ExpectedEvent mkev(int flags) + { + return new ExpectedEvent(flags); + } + + ExpectedEvent mkev(int flags, int offs, int len, String ref) + { + return new ExpectedEvent(flags, offs, len, ref); + } + public void testPlainMap() { String ys = "a: 1"; @@ -174,14 +187,14 @@ public void testPlainMap() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 1, "a"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 3, 1, "1"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 1, "a"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 3, 1, "1"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -199,14 +212,14 @@ public void testUtf8() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "𝄞"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "✅"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "𝄞"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "✅"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -223,14 +236,14 @@ public void testTaggedInt() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 2, 5, "!!int"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 8, 2, "42"), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.TAG_, 2, 5, "!!int"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 8, 2, "42"), + mkev(Evt.ESEQ), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -248,15 +261,15 @@ public void testTaggedSeq() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 2, 5, "!!seq"), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.TAG_, 2, 5, "!!seq"), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.ESEQ), + mkev(Evt.ESEQ), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -271,9 +284,11 @@ public void testLargeCase() "- 'foo'\n" + "- \"foo\"\n" + "- |\n" + - " foo\n" + + " foo\n" + + " literal\n" + "- >\n" + - " foo\n" + + " foo\n" + + " folded\n" + "- [1, 2, true, false, null]\n" + "- &anchor-1 !tag-1 foobar\n" + "---\n" + @@ -294,8 +309,8 @@ public void testLargeCase() "{:+ \"=VAL\", := \"foo\"}\n" + "{:+ \"=VAL\", :' \"foo\"}\n" + "{:+ \"=VAL\", :$ \"foo\"}\n" + - "{:+ \"=VAL\", :| \"foo\\n\"}\n" + - "{:+ \"=VAL\", :> \"foo\\n\"}\n" + + "{:+ \"=VAL\", :| \"foo\\nliteral\\n\"}\n" + + "{:+ \"=VAL\", :> \"foo folded\\n\"}\n" + "{:+ \"+SEQ\", :flow true}\n" + "{:+ \"=VAL\", := \"1\"}\n" + "{:+ \"=VAL\", := \"2\"}\n" + @@ -316,46 +331,46 @@ public void testLargeCase() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC|Evt.EXPL), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 5, 13, "yamlscript/v0"), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 25, 0, ""), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.FLOW), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 29, 1, "x"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 32, 1, "y"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 38, 1, "x"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 41, 1, "y"), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 46, 3, "foo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 53, 3, "foo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 61, 3, "foo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 70, 4, "foo\n"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 80, 4, "foo\n"), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 89, 1, "1"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 92, 1, "2"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 95, 4, "true"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 101, 5, "false"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 108, 4, "null"), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 127, 5, "tag-1"), - new ExpectedEvent(Evt.VAL_|Evt.ANCH, 117, 8, "anchor-1"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 133, 6, "foobar"), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.BDOC|Evt.EXPL), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 144, 7, "another"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 153, 3, "doc"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC|Evt.EXPL), + mkev(Evt.VAL_|Evt.TAG_, 5, 13, "yamlscript/v0"), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), + mkev(Evt.VAL_|Evt.TAG_, 25, 0, ""), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.BMAP|Evt.FLOW), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 29, 1, "x"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 32, 1, "y"), + mkev(Evt.EMAP), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 38, 1, "x"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 41, 1, "y"), + mkev(Evt.ESEQ), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 46, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.SQUO, 53, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.DQUO, 61, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.LITL, 70, 12, "foo\nliteral\n"), + mkev(Evt.VAL_|Evt.SCLR|Evt.FOLD, 98, 11, "foo folded\n"), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 124, 1, "1"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 127, 1, "2"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 130, 4, "true"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 136, 5, "false"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 143, 4, "null"), + mkev(Evt.ESEQ), + mkev(Evt.VAL_|Evt.TAG_, 162, 5, "tag-1"), + mkev(Evt.VAL_|Evt.ANCH, 152, 8, "anchor-1"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 168, 6, "foobar"), + mkev(Evt.ESEQ), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.BDOC|Evt.EXPL), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 179, 7, "another"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 188, 3, "doc"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -394,22 +409,22 @@ public void testFilterCase() "{:+ \"-DOC\"}\n" + ")\n"); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 5, "plain"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 7, 10, "well a b c"), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 24, 4, "squo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 31, 12, "single'quote"), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 46, 4, "dquo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 53, 4, "x\t\ny"), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 61, 3, "lit"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 68, 6, "X\nY\nZ\n"), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 89, 4, "fold"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 97, 6, "U V W\n"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 5, "plain"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 7, 10, "well a b c"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 24, 4, "squo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.SQUO, 31, 12, "single'quote"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 46, 4, "dquo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.DQUO, 53, 4, "x\t\ny"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 61, 3, "lit"), + mkev(Evt.VAL_|Evt.SCLR|Evt.LITL, 68, 6, "X\nY\nZ\n"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 89, 4, "fold"), + mkev(Evt.VAL_|Evt.SCLR|Evt.FOLD, 97, 6, "U V W\n"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } From d7b2000383e7e473f8cc1a44aafbca8fb22de43f Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 23 Feb 2025 11:17:16 +0000 Subject: [PATCH 13/21] improve workflow --- .github/workflows/rapidyaml.yml | 89 ++++++++++++++----- common/java.mk | 7 ++ common/vars.mk | 8 ++ rapidyaml/Makefile | 12 +-- rapidyaml/native/Makefile | 44 +++++---- .../java/org/rapidyaml/RapidyamlTest.java | 2 +- 6 files changed, 116 insertions(+), 46 deletions(-) diff --git a/.github/workflows/rapidyaml.yml b/.github/workflows/rapidyaml.yml index fd15daa1e..8b9ceb0c8 100644 --- a/.github/workflows/rapidyaml.yml +++ b/.github/workflows/rapidyaml.yml @@ -7,11 +7,8 @@ defaults: workflow_dispatch: null push: branches: - - master + - main pull_request: - branches: - - master -env: jobs: @@ -26,7 +23,6 @@ jobs: include: - bt: Debug - bt: Release - env: steps: - name: checkout (action) uses: actions/checkout@v4 @@ -48,43 +44,85 @@ jobs: cd rapidyaml/native/rapidyaml cmake --build build --target ryml-test-run - # run the c++ tests + # run the c++ tests, also in Debug to test with assertions cpp: runs-on: ubuntu-24.04 if: always() continue-on-error: false strategy: fail-fast: false - env: + matrix: + include: + - bt: Debug + - bt: Release steps: - name: checkout (action) uses: actions/checkout@v4 with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched - - name: run c++ tests + - name: check jni header up to date run: | + make -C rapidyaml/native -B jni jnicheck + - name: get rapidyaml + run: | + make -C rapidyaml/native rapidyaml + - name: c++ tests, no timing --------------------------------------------------- + run: echo + - name: cfg c++ tests, no timing + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + RAPIDYAML_TIMED=0 \ + make -C rapidyaml/native cfg + - name: build c++ tests, no timing + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + RAPIDYAML_TIMED=0 \ + make -C rapidyaml/native build + - name: run c++ tests, no timing + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + RAPIDYAML_TIMED=0 \ + make -C rapidyaml/native test + - name: c++ tests, with timing --------------------------------------------------- + run: echo + - name: cfg c++ tests, with timing + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + RAPIDYAML_TIMED=1 \ + make -C rapidyaml/native cfg + - name: build c++ tests, with timing + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + RAPIDYAML_TIMED=1 \ + make -C rapidyaml/native build + - name: run c++ tests, with timing + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + RAPIDYAML_TIMED=1 \ make -C rapidyaml/native test - # run the java tests + # run the java tests, also in Debug to test with assertions java: runs-on: ubuntu-24.04 if: always() continue-on-error: false strategy: fail-fast: false - env: + matrix: + include: + - bt: Debug + - bt: Release steps: - name: checkout (action) uses: actions/checkout@v4 with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched - - make -C rapidyaml| - cd rapidyaml - name: run java tests run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ make -C rapidyaml test - # run core and ys tests - ystest: - name: core/${{matrix.ysparser}} + # run core and ys tests, also in Debug to test with assertions + ys: + name: ys/${{matrix.ysparser}}/${{matrix.bt}} runs-on: ubuntu-24.04 if: always() continue-on-error: false @@ -92,9 +130,10 @@ jobs: fail-fast: false matrix: include: - - ysparser: ryml-evt - #- ysparser: ryml-edn - env: + - {ysparser: ryml-evt, bt: Debug} + - {ysparser: ryml-evt, bt: Release} + - {ysparser: ryml-edn, bt: Debug} + - {ysparser: ryml-edn, bt: Release} steps: - name: checkout (action) uses: actions/checkout@v4 @@ -102,8 +141,18 @@ jobs: - name: run core tests run: | . .profile - YS_PARSER=${{matrix.ysparser}} make test-core v=1 + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + YS_PARSER=${{matrix.ysparser}} \ + make test-core v=1 - name: run ys tests run: | . .profile - YS_PARSER=${{matrix.ysparser}} make test-ys v=1 + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + YS_PARSER=${{matrix.ysparser}} \ + make -C ys test-run v=1 + - name: run ys tests ? + run: | + . .profile + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + YS_PARSER=${{matrix.ysparser}} \ + make test-ys v=1 # is this the same as above? diff --git a/common/java.mk b/common/java.mk index bc9997d8c..f95a424de 100644 --- a/common/java.mk +++ b/common/java.mk @@ -5,6 +5,10 @@ export JAVA_HOME := $(GRAALVM_HOME) export PATH := $(JAVA_HOME)/bin:$(PATH) +MVN ?= mvn +JAR ?= $(GRAALVM_HOME)/bin/jar +JAVAC ?= $(GRAALVM_HOME)/bin/javac + YAMLSCRIPT_JAVA_INSTALLED := \ $(MAVEN_REPOSITORY)/org/yamlscript/yamlscript/maven-metadata-local.xml @@ -31,3 +35,6 @@ endif $(YAMLSCRIPT_JAVA_INSTALLED): $(YAMLSCRIPT_JAVA_SRC) $(MAKE) -C $(ROOT)/java install + +$(JAVAC): $(GRAALVM_INSTALLED) +$(JAR): $(GRAALVM_INSTALLED) diff --git a/common/vars.mk b/common/vars.mk index 3c8ad8341..d129647f1 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -249,6 +249,14 @@ RAPIDYAML_INSTALLED := \ $(RAPIDYAML_INSTALLED_DIR)/rapidyaml-$(RAPIDYAML_VERSION).jar +#------------------------------------------------------------------------------ +# Programs +#------------------------------------------------------------------------------ + +GIT ?= git +CMAKE ?= cmake + + #------------------------------------------------------------------------------ default:: diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index c01c44d09..af0d7ea2b 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -30,11 +30,11 @@ install:: $(RAPIDYAML_INSTALLED) test:: build $(JAVA_INSTALLED) $(MAKE) -C native $@ - mvn $@ + $(MVN) $@ test-x: build $(JAVA_INSTALLED) $(MAKE) -C native $@ - mvn -X -e test + $(MVN) -X -e test clean:: $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @@ -57,16 +57,16 @@ $(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ $(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED) - mvn install + $(MVN) install $(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED) - jar -cvf $@ $< + $(JAR) -cvf $@ $< $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) @# this doesn't work: - @#javac $< + @#$(JAVAC) $< @# ... but this does: - javac $(RAPIDYAML_JAVA) + $(JAVAC) $(RAPIDYAML_JAVA) $(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) $(MAKE) -C native $@ diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index f86b2e630..ec0037293 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -9,7 +9,7 @@ include $(COMMON)/python.mk # https://stackoverflow.com/questions/24493337/linking-static-library-with-jni THIS_DIR := $(shell pwd) -BDIR := $(THIS_DIR)/_build/$(RAPIDYAML_BUILD_TYPE)-shared$(RAPIDYAML_SHARED)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) +BDIR := $(THIS_DIR)/_build/$(RAPIDYAML_BUILD_TYPE)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) RAPIDYAML_DEPS := \ Makefile \ @@ -30,46 +30,52 @@ CMK_FLAGS := \ #------------------------------------------------------------------------------ default:: +cfg:: cfg-static cfg-shared build:: build-static build-shared test: test-static test-shared rapidyaml: mkdir -p $@ - git -C $@ init -q . - git -C $@ remote add origin $(RAPIDYAML_REPO) - git -C $@ fetch origin $(RAPIDYAML_TAG) - git -C $@ reset --hard FETCH_HEAD - git -C $@ submodule update --init --recursive + $(GIT) -C $@ init -q . + $(GIT) -C $@ remote add origin $(RAPIDYAML_REPO) + $(GIT) -C $@ fetch origin $(RAPIDYAML_TAG) + $(GIT) -C $@ reset --hard FETCH_HEAD + $(GIT) -C $@ submodule update --init --recursive clean:: $(RM) librapidyaml.* - $(RM) -r $(BDIR) + $(RM) -r _build $(RM) -r rapidyaml-install realclean:: clean $(RM) -r rapidyaml +jni: $(RAPIDYAML_JNI_H) + +jnicheck: jni + $(GIT) diff --exit-code $(RAPIDYAML_JNI_H) + #------------------------------------------------------------------------------ -$(RAPIDYAML_LIB): rapidyaml build-static: $(RAPIDYAML_LIB) +cfg-static: rapidyaml + $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF test-static: build-static - $(CMK_ENV) cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml-test-run -$(RAPIDYAML_LIB): $(RAPIDYAML_DEPS) - $(CMK_ENV) cmake -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF - cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run +$(RAPIDYAML_LIB): cfg-static $(RAPIDYAML_DEPS) + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --target rapidyaml --parallel --verbose cp -fv $(BDIR)-static/*.a $@ -$(RAPIDYAML_SO): rapidyaml build-shared: $(RAPIDYAML_SO) +cfg-shared: rapidyaml + $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON test-shared: build-shared - cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml-test-run -$(RAPIDYAML_SO): $(RAPIDYAML_DEPS) - $(CMK_ENV) cmake -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON - $(CMK_ENV) cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --target rapidyaml-test-run --verbose +$(RAPIDYAML_SO): cfg-shared $(RAPIDYAML_DEPS) + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --target rapidyaml --parallel --verbose cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so -$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) - javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work +$(RAPIDYAML_JNI_H): $(JAVAC) $(RAPIDYAML_JAVA) + $(JAVAC) -h . $(RAPIDYAML_JAVA) # $^ doesn't work diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index 357191860..b78b876be 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -93,7 +93,7 @@ private static int required_size_(ExpectedEvent[] evts) private void testEvt_(String ys, ExpectedEvent[] expected) { - boolean dbglog = true; + boolean dbglog = false; Rapidyaml rapidyaml = new Rapidyaml(); try { int[] actual = new int[2 * required_size_(expected)]; From 86043412a53eb6dce5cebc29338abbbafa526989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sun, 23 Feb 2025 09:01:02 -0500 Subject: [PATCH 14/21] core: Add a YS_PARSER_TIME variable to time parsing --- core/src/yamlscript/parser.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 1c799a78e..26b5248c0 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -40,7 +40,9 @@ (let [has-code-mode-shebang (or (re-find shebang-ys yaml-string) (re-find shebang-bash yaml-string)) - events (parse-fn yaml-string) + events (if (System/getenv "YS_PARSER_TIME") + (time (parse-fn yaml-string)) + (parse-fn yaml-string)) [first-event & rest-events] events first-event-tag (:! first-event) first-event (if (and has-code-mode-shebang From c4cab670fa3ec5ab17c1b00fede5cd1db14fb6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sun, 23 Feb 2025 09:01:23 -0500 Subject: [PATCH 15/21] core: Refactor parser to get rid of rapidyaml-edn --- core/src/yamlscript/parser.clj | 184 ++++++++++++++++----------------- 1 file changed, 90 insertions(+), 94 deletions(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 26b5248c0..8cfdd70a2 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -10,7 +10,6 @@ [yamlscript.common]) (:import (java.util Optional) - (java.nio ByteBuffer) (java.nio.charset StandardCharsets) (org.rapidyaml Evt Rapidyaml) (org.snakeyaml.engine.v2.api LoadSettings) @@ -57,6 +56,10 @@ (declare snake-event) +;; +;; SnakeYAML Parser +;; + ;; TODO - Set bigger buffer size in scanner class (defn parse-snakeyaml [yaml-string] (let [parser (new Parse (.build (LoadSettings/builder)))] @@ -66,99 +69,6 @@ (remove nil?) rest))) -(defn parse-rapidyaml-edn [yaml-string] - (let [rapid-parser (new Rapidyaml)] - (->> yaml-string - (#(.parseYsToEdn ^Rapidyaml rapid-parser %1)) - read-string))) - -(defn event-type [mask] - (condp = (bit-and mask 2r11111111111) - Evt/BSTR nil - Evt/ESTR nil - Evt/BDOC "+DOC" - Evt/EDOC "-DOC" - Evt/BMAP "+MAP" - Evt/EMAP "-MAP" - Evt/BSEQ "+SEQ" - Evt/ESEQ "-SEQ" - Evt/SCLR "=VAL" - Evt/ALIA "=ALI" - nil)) - -(defmacro flag? [flag mask] - `(pos? (bit-and ~mask (. Evt ~flag)))) - -(defn get-skey [mask] - (condp = (bit-and mask 2r111110000000000000000) - Evt/PLAI := - Evt/SQUO :' - Evt/DQUO :$ - Evt/LITL :| - Evt/FOLD :> - nil)) - -(defn parse-rapidyaml-evt [^String yaml-string] - (rest - (let [parser ^Rapidyaml (new Rapidyaml) - buffer (.getBytes yaml-string StandardCharsets/UTF_8) - masks (int-array 5) - needed (.parseYsToEvt parser buffer masks) - buffer (.getBytes yaml-string StandardCharsets/UTF_8) - masks (int-array needed) - _ (.parseYsToEvt parser buffer masks) - get-str (fn [i] - (let [off (aget masks (inc i)) - len (aget masks (+ i 2))] - (reduce - (fn [slice i] (str slice (char (aget buffer i)))) - "" (range off (+ off len)))))] - - (loop [i 0, tag nil, anchor nil, events []] - (if (< i needed) - (let [mask (aget masks i) - type (event-type mask) - ; _ (WWW (Integer/toString mask 2) type) - sval (when (flag? HAS_STR mask) (get-str i)) - tag (if (flag? TAG_ mask) sval tag) - anchor (if (flag? ANCH mask) sval anchor) - event (when type - (let [event {:+ type} - event (if (flag? FLOW mask) - (assoc event :flow true) event) - event (if anchor (assoc event :& anchor) event) - event (if tag - (let [tag (str/replace tag - #"^!!" - "tag:yaml.org,2002:")] - (assoc event :! tag)) event) - event (if sval (assoc event - (get-skey mask) sval) event) - event (if (= type "=ALI") - {:+ "=ALI" :* sval} - event)] - event)) - events (if event (conj events event) events) - i (+ i (if sval 3 1))] - (if event - (recur i nil nil events) - (recur i tag anchor events))) - events))))) - -(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")] - (condp = parser-name - "" parse-snakeyaml - "snake" parse-snakeyaml - "ry1" parse-rapidyaml-edn - "ry2" parse-rapidyaml-evt - "ry-edn" parse-rapidyaml-edn - "ry-evt" parse-rapidyaml-evt - "ryml-edn" parse-rapidyaml-edn - "ryml-evt" parse-rapidyaml-evt - (die "Unknown YS_PARSER value: " parser-name)) - ; parse-rapidyaml-evt - parse-snakeyaml)) - ;; ;; Functions to turn Java event objects into Clojure objects ;; @@ -241,5 +151,91 @@ (defmethod snake-event AliasEvent [event] (alias-val event)) (defmethod snake-event :default [_] nil) +;; +;; RapidYAML Parser +;; + +(defn event-type [mask] + (condp = (bit-and mask 2r11111111111) + Evt/BSTR nil + Evt/ESTR nil + Evt/BDOC "+DOC" + Evt/EDOC "-DOC" + Evt/BMAP "+MAP" + Evt/EMAP "-MAP" + Evt/BSEQ "+SEQ" + Evt/ESEQ "-SEQ" + Evt/SCLR "=VAL" + Evt/ALIA "=ALI" + nil)) + +(defmacro flag? [flag mask] + `(pos? (bit-and ~mask (. Evt ~flag)))) + +(defn get-skey [mask] + (condp = (bit-and mask 2r111110000000000000000) + Evt/PLAI := + Evt/SQUO :' + Evt/DQUO :$ + Evt/LITL :| + Evt/FOLD :> + nil)) + +(defn parse-rapidyaml [^String yaml-string] + (rest + (let [parser ^Rapidyaml (new Rapidyaml) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array 5) + needed (.parseYsToEvt parser buffer masks) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array needed) + _ (.parseYsToEvt parser buffer masks) + get-str (fn [i] + (let [off (aget masks (inc i)) + len (aget masks (+ i 2))] + (reduce + (fn [slice i] (str slice (char (aget buffer i)))) + "" (range off (+ off len)))))] + + (loop [i 0, tag nil, anchor nil, events []] + (if (< i needed) + (let [mask (aget masks i) + type (event-type mask) + ; _ (WWW (Integer/toString mask 2) type) + sval (when (flag? HAS_STR mask) (get-str i)) + tag (if (flag? TAG_ mask) sval tag) + anchor (if (flag? ANCH mask) sval anchor) + event (when type + (let [event {:+ type} + event (if (flag? FLOW mask) + (assoc event :flow true) event) + event (if anchor (assoc event :& anchor) event) + event (if tag + (let [tag (str/replace tag + #"^!!" + "tag:yaml.org,2002:")] + (assoc event :! tag)) event) + event (if sval (assoc event + (get-skey mask) sval) event) + event (if (= type "=ALI") + {:+ "=ALI" :* sval} + event)] + event)) + events (if event (conj events event) events) + i (+ i (if sval 3 1))] + (if event + (recur i nil nil events) + (recur i tag anchor events))) + events))))) + +(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")] + (condp = parser-name + "" parse-snakeyaml + "snake" parse-snakeyaml + "rapid" parse-rapidyaml + (die "Unknown YS_PARSER value: " parser-name)) + ; parse-rapidyaml-evt + parse-snakeyaml)) + (comment ) From c422a8c410d039c7d4940a24a3a98ae08e62c0d7 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 17 Feb 2025 18:38:35 +0000 Subject: [PATCH 16/21] rapidyaml: Add directbuffer JNI methods, save >14ms/call ----- ./yamllm.ys ys.length=9973 call:edn... java:ys2edn... 0,001334ms: jni_ys2edn_parse/GetByteArray(src) 2,034402ms: jni_ys2edn_parse/GetByteArray(dst) 0,001137ms: jni_ys2evt_parse/GetStringUTFChars() 2,054268ms: jni_ys2edn_parse/get_jni_data 4,855MB/s 0,012533ms: reset + reserve 0,171602ms: parse_in_place 58,117MB/s 0,002153ms: ys2edn_retry_get 0,196662ms: ys2edn_parse 50,711MB/s 0,904333ms: jni_ys2edn_parse/release 3,171271ms: jni_ys2edn_parse 3,145MB/s java:ys2edn: 3.207848ms 3.109MB/s 9973B call:edn: 5.332002ms 1.870MB/s 9973B call:ednBuf... java:ys2ednBuf... 0,000844ms: jni_ys2edn_parse/get_jni_data 11816,350MB/s 0,001602ms: reset + reserve 0,113475ms: parse_in_place 87,887MB/s 0,001107ms: ys2edn_retry_get 0,124749ms: ys2edn_parse 79,945MB/s 0,127846ms: jni_ys2edn_parse/call_parse 78,008MB/s 0,139938ms: jni_ys2edn_parse 71,267MB/s java:ys2ednBuf: 0.168987ms 59.016MB/s 9973B call:ednBuf: 1.989086ms 5.014MB/s 9973B call:evt... java:ys2evtBuf... 0,017777ms: jni_ys2evt_parse/GetByteArray(src) 8,084726ms: jni_ys2evt_parse/GetIntArray(dst) 0,003211ms: jni_ys2evt_parse/GetStringUTFChars() 8,132883ms: jni_ys2evt_parse/get_jni 1,226MB/s 0,001486ms: reset + reserve 0,124894ms: parse_in_place 79,852MB/s 0,131290ms: ys2evt_parse 75,962MB/s 0,139889ms: jni_ys2evt_parse/call_parse 71,292MB/s 0,001429ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,264938ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000834ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,282080ms: jni_ys2evt_parse/release 13,560444ms: jni_ys2evt_parse 0,735MB/s java:ys2evt: 13.608809ms 0.733MB/s 9973B call:evt: 19.330048ms 0.516MB/s 9973B call:evtBuf... java:ys2evtBuf... 0,001106ms: jni_ys2evt_parse/get_jni 9017,179MB/s 0,000639ms: reset + reserve 0,067489ms: parse_in_place 147,772MB/s 0,073704ms: ys2evt_parse 135,312MB/s 0,077551ms: jni_ys2evt_parse/call_parse 128,599MB/s 0,092300ms: jni_ys2evt_parse 108,050MB/s java:ys2evtBuf: 0.117877ms 84.605MB/s 9973B call:evtBuf: 5.819274ms 1.714MB/s 9973B ----- /home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml ys.length=2129 call:edn... java:ys2edn... 0,000978ms: jni_ys2edn_parse/GetByteArray(src) 2,058415ms: jni_ys2edn_parse/GetByteArray(dst) 0,001872ms: jni_ys2evt_parse/GetStringUTFChars() 2,082492ms: jni_ys2edn_parse/get_jni_data 1,022MB/s 0,002746ms: reset + reserve 0,034371ms: parse_in_place 61,942MB/s 0,000172ms: ys2edn_retry_get 0,045708ms: ys2edn_parse 46,578MB/s 0,773825ms: jni_ys2edn_parse/release 2,913689ms: jni_ys2edn_parse 0,731MB/s java:ys2edn: 2.921583ms 0.729MB/s 2129B call:edn: 4.583358ms 0.465MB/s 2129B call:ednBuf... java:ys2ednBuf... 0,001411ms: jni_ys2edn_parse/get_jni_data 1508,859MB/s 0,001593ms: reset + reserve 0,030772ms: parse_in_place 69,186MB/s 0,000219ms: ys2edn_retry_get 0,042616ms: ys2edn_parse 49,958MB/s 0,045226ms: jni_ys2edn_parse/call_parse 47,075MB/s 0,055543ms: jni_ys2edn_parse 38,331MB/s java:ys2ednBuf: 0.059724ms 35.647MB/s 2129B call:ednBuf: 0.811932ms 2.622MB/s 2129B call:evt... java:ys2evtBuf... 0,001258ms: jni_ys2evt_parse/GetByteArray(src) 8,076122ms: jni_ys2evt_parse/GetIntArray(dst) 0,002259ms: jni_ys2evt_parse/GetStringUTFChars() 8,100922ms: jni_ys2evt_parse/get_jni 0,263MB/s 0,000875ms: reset + reserve 0,022655ms: parse_in_place 93,975MB/s 0,031664ms: ys2evt_parse 67,237MB/s 0,034036ms: jni_ys2evt_parse/call_parse 62,551MB/s 0,001057ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,419873ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000952ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,435716ms: jni_ys2evt_parse/release 13,580720ms: jni_ys2evt_parse 0,157MB/s java:ys2evt: 13.590641ms 0.157MB/s 2129B call:evt: 19.339376ms 0.110MB/s 2129B call:evtBuf... java:ys2evtBuf... 0,002461ms: jni_ys2evt_parse/get_jni 865,095MB/s 0,000434ms: reset + reserve 0,025479ms: parse_in_place 83,559MB/s 0,033737ms: ys2evt_parse 63,106MB/s 0,038801ms: jni_ys2evt_parse/call_parse 54,870MB/s 0,056482ms: jni_ys2evt_parse 37,693MB/s java:ys2evtBuf: 0.062105ms 34.281MB/s 2129B call:evtBuf: 5.527016ms 0.385MB/s 2129B ----- /home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json ys.length=46825 call:edn... java:ys2edn... 0,021512ms: jni_ys2edn_parse/GetByteArray(src) 2,120464ms: jni_ys2edn_parse/GetByteArray(dst) 0,001398ms: jni_ys2evt_parse/GetStringUTFChars() 2,160656ms: jni_ys2edn_parse/get_jni_data 21,672MB/s 0,002448ms: reset + reserve 0,219965ms: parse_in_place 212,875MB/s 0,003424ms: ys2edn_retry_get 0,236103ms: ys2edn_parse 198,324MB/s 0,802344ms: jni_ys2edn_parse/release 3,211247ms: jni_ys2edn_parse 14,582MB/s java:ys2edn: 3.219141ms 14.546MB/s 46825B call:edn: 5.002863ms 9.360MB/s 46825B call:ednBuf... java:ys2ednBuf... 0,001421ms: jni_ys2edn_parse/get_jni_data 32952,146MB/s 0,001445ms: reset + reserve 0,183100ms: parse_in_place 255,735MB/s 0,003680ms: ys2edn_retry_get 0,197663ms: ys2edn_parse 236,893MB/s 0,201021ms: jni_ys2edn_parse/call_parse 232,936MB/s 0,213063ms: jni_ys2edn_parse 219,771MB/s java:ys2ednBuf: 0.219173ms 213.644MB/s 46825B call:ednBuf: 0.979759ms 47.792MB/s 46825B call:evt... java:ys2evtBuf... 0,007270ms: jni_ys2evt_parse/GetByteArray(src) 8,447130ms: jni_ys2evt_parse/GetIntArray(dst) 0,001928ms: jni_ys2evt_parse/GetStringUTFChars() 8,480045ms: jni_ys2evt_parse/get_jni 5,522MB/s 0,000848ms: reset + reserve 0,095276ms: parse_in_place 491,467MB/s 0,101429ms: ys2evt_parse 461,653MB/s 0,104018ms: jni_ys2evt_parse/call_parse 450,162MB/s 0,003393ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,601116ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000839ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,620317ms: jni_ys2evt_parse/release 14,210879ms: jni_ys2evt_parse 3,295MB/s java:ys2evt: 14.220492ms 3.293MB/s 46825B call:evt: 19.957447ms 2.346MB/s 46825B call:evtBuf... java:ys2evtBuf... 0,002473ms: jni_ys2evt_parse/get_jni 18934,492MB/s 0,000712ms: reset + reserve 0,071757ms: parse_in_place 652,550MB/s 0,077564ms: ys2evt_parse 603,695MB/s 0,081212ms: jni_ys2evt_parse/call_parse 576,577MB/s 0,097283ms: jni_ys2evt_parse 481,328MB/s java:ys2evtBuf: 0.101433ms 461.635MB/s 46825B call:evtBuf: 5.983458ms 7.826MB/s 46825B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml ys.length=393000 call:edn... java:ys2edn... 0,142092ms: jni_ys2edn_parse/GetByteArray(src) 2,126455ms: jni_ys2edn_parse/GetByteArray(dst) 0,001700ms: jni_ys2evt_parse/GetStringUTFChars() 2,285052ms: jni_ys2edn_parse/get_jni_data 171,987MB/s 0,002109ms: reset + reserve 8,302745ms: parse_in_place 47,334MB/s 0,166759ms: ys2edn_retry_get 8,485969ms: ys2edn_parse 46,312MB/s 0,999833ms: jni_ys2edn_parse/release 11,780786ms: jni_ys2edn_parse 33,359MB/s java:ys2edn: 11.790444ms 33.332MB/s 393000B call:edn: 14.609382ms 26.901MB/s 393000B call:ednBuf... java:ys2ednBuf... 0,002017ms: jni_ys2edn_parse/get_jni_data 194843,824MB/s 0,001862ms: reset + reserve 7,312239ms: parse_in_place 53,746MB/s 0,158746ms: ys2edn_retry_get 7,486341ms: ys2edn_parse 52,496MB/s 7,489023ms: jni_ys2edn_parse/call_parse 52,477MB/s 7,501143ms: jni_ys2edn_parse 52,392MB/s java:ys2ednBuf: 7.508824ms 52.338MB/s 393000B call:ednBuf: 8.480082ms 46.344MB/s 393000B call:evt... java:ys2evtBuf... 0,045725ms: jni_ys2evt_parse/GetByteArray(src) 8,427749ms: jni_ys2evt_parse/GetIntArray(dst) 0,002128ms: jni_ys2evt_parse/GetStringUTFChars() 8,498642ms: jni_ys2evt_parse/get_jni 46,243MB/s 0,000749ms: reset + reserve 2,269042ms: parse_in_place 173,201MB/s 2,277721ms: ys2evt_parse 172,541MB/s 2,280334ms: jni_ys2evt_parse/call_parse 172,343MB/s 0,030380ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,384011ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,001033ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,426795ms: jni_ys2evt_parse/release 16,211987ms: jni_ys2evt_parse 24,241MB/s java:ys2evt: 16.221876ms 24.227MB/s 393000B call:evt: 22.275141ms 17.643MB/s 393000B call:evtBuf... java:ys2evtBuf... 0,002334ms: jni_ys2evt_parse/get_jni 168380,464MB/s 0,000480ms: reset + reserve 2,258723ms: parse_in_place 173,992MB/s 2,268857ms: ys2evt_parse 173,215MB/s 2,271363ms: jni_ys2evt_parse/call_parse 173,024MB/s 2,286242ms: jni_ys2evt_parse 171,898MB/s java:ys2evtBuf: 2.296298ms 171.145MB/s 393000B call:evtBuf: 8.421487ms 46.666MB/s 393000B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml ys.length=783000 call:edn... java:ys2edn... 0,092991ms: jni_ys2edn_parse/GetByteArray(src) 0,835160ms: jni_ys2edn_parse/GetByteArray(dst) 0,001818ms: jni_ys2evt_parse/GetStringUTFChars() 0,945805ms: jni_ys2edn_parse/get_jni_data 827,866MB/s 0,002132ms: reset + reserve 14,063376ms: parse_in_place 55,677MB/s 0,322975ms: ys2edn_retry_get 14,405284ms: ys2edn_parse 54,355MB/s 1,018919ms: jni_ys2edn_parse/release 16,381964ms: jni_ys2edn_parse 47,796MB/s java:ys2edn: 16.394997ms 47.758MB/s 783000B call:edn: 23.038956ms 33.986MB/s 783000B call:ednBuf... java:ys2ednBuf... 0,001667ms: jni_ys2edn_parse/get_jni_data 469706,048MB/s 0,001485ms: reset + reserve 13,745988ms: parse_in_place 56,962MB/s 0,314753ms: ys2edn_retry_get 14,076574ms: ys2edn_parse 55,624MB/s 14,081208ms: jni_ys2edn_parse/call_parse 55,606MB/s 14,093569ms: jni_ys2edn_parse 55,557MB/s java:ys2ednBuf: 14.105751ms 55.509MB/s 783000B call:ednBuf: 15.099686ms 51.855MB/s 783000B call:evt... java:ys2evtBuf... 0,077090ms: jni_ys2evt_parse/GetByteArray(src) 8,194263ms: jni_ys2evt_parse/GetIntArray(dst) 0,002475ms: jni_ys2evt_parse/GetStringUTFChars() 8,297913ms: jni_ys2evt_parse/get_jni 94,361MB/s 0,001157ms: reset + reserve 4,913968ms: parse_in_place 159,342MB/s 4,923253ms: ys2evt_parse 159,041MB/s 4,926467ms: jni_ys2evt_parse/call_parse 158,937MB/s 0,073749ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,589391ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000823ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,674467ms: jni_ys2evt_parse/release 18,906420ms: jni_ys2evt_parse 41,415MB/s java:ys2evt: 18.918636ms 41.388MB/s 783000B call:evt: 21.359373ms 36.658MB/s 783000B call:evtBuf... java:ys2evtBuf... 0,002228ms: jni_ys2evt_parse/get_jni 351436,256MB/s 0,000757ms: reset + reserve 4,970937ms: parse_in_place 157,516MB/s 4,981731ms: ys2evt_parse 157,174MB/s 4,984258ms: jni_ys2evt_parse/call_parse 157,095MB/s 5,000104ms: jni_ys2evt_parse 156,597MB/s java:ys2evtBuf: 5.007758ms 156.357MB/s 783000B call:evtBuf: 10.750488ms 72.834MB/s 783000B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml ys.length=3894000 call:edn... java:ys2edn... 0,298067ms: jni_ys2edn_parse/GetByteArray(src) 1,110594ms: jni_ys2edn_parse/GetByteArray(dst) 0,001205ms: jni_ys2evt_parse/GetStringUTFChars() 1,427416ms: jni_ys2edn_parse/get_jni_data 2728,006MB/s 0,001941ms: reset + reserve 84,243172ms: parse_in_place 46,223MB/s 0,000034ms: ys2edn_retry_get 84,266029ms: ys2edn_parse 46,211MB/s 1,505299ms: jni_ys2edn_parse/release 87,211784ms: jni_ys2edn_parse 44,650MB/s java:ys2edn: 87.222893ms 44.644MB/s 3894000B java:ys2edn... 0,459691ms: jni_ys2edn_parse/GetByteArray(src) 5,877667ms: jni_ys2edn_parse/GetByteArray(dst) 0,001699ms: jni_ys2evt_parse/GetStringUTFChars() 6,357506ms: jni_ys2edn_parse/get_jni_data 612,504MB/s 0,135379ms: reset + reserve 75,206131ms: parse_in_place 51,778MB/s 2,077905ms: ys2edn_retry_get 77,437271ms: ys2edn_parse 50,286MB/s 2,688614ms: jni_ys2edn_parse/release 86,492966ms: jni_ys2edn_parse 45,021MB/s java:ys2edn: 86.502785ms 45.016MB/s 3894000B call:edn: 184.363892ms 21.121MB/s 3894000B call:ednBuf... java:ys2ednBuf... 0,001956ms: jni_ys2edn_parse/get_jni_data 1990797,568MB/s 0,002000ms: reset + reserve 71,676682ms: parse_in_place 54,327MB/s 0,000020ms: ys2edn_retry_get 71,692154ms: ys2edn_parse 54,316MB/s 71,694817ms: jni_ys2edn_parse/call_parse 54,314MB/s 71,707664ms: jni_ys2edn_parse 54,304MB/s java:ys2ednBuf: 71.715118ms 54.298MB/s 3894000B java:ys2ednBuf... 0,003304ms: jni_ys2edn_parse/get_jni_data 1178571,392MB/s 0,001898ms: reset + reserve 70,793625ms: parse_in_place 55,005MB/s 2,278467ms: ys2edn_retry_get 73,093513ms: ys2edn_parse 53,274MB/s 73,096558ms: jni_ys2edn_parse/call_parse 53,272MB/s 73,112923ms: jni_ys2edn_parse 53,260MB/s java:ys2ednBuf: 73.125305ms 53.251MB/s 3894000B call:ednBuf: 150.192001ms 25.927MB/s 3894000B call:evt... java:ys2evtBuf... 0,428085ms: jni_ys2evt_parse/GetByteArray(src) 8,114376ms: jni_ys2evt_parse/GetIntArray(dst) 0,001871ms: jni_ys2evt_parse/GetStringUTFChars() 8,565105ms: jni_ys2evt_parse/get_jni 454,635MB/s 0,000878ms: reset + reserve 21,515081ms: parse_in_place 180,989MB/s 21,525742ms: ys2evt_parse 180,900MB/s 21,528959ms: jni_ys2evt_parse/call_parse 180,873MB/s 0,447203ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,624267ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,001076ms: jni_ys2evt_parse/ReleaseStringUTFChars() 6,085247ms: jni_ys2evt_parse/release 36,185322ms: jni_ys2evt_parse 107,613MB/s java:ys2evt: 36.195786ms 107.582MB/s 3894000B call:evt: 40.740360ms 95.581MB/s 3894000B call:evtBuf... java:ys2evtBuf... 0,002188ms: jni_ys2evt_parse/get_jni 1779707,520MB/s 0,000720ms: reset + reserve 21,331545ms: parse_in_place 182,547MB/s 21,341160ms: ys2evt_parse 182,464MB/s 21,343897ms: jni_ys2evt_parse/call_parse 182,441MB/s 21,357212ms: jni_ys2evt_parse 182,327MB/s java:ys2evtBuf: 21.365120ms 182.260MB/s 3894000B call:evtBuf: 27.394588ms 142.145MB/s 3894000B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml ys.length=8784000 call:edn... java:ys2edn... 1,007704ms: jni_ys2edn_parse/GetByteArray(src) 1,068609ms: jni_ys2edn_parse/GetByteArray(dst) 0,001282ms: jni_ys2evt_parse/GetStringUTFChars() 2,093804ms: jni_ys2edn_parse/get_jni_data 4195,235MB/s 0,001659ms: reset + reserve 151,732574ms: parse_in_place 57,891MB/s 0,000019ms: ys2edn_retry_get 151,752396ms: ys2edn_parse 57,884MB/s 2,056400ms: jni_ys2edn_parse/release 155,915070ms: jni_ys2edn_parse 56,338MB/s java:ys2edn: 155.926132ms 56.334MB/s 8784000B java:ys2edn... 0,968508ms: jni_ys2edn_parse/GetByteArray(src) 11,112817ms: jni_ys2edn_parse/GetByteArray(dst) 0,001311ms: jni_ys2evt_parse/GetStringUTFChars() 12,105033ms: jni_ys2edn_parse/get_jni_data 725,649MB/s 0,002236ms: reset + reserve 136,510681ms: parse_in_place 64,347MB/s 4,374031ms: ys2edn_retry_get 140,909515ms: ys2edn_parse 62,338MB/s 5,479193ms: jni_ys2edn_parse/release 158,506912ms: jni_ys2edn_parse 55,417MB/s java:ys2edn: 158.517197ms 55.414MB/s 8784000B call:edn: 336.565643ms 26.099MB/s 8784000B call:ednBuf... java:ys2ednBuf... 0,001840ms: jni_ys2edn_parse/get_jni_data 4773913,088MB/s 0,002572ms: reset + reserve 136,738342ms: parse_in_place 64,239MB/s 0,000035ms: ys2edn_retry_get 136,756714ms: ys2edn_parse 64,231MB/s 136,759857ms: jni_ys2edn_parse/call_parse 64,229MB/s 136,772003ms: jni_ys2edn_parse 64,224MB/s java:ys2ednBuf: 136.783600ms 64.218MB/s 8784000B java:ys2ednBuf... 0,003399ms: jni_ys2edn_parse/get_jni_data 2584289,536MB/s 0,002487ms: reset + reserve 136,512512ms: parse_in_place 64,346MB/s 4,273001ms: ys2edn_retry_get 140,816635ms: ys2edn_parse 62,379MB/s 140,819626ms: jni_ys2edn_parse/call_parse 62,378MB/s 140,835098ms: jni_ys2edn_parse 62,371MB/s java:ys2ednBuf: 140.862976ms 62.358MB/s 8784000B call:ednBuf: 286.745575ms 30.633MB/s 8784000B call:evt... java:ys2evtBuf... 1,004054ms: jni_ys2evt_parse/GetByteArray(src) 8,102132ms: jni_ys2evt_parse/GetIntArray(dst) 0,002797ms: jni_ys2evt_parse/GetStringUTFChars() 9,131552ms: jni_ys2evt_parse/get_jni 961,939MB/s 0,000792ms: reset + reserve 49,813385ms: parse_in_place 176,338MB/s 49,824718ms: ys2evt_parse 176,298MB/s 49,827583ms: jni_ys2evt_parse/call_parse 176,288MB/s 1,033283ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,743564ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,001015ms: jni_ys2evt_parse/ReleaseStringUTFChars() 6,790257ms: jni_ys2evt_parse/release 65,755936ms: jni_ys2evt_parse 133,585MB/s java:ys2evt: 65.765808ms 133.565MB/s 8784000B call:evt: 72.783669ms 120.686MB/s 8784000B call:evtBuf... java:ys2evtBuf... 0,018377ms: jni_ys2evt_parse/get_jni 477988,768MB/s 0,000729ms: reset + reserve 50,390553ms: parse_in_place 174,318MB/s 50,402847ms: ys2evt_parse 174,276MB/s 50,405594ms: jni_ys2evt_parse/call_parse 174,266MB/s 50,435165ms: jni_ys2evt_parse 174,164MB/s java:ys2evtBuf: 50.444641ms 174.131MB/s 8784000B call:evtBuf: 56.956013ms 154.224MB/s 8784000B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json ys.length=3894002 call:edn... java:ys2edn... 0,330166ms: jni_ys2edn_parse/GetByteArray(src) 1,096978ms: jni_ys2edn_parse/GetByteArray(dst) 0,002020ms: jni_ys2evt_parse/GetStringUTFChars() 1,450136ms: jni_ys2edn_parse/get_jni_data 2685,267MB/s 0,002442ms: reset + reserve 71,063118ms: parse_in_place 54,796MB/s 0,000024ms: ys2edn_retry_get 71,079315ms: ys2edn_parse 54,784MB/s 1,544521ms: jni_ys2edn_parse/release 74,087189ms: jni_ys2edn_parse 52,560MB/s java:ys2edn: 74.098160ms 52.552MB/s 3894002B java:ys2edn... 0,426603ms: jni_ys2edn_parse/GetByteArray(src) 1,911429ms: jni_ys2edn_parse/GetByteArray(dst) 0,001644ms: jni_ys2evt_parse/GetStringUTFChars() 2,354758ms: jni_ys2edn_parse/get_jni_data 1653,674MB/s 0,001464ms: reset + reserve 69,939552ms: parse_in_place 55,677MB/s 2,210632ms: ys2edn_retry_get 72,166901ms: ys2edn_parse 53,958MB/s 2,714381ms: jni_ys2edn_parse/release 77,247299ms: jni_ys2edn_parse 50,410MB/s java:ys2edn: 77.256142ms 50.404MB/s 3894002B call:edn: 162.865555ms 23.909MB/s 3894002B call:ednBuf... java:ys2ednBuf... 0,002270ms: jni_ys2edn_parse/get_jni_data 1715419,392MB/s 0,002090ms: reset + reserve 70,507896ms: parse_in_place 55,228MB/s 0,000025ms: ys2edn_retry_get 70,521866ms: ys2edn_parse 55,217MB/s 70,524414ms: jni_ys2edn_parse/call_parse 55,215MB/s 70,535355ms: jni_ys2edn_parse 55,206MB/s java:ys2ednBuf: 70.543930ms 55.200MB/s 3894002B java:ys2ednBuf... 0,001999ms: jni_ys2edn_parse/get_jni_data 1947974,912MB/s 0,017705ms: reset + reserve 71,052269ms: parse_in_place 54,805MB/s 2,230100ms: ys2edn_retry_get 73,315285ms: ys2edn_parse 53,113MB/s 73,318420ms: jni_ys2edn_parse/call_parse 53,111MB/s 73,327438ms: jni_ys2edn_parse 53,104MB/s java:ys2ednBuf: 73.337601ms 53.097MB/s 3894002B call:ednBuf: 147.542007ms 26.392MB/s 3894002B call:evt... java:ys2evtBuf... 0,442496ms: jni_ys2evt_parse/GetByteArray(src) 8,129780ms: jni_ys2evt_parse/GetIntArray(dst) 0,002011ms: jni_ys2evt_parse/GetStringUTFChars() 8,595941ms: jni_ys2evt_parse/get_jni 453,005MB/s 0,000894ms: reset + reserve 21,720827ms: parse_in_place 179,275MB/s 21,730803ms: ys2evt_parse 179,193MB/s 21,734653ms: jni_ys2evt_parse/call_parse 179,161MB/s 0,421911ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,674166ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000743ms: jni_ys2evt_parse/ReleaseStringUTFChars() 6,109232ms: jni_ys2evt_parse/release 36,447289ms: jni_ys2evt_parse 106,839MB/s java:ys2evt: 36.457016ms 106.811MB/s 3894002B call:evt: 40.910328ms 95.184MB/s 3894002B call:evtBuf... java:ys2evtBuf... 0,002744ms: jni_ys2evt_parse/get_jni 1419096,960MB/s 0,000659ms: reset + reserve 21,838436ms: parse_in_place 178,310MB/s 21,850664ms: ys2evt_parse 178,210MB/s 21,853758ms: jni_ys2evt_parse/call_parse 178,185MB/s 21,869469ms: jni_ys2evt_parse 178,057MB/s java:ys2evtBuf: 21.878090ms 177.986MB/s 3894002B call:evtBuf: 28.053461ms 138.806MB/s 3894002B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml ys.length=8784002 call:edn... java:ys2edn... 1,888226ms: jni_ys2edn_parse/GetByteArray(src) 2,544243ms: jni_ys2edn_parse/GetByteArray(dst) 0,002253ms: jni_ys2evt_parse/GetStringUTFChars() 4,454822ms: jni_ys2edn_parse/get_jni_data 1971,796MB/s 0,002021ms: reset + reserve 136,622375ms: parse_in_place 64,294MB/s 0,000020ms: ys2edn_retry_get 136,640030ms: ys2edn_parse 64,286MB/s 2,347279ms: jni_ys2edn_parse/release 143,458054ms: jni_ys2edn_parse 61,230MB/s java:ys2edn: 143.469604ms 61.226MB/s 8784002B java:ys2edn... 1,991961ms: jni_ys2edn_parse/GetByteArray(src) 11,108978ms: jni_ys2edn_parse/GetByteArray(dst) 0,002018ms: jni_ys2evt_parse/GetStringUTFChars() 13,122984ms: jni_ys2edn_parse/get_jni_data 669,360MB/s 0,001842ms: reset + reserve 136,570084ms: parse_in_place 64,319MB/s 4,254542ms: ys2edn_retry_get 140,848358ms: ys2edn_parse 62,365MB/s 5,660944ms: jni_ys2edn_parse/release 159,643997ms: jni_ys2edn_parse 55,022MB/s java:ys2edn: 159.653351ms 55.019MB/s 8784002B call:edn: 323.496796ms 27.153MB/s 8784002B call:ednBuf... java:ys2ednBuf... 0,001906ms: jni_ys2edn_parse/get_jni_data 4608605,696MB/s 0,002339ms: reset + reserve 136,883469ms: parse_in_place 64,171MB/s 0,000038ms: ys2edn_retry_get 136,904175ms: ys2edn_parse 64,162MB/s 136,907028ms: jni_ys2edn_parse/call_parse 64,160MB/s 136,920837ms: jni_ys2edn_parse 64,154MB/s java:ys2ednBuf: 136.933243ms 64.148MB/s 8784002B java:ys2ednBuf... 0,002604ms: jni_ys2edn_parse/get_jni_data 3373272,576MB/s 0,002970ms: reset + reserve 136,688339ms: parse_in_place 64,263MB/s 4,348243ms: ys2edn_retry_get 141,061874ms: ys2edn_parse 62,271MB/s 141,064972ms: jni_ys2edn_parse/call_parse 62,269MB/s 141,080063ms: jni_ys2edn_parse 62,263MB/s java:ys2ednBuf: 141.092941ms 62.257MB/s 8784002B call:ednBuf: 287.268158ms 30.578MB/s 8784002B call:evt... java:ys2evtBuf... 0,985668ms: jni_ys2evt_parse/GetByteArray(src) 8,121219ms: jni_ys2evt_parse/GetIntArray(dst) 0,002764ms: jni_ys2evt_parse/GetStringUTFChars() 9,133806ms: jni_ys2evt_parse/get_jni 961,702MB/s 0,000989ms: reset + reserve 50,007858ms: parse_in_place 175,652MB/s 50,019047ms: ys2evt_parse 175,613MB/s 50,022156ms: jni_ys2evt_parse/call_parse 175,602MB/s 0,965973ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,844872ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000858ms: jni_ys2evt_parse/ReleaseStringUTFChars() 6,827047ms: jni_ys2evt_parse/release 65,990753ms: jni_ys2evt_parse 133,110MB/s java:ys2evt: 66.000282ms 133.090MB/s 8784002B call:evt: 73.179230ms 120.034MB/s 8784002B call:evtBuf... java:ys2evtBuf... 0,004144ms: jni_ys2evt_parse/get_jni 2119691,648MB/s 0,000963ms: reset + reserve 49,682919ms: parse_in_place 176,801MB/s 49,695457ms: ys2evt_parse 176,757MB/s 49,698326ms: jni_ys2evt_parse/call_parse 176,746MB/s 49,730080ms: jni_ys2evt_parse 176,634MB/s java:ys2evtBuf: 49.739895ms 176.599MB/s 8784002B call:evtBuf: 56.250931ms 156.157MB/s 8784002B --- common/vars.mk | 2 +- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 320 +++++++----- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 52 +- rapidyaml/native/ysparse_common.hpp | 13 +- .../main/java/org/rapidyaml/Rapidyaml.java | 174 +++++-- .../java/org/rapidyaml/cmp/CmpEdnEvt.java | 159 ++++-- .../java/org/rapidyaml/RapidyamlTest.java | 467 ++++++++++++------ 7 files changed, 821 insertions(+), 366 deletions(-) diff --git a/common/vars.mk b/common/vars.mk index d129647f1..ccb0cc1a2 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -234,7 +234,7 @@ RAPIDYAML_TAG ?= 8c37616378aefd376690a19459c31a56ce596b5e RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml RAPIDYAML_BUILD_TYPE ?= Release RAPIDYAML_DBG ?= 0 -RAPIDYAML_TIMED ?= 0 +RAPIDYAML_TIMED ?= 1 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \ diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 48d3d8b87..34730c6cc 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -3,47 +3,21 @@ #include "ysparse_evt.hpp" #include -#ifndef _Included_org_rapidyaml_Rapidyaml -#define _Included_org_rapidyaml_Rapidyaml #ifdef __cplusplus extern "C" { #endif -void throw_java_exception(JNIEnv * env, - const char* type, - const char* msg) -{ - jclass clazz = env->FindClass(type); - if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown - env->ThrowNew(clazz, msg); -} +static C4_NO_INLINE void throw_runtime_exception(JNIEnv * env, const char* msg); +static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg); + -void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg) +JNIEXPORT jlong JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) { - // see https://stackoverflow.com/questions/55013243/jni-custom-exceptions-with-more-than-one-parameter - jclass clazz = env->FindClass("org/rapidyaml/YamlParseErrorException"); - if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown - { - jstring jmsg = env->NewStringUTF(msg); - jint joffset = (jint)offset; - jint jline = (jint)line; - jint jcol = (jint)column; - // see https://www.rgagnon.com/javadetails/java-0286.html - // about the proper signature. - // we want (int, int, int, String): - const char * const signature = "(IIILjava/lang/String;)V"; - jmethodID ctor = env->GetMethodID(clazz, "", signature); - jobject jexc = env->NewObject(clazz, ctor, joffset, jline, jcol, jmsg); - env->Throw((jthrowable)jexc); // https://stackoverflow.com/questions/2455668/jni-cast-between-jobect-and-jthrowable - } + Ryml2Evt *obj = ys2evt_init(); + return (jlong)obj; } - -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_init - * Signature: ()Ljava/lang/Object; - */ JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) { @@ -51,61 +25,132 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) return (jlong)obj; } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_init - * Signature: ()J - */ -JNIEXPORT jlong JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) { - Ryml2Evt *obj = ys2evt_init(); - return (jlong)obj; + ys2evt_destroy((Ryml2Evt*)obj); } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_destroy - * Signature: (Ljava/lang/Object;)V - */ JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) { ys2edn_destroy((Ryml2Edn*)obj); } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_destroy - * Signature: (Ljava/lang/Object;)V - */ -JNIEXPORT void JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) + +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jbyteArray src, jint src_len, + jintArray dst, jint dst_len) { - ys2evt_destroy((Ryml2Evt*)obj); + TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + jbyte* src_ = nullptr; + int* dst_ = nullptr; + const char *filename = nullptr; + jboolean dst_is_copy = false; + jboolean src_is_copy = false; + { + TIMED_SECTION("jni_ys2evt_parse/get_jni", (size_type)src_len); + { + TIMED_SECTION("jni_ys2evt_parse/GetByteArray(src)"); + src_ = env->GetByteArrayElements(src, &src_is_copy); + } + { + // TODO this is __S__L__O__W__ + // + // the problem is with GetIntArrayElements(). we should + // use GetDirectBufferAddress(), but that requires a ByteBuffer->jobject + // instead of a int[]->jintArray + // + // see: + // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time + // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string + TIMED_SECTION("jni_ys2evt_parse/GetIntArray(dst)"); + dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + } + { + TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); + filename = env->GetStringUTFChars(jfilename, 0); + } + } + int rc = 0; + { + TIMED_SECTION("jni_ys2evt_parse/call_parse", (size_type)src_len); + try + { + rc = ys2evt_parse((Ryml2Evt*)obj, filename, + (char*)src_, src_len, + dst_, dst_len); + } + catch (YsParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + catch (std::exception const& exc) + { + throw_runtime_exception(env, exc.what()); + } + } + { + TIMED_SECTION("jni_ys2evt_parse/release"); + { + TIMED_SECTION("jni_ys2evt_parse/ReleaseByteArray(src)"); + env->ReleaseByteArrayElements(src, src_, 0); + } + { + // TODO __S__L__O__W__ + TIMED_SECTION("jni_ys2evt_parse/ReleaseIntArray(dst)"); + env->ReleaseIntArrayElements(dst, dst_, 0); + } + { + TIMED_SECTION("jni_ys2evt_parse/ReleaseStringUTFChars()"); + env->ReleaseStringUTFChars(jfilename, filename); + } + } + return rc; } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_retry_get - * Signature: (Ljava/lang/Object;[BI)I - */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, - jlong obj, - jbyteArray dst, jint dst_len) + +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jobject src, jint src_len, + jobject dst, jint dst_len) { - jboolean dst_is_copy; - jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); - int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); - env->ReleaseByteArrayElements(dst, dst_, 0); - return rc; + TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + char* src_ = nullptr; + int* dst_ = nullptr; + const char *filename = nullptr; + { + TIMED_SECTION("jni_ys2evt_parse/get_jni", (size_type)src_len); + src_ = (char*)env->GetDirectBufferAddress(src); + dst_ = (int*)env->GetDirectBufferAddress(dst); + filename = env->GetStringUTFChars(jfilename, 0); + if(!src_) + throw_runtime_exception(env, "null pointer: src"); + if(!dst_) + throw_runtime_exception(env, "null pointer: dst"); + } + { + TIMED_SECTION("jni_ys2evt_parse/call_parse", (size_type)src_len); + try + { + return ys2evt_parse((Ryml2Evt*)obj, filename, src_, src_len, dst_, dst_len); + } + catch (YsParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + catch (std::exception const& exc) + { + throw_runtime_exception(env, exc.what()); + } + } + return 0; // this is executed even if there is an exception } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_parse - * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I - */ + JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, jlong obj, jstring jfilename, @@ -153,78 +198,97 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, return rc; } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_parse - * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I - */ + JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, - jlong obj, jstring jfilename, - jbyteArray src, jint src_len, - jintArray dst, jint dst_len) +Java_org_rapidyaml_Rapidyaml_ys2edn_1parse_1buf(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jobject src, jint src_len, + jobject dst, jint dst_len) { - TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); - jbyte* src_ = nullptr; - int* dst_ = nullptr; + TIMED_SECTION("jni_ys2edn_parse", (size_type)src_len); + char* src_ = nullptr; + char* dst_ = nullptr; const char *filename = nullptr; - jboolean dst_is_copy = false; - jboolean src_is_copy = false; { - TIMED_SECTION("jni_ys2evt_parse/get_jni_data", (size_type)src_len); + TIMED_SECTION("jni_ys2edn_parse/get_jni_data", (size_type)src_len); + src_ = (char*)env->GetDirectBufferAddress(src); + dst_ = (char*)env->GetDirectBufferAddress(dst); + filename = env->GetStringUTFChars(jfilename, 0); + if(!src_) + throw_runtime_exception(env, "null pointer: src"); + if(!dst_) + throw_runtime_exception(env, "null pointer: dst"); + } + { + TIMED_SECTION("jni_ys2edn_parse/call_parse", (size_type)src_len); + try { - TIMED_SECTION("jni_ys2evt_parse/GetByteArray(src)"); - src_ = env->GetByteArrayElements(src, &src_is_copy); + return ys2edn_parse((Ryml2Edn*)obj, filename, src_, src_len, dst_, dst_len); } + catch (YsParseError const& exc) { - // TODO this is __S__L__O__W__ - // - // the problem is with GetIntArrayElements(). we should - // use GetDirectBufferAddress(), but that requires a ByteBuffer->jobject - // instead of a int[]->jintArray - // - // see: - // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time - // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string - TIMED_SECTION("jni_ys2evt_parse/GetIntArray(dst)"); - dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); } + catch (std::exception const& exc) { - TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); - filename = env->GetStringUTFChars(jfilename, 0); + throw_runtime_exception(env, exc.what()); } } - int rc = 0; - try - { - rc = ys2evt_parse((Ryml2Evt*)obj, filename, - (char*)src_, src_len, - dst_, dst_len); - } - catch (YsParseError const& exc) + return 0; // this is executed even if there is an exception +} + + +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, + jlong obj, + jobject dst, jint dst_len) +{ + char* dst_ = nullptr; { - throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + TIMED_SECTION("jni_ys2evt_retry_get/get_dst"); + dst_ = (char*)env->GetDirectBufferAddress(dst); + printf("edn: aqui 1 %p\n", dst_); } + int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); + return rc; +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +static C4_NO_INLINE void throw_java_exception(JNIEnv * env, const char* type, const char* msg) +{ + jclass clazz = env->FindClass(type); + if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown + env->ThrowNew(clazz, msg); +} + +static C4_NO_INLINE void throw_runtime_exception(JNIEnv *env, const char* msg) +{ + throw_java_exception(env, "java/lang/RuntimeException", msg); +} + +static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg) +{ + // see https://stackoverflow.com/questions/55013243/jni-custom-exceptions-with-more-than-one-parameter + jclass clazz = env->FindClass("org/rapidyaml/YamlParseErrorException"); + if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown { - TIMED_SECTION("jni_ys2evt_parse/release"); - { - TIMED_SECTION("jni_ys2evt_parse/ReleaseByteArray(src)"); - env->ReleaseByteArrayElements(src, src_, 0); - } - { - // TODO __S__L__O__W__ - TIMED_SECTION("jni_ys2evt_parse/ReleaseIntArray(dst)"); - env->ReleaseIntArrayElements(dst, dst_, 0); - } - { - TIMED_SECTION("jni_ys2evt_parse/ReleaseStringUTFChars()"); - env->ReleaseStringUTFChars(jfilename, filename); - } + jstring jmsg = env->NewStringUTF(msg); + jint joffset = (jint)offset; + jint jline = (jint)line; + jint jcol = (jint)column; + // see https://www.rgagnon.com/javadetails/java-0286.html + // about the proper signature. + // we want (int, int, int, String): + const char * const signature = "(IIILjava/lang/String;)V"; + jmethodID ctor = env->GetMethodID(clazz, "", signature); + jobject jexc = env->NewObject(clazz, ctor, joffset, jline, jcol, jmsg); + env->Throw((jthrowable)jexc); // https://stackoverflow.com/questions/2455668/jni-cast-between-jobect-and-jthrowable } - return rc; } #ifdef __cplusplus } #endif -#endif diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index daba52eff..74c7a5eb4 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -7,6 +7,14 @@ #ifdef __cplusplus extern "C" { #endif +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init + (JNIEnv *, jobject); + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_init @@ -17,51 +25,51 @@ JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_destroy + * Method: ys2evt_destroy * Signature: (J)V */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy (JNIEnv *, jobject, jlong); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_parse - * Signature: (JLjava/lang/String;[BI[BI)I + * Method: ys2edn_destroy + * Signature: (J)V */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse - (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint); +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy + (JNIEnv *, jobject, jlong); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_retry_get - * Signature: (J[BI)I + * Method: ys2evt_parse + * Signature: (JLjava/lang/String;[BI[II)I */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get - (JNIEnv *, jobject, jlong, jbyteArray, jint); +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_init - * Signature: ()J + * Method: ys2evt_parse_buf + * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/IntBuffer;I)I */ -JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init - (JNIEnv *, jobject); +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf + (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_destroy - * Signature: (J)V + * Method: ys2edn_parse + * Signature: (JLjava/lang/String;[BI[BI)I */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy - (JNIEnv *, jobject, jlong); +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_parse - * Signature: (JLjava/lang/String;[BI[II)I + * Method: ys2edn_parse_buf + * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/ByteBuffer;I)I */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse - (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse_1buf + (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); #ifdef __cplusplus } diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp index 9026f951e..ce3c3d1b4 100644 --- a/rapidyaml/native/ysparse_common.hpp +++ b/rapidyaml/native/ysparse_common.hpp @@ -20,7 +20,9 @@ struct YsParseError : public std::exception #ifndef YSPARSE_TIMED #define TIMED_SECTION(...) +#error #else +#include #include #define TIMED_SECTION(...) timed_section C4_XCAT(ts, __LINE__)(__VA_ARGS__) struct timed_section @@ -29,13 +31,18 @@ struct timed_section ryml::csubstr name; size_type len; myclock::time_point start; - timed_section(ryml::csubstr n, size_type len_=0) : name(n), len(len_), start(myclock::now()) {} + timed_section(ryml::csubstr n, size_type len_=0) + : name(n) + , len(len_) + , start(myclock::now()) + { + } ~timed_section() { - const std::chrono::duration t = myclock::now() - start; + const std::chrono::duration t = myclock::now() - start; fprintf(stderr, "%.6fms: %.*s", t.count(), (int)name.len, name.str); if(len) - fprintf(stderr, " %.3fMB/s", (double)len / t.count() * 1.e-3); + fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); fprintf(stderr, "\n"); } }; diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 58d6cb729..f77b519d7 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -3,28 +3,36 @@ import org.rapidyaml.YamlParseErrorException; import java.nio.charset.StandardCharsets; import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.ByteOrder; /** * Interface with the shared librapidyaml library */ -public class Rapidyaml { +public class Rapidyaml +{ public static String RAPIDYAML_VERSION = "0.8.0"; + private native long ys2evt_init(); private native long ys2edn_init(); - private native void ys2edn_destroy(long ryml2edn); - private native int ys2edn_parse(long ryml2edn, String filename, - byte[] ys, int ys_length, - byte[] edn, int edn_length); - private native int ys2edn_retry_get( - long ryml2edn, byte[] edn, int edn_size - ); - private final long ryml2edn; - private native long ys2evt_init(); private native void ys2evt_destroy(long ryml2evt); + private native void ys2edn_destroy(long ryml2edn); + private native int ys2evt_parse(long ryml2evt, String filename, byte[] ys, int ys_length, int[] evt, int evt_length); + private native int ys2evt_parse_buf(long ryml2evt, String filename, + ByteBuffer ys, int ys_length, + IntBuffer evt, int evt_length); + private native int ys2edn_parse(long ryml2edn, String filename, + byte[] ys, int ys_length, + byte[] edn, int edn_length); + private native int ys2edn_parse_buf(long ryml2edn, String filename, + ByteBuffer ys, int ys_length, + ByteBuffer edn, int edn_length); + + private final long ryml2edn; private final long ryml2evt; public Rapidyaml() @@ -33,6 +41,8 @@ public Rapidyaml() System.loadLibrary(library_name); this.ryml2edn = this.ys2edn_init(); this.ryml2evt = this.ys2evt_init(); + // TODO: receive this argument as ctor parameter + timingEnabled(System.getenv("YS_RYML_TIMER") != null); } // Likely bad idea to implement finalize: @@ -50,41 +60,131 @@ protected void finalize() throws Throwable } } - public String parseYsToEdn(String srcstr) - throws RuntimeException, org.rapidyaml.YamlParseErrorException + + //------------------------ + // EVT + //------------------------ + + public int parseYsToEvt(byte[] src, int[] evts) throws Exception + { + return parseYsToEvt("yamlscript", src, evts); + } + + public int parseYsToEvtBuf(ByteBuffer src, IntBuffer evt) throws Exception + { + return parseYsToEvtBuf("yamlscript", src, evt); + } + + public int parseYsToEvt(String filename, byte[] src, int[] evts) throws Exception + { + long t = timingStart("ys2evtBuf"); + int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); + timingStop("ys2evt", t, src.length); + return required_size; + } + + public int parseYsToEvtBuf(String filename, ByteBuffer src, IntBuffer evt) throws Exception + { + if(!src.isDirect()) + throw new RuntimeException("src must be direct"); + if(!evt.isDirect()) + throw new RuntimeException("evt must be direct"); + // the byte order for src does not matter + // but for evt it really does + if(evt.order() != ByteOrder.nativeOrder()) + throw new RuntimeException("evt byte order must be native"); + long t = timingStart("ys2evtBuf"); + evt.position(evt.capacity()); + int reqsize = ys2evt_parse_buf(this.ryml2evt, filename, src, src.position(), evt, evt.capacity()); + if(reqsize <= evt.capacity()) { + evt.position(reqsize); + } + timingStop("ys2evtBuf", t, src.position()); + return reqsize; + } + + public static IntBuffer mkIntBuffer(int numInts) { - String filename = "yamlscript"; // fixme - long t = System.nanoTime(); - byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); - int edn_size = 10 * src.length; - byte[] edn = new byte[edn_size]; - int required_size = ys2edn_parse(this.ryml2edn, filename, src, src.length, edn, edn_size); - if(required_size > edn_size) { - edn_size = required_size; - edn = new byte[edn_size]; - required_size = ys2edn_retry_get(this.ryml2edn, edn, edn_size); - if(required_size != edn_size) { - throw new RuntimeException("inconsistent size"); - } + ByteBuffer bb = ByteBuffer.allocateDirect(/*numBytes*/4 * numInts); + // !!! need to explicitly set the byte order to the native order + return bb.order(ByteOrder.nativeOrder()).asIntBuffer(); + } + + + //------------------------ + // EDN + //------------------------ + + public int parseYsToEdn(byte[] src, byte[] edn) throws Exception + { + return parseYsToEdn("yamlscript", src, edn); + } + + public int parseYsToEdnBuf(ByteBuffer src, ByteBuffer edn) throws Exception + { + return parseYsToEdnBuf("yamlscript", src, edn); + } + + public int parseYsToEdn(String filename, byte[] src, byte[] edn) throws Exception + { + long t = timingStart("ys2edn"); + int ret = ys2edn_parse(this.ryml2edn, filename, src, src.length, edn, edn.length); + timingStop("ys2edn", t, src.length); + return ret; + } + + public int parseYsToEdnBuf(String filename, ByteBuffer src, ByteBuffer edn) throws Exception + { + if(!src.isDirect()) + throw new RuntimeException("src must be direct"); + if(!edn.isDirect()) + throw new RuntimeException("edn must be direct"); + long t = timingStart("ys2ednBuf"); + edn.position(edn.capacity()); + int reqsize = ys2edn_parse_buf(this.ryml2edn, filename, src, src.position(), edn, edn.capacity()); + if(reqsize <= edn.capacity()) { + edn.position(reqsize); } - if (System.getenv("YS_RYML_TIMER") != null) { + timingStop("ys2ednBuf", t, src.position()); + return reqsize; + } + + + //------------------------ + // TIME + //------------------------ + + private boolean showTiming = true; + + public void timingEnabled(boolean yes) + { + showTiming = yes; + } + + private long timingStart(String name) + { + if(showTiming) { + System.out.printf(" java:%s...\n", name); + return System.nanoTime(); + } + return 0; + } + + private void timingStop(String name, long t) + { + if(showTiming) { t = System.nanoTime() - t; - System.out.printf(" edn@java=%.6fms\n", (double)t / 1.e6); + System.out.printf(" java:%s: %.6fms\n", name, (float)t/1.e6f); } - String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); - return ret; } - public int parseYsToEvt(byte[] src, int[] evts) - throws RuntimeException, org.rapidyaml.YamlParseErrorException + private void timingStop(String name, long t, int numBytes) { - String filename = "yamlscript"; // fixme - long t = System.nanoTime(); - int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); - if (System.getenv("YS_RYML_TIMER") != null) { + if(showTiming) { t = System.nanoTime() - t; - System.out.printf(" evt@java=%.6fms\n", (double)t / 1.e6); + float dt = (float)t; + float fb = (float)numBytes; + System.out.printf(" java:%s: %.6fms %.3fMB/s %dB\n", name, dt/1.e6f, 1.e3f*fb/dt, numBytes); } - return required_size; } } diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java index 192ffc9a0..cd86ae189 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java @@ -5,49 +5,144 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; // https://stackoverflow.com/questions/804466/how-do-i-create-executable-java-program public class CmpEdnEvt { - public static void main(String[] args) throws Throwable + public static void main(String[] args) throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); - int evtSize = 10000000; - compareEdnEvt(evtSize, rapidyaml, "./yamllm.ys"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); - } - - public static void compareEdnEvt(int evtSize, Rapidyaml rapidyaml, String path) throws Throwable - { - String ys = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); - int[] evt = new int[evtSize]; + rapidyaml.timingEnabled(true); + compareEdnEvt(rapidyaml, "./yamllm.ys"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); + } + + public static void compareEdnEvt(Rapidyaml rapidyaml, String path) throws Exception + { + String ys_ = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); + byte[] ys = ys_.getBytes(StandardCharsets.UTF_8); + byte[] ysarr = new byte[ys.length]; + ByteBuffer ysbuf = ByteBuffer.allocateDirect(ys.length); // - long t0 = System.nanoTime(); - byte[] ysBytes = ys.getBytes(StandardCharsets.UTF_8); - long tys2Bytes = System.nanoTime() - t0; System.out.printf("-----\n"); System.out.printf("%s\n", path); - System.out.printf(" ys.length=%d\n", ys.length()); - System.out.printf(" ys2bytes=%fms\n", (double)tys2Bytes / 1.e6); + System.out.printf(" ys.length=%d\n", ys.length); // - System.out.printf(" edn...\n"); - t0 = System.nanoTime(); - String edn = rapidyaml.parseYsToEdn(ys); - long tEdn = System.nanoTime() - t0; - System.out.printf(" edn=%.6fms, length=%d -> %dB @%.3fMB/s\n", (double)tEdn / 1.e6, edn.length(), edn.length(), (double)ysBytes.length / tEdn * 1.e3); + long t = timingStart("edn"); + String ednStr = callEdn(rapidyaml, ys, ysarr); + timingStop("edn", t, ys.length); // - System.out.printf(" evt...\n"); - t0 = System.nanoTime(); - int numEvts = rapidyaml.parseYsToEvt(ysBytes, evt); - long tEvt = System.nanoTime() - t0; - System.out.printf(" evt=%.6fms, length=%d -> %dB @%.3fMB/s\n", (double)tEvt / 1.e6, numEvts, 4*numEvts, (double)ysBytes.length / tEvt * 1.e3); + t = timingStart("ednBuf"); + ByteBuffer edn = callEdnBuf(rapidyaml, ys, ysbuf); + timingStop("ednBuf", t, ys.length); // + t = timingStart("evt"); + int[] evtarr = callEvt(rapidyaml, ys, ysarr); + timingStop("evt", t, ys.length); + // + t = timingStart("evtBuf"); + IntBuffer evtbuf = callEvtBuf(rapidyaml, ys, ysbuf); + timingStop("evtBuf", t, ys.length); + } + + static String callEdn(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception + { + System.arraycopy(src, 0, srcbuf, 0, src.length); + byte[] edn = new byte[10000000]; + int reqsize = rapidyaml.parseYsToEdn(srcbuf, edn); + if(reqsize > edn.length) { + edn = new byte[reqsize]; + System.arraycopy(src, 0, srcbuf, 0, src.length); + int reqsize2 = rapidyaml.parseYsToEdn(srcbuf, edn); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + String ret = new String(edn, 0, reqsize-1, StandardCharsets.UTF_8); + return ret; + } + + static ByteBuffer callEdnBuf(Rapidyaml rapidyaml, byte[] src, ByteBuffer srcbuf) throws Exception + { + srcbuf.position(0); + srcbuf.put(src); + ByteBuffer edn = ByteBuffer.allocateDirect(10000000); + int reqsize = rapidyaml.parseYsToEdnBuf(srcbuf, edn); + if(reqsize > edn.capacity()) { + edn = ByteBuffer.allocateDirect(reqsize); + srcbuf.position(0); + srcbuf.put(src); + int reqsize2 = rapidyaml.parseYsToEdnBuf(srcbuf, edn); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + edn.position(reqsize); + return edn; + } + + static int[] callEvt(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception + { + System.arraycopy(src, 0, srcbuf, 0, src.length); + int[] evt = new int[10000000]; + int reqsize = rapidyaml.parseYsToEvt(srcbuf, evt); + if(reqsize > evt.length) { + evt = new int[reqsize]; + System.arraycopy(src, 0, srcbuf, 0, src.length); + int reqsize2 = rapidyaml.parseYsToEvt(srcbuf, evt); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + return evt; + } + int[] ret = new int[reqsize]; + System.arraycopy(evt, 0, ret, 0, reqsize); + return ret; + } + + static IntBuffer callEvtBuf(Rapidyaml rapidyaml, byte[] src, ByteBuffer srcbuf) throws Exception + { + srcbuf.position(0); + srcbuf.put(src); + IntBuffer evt = Rapidyaml.mkIntBuffer(10000000); + int reqsize = rapidyaml.parseYsToEvtBuf(srcbuf, evt); + if(reqsize > evt.capacity()) { + evt = Rapidyaml.mkIntBuffer(reqsize); + srcbuf.position(0); + srcbuf.put(src); + int reqsize2 = rapidyaml.parseYsToEvtBuf(srcbuf, evt); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + evt.position(reqsize); + return evt; + } + + static private long timingStart(String name) + { + System.out.printf(" call:%s...\n", name); + return System.nanoTime(); + } + static private void timingStop(String name, long t) + { + t = System.nanoTime() - t; + System.out.printf(" call:%s: %.6fms\n", name, (float)t/1.e6f); + } + static private void timingStop(String name, long t, int numBytes) + { + t = System.nanoTime() - t; + float dt = (float)t; + float fb = (float)numBytes; + System.out.printf(" call:%s: %.6fms %.3fMB/s %dB\n", name, dt/1.e6f, 1.e3f*fb/dt, numBytes); } } diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index b78b876be..174dde7aa 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -6,10 +6,13 @@ import junit.framework.TestSuite; import java.nio.charset.StandardCharsets; import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.IntBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; + /** * Unit test for simple App. */ @@ -33,147 +36,6 @@ public static Test suite() return new TestSuite(RapidyamlTest.class); } - private void testEdn_(String ys, String expected) - { - Rapidyaml rapidyaml = new Rapidyaml(); - try { - String actual = rapidyaml.parseYsToEdn(ys); - try { - assertEquals(expected, actual); - assertEquals(expected.length(), actual.length()); - } - catch (Exception e) { - System.err.println("expected:"); - System.err.println(expected); - System.err.println("actual"); - System.err.println(actual); - throw e; - } - } - catch (YamlParseErrorException e) { - fail("parse error:\n" + e.getMessage()); - } - } - - // the result is an array of integers, but we use this to simplify - // running the tests - private class ExpectedEvent - { - int flags; - int str_start; - int str_len; - String str; - ExpectedEvent(int flags) - { - this.flags = flags; - this.str_start = 0; - this.str_len = 0; - this.str = ""; - } - ExpectedEvent(int flags, int str_start, int str_len, String str) - { - this.flags = flags; - this.str_start = str_start; - this.str_len = str_len; - this.str = str; - } - int required_size() - { - return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; - } - }; - private static int required_size_(ExpectedEvent[] evts) - { - int sz = 0; - for(int i = 0; i < evts.length; ++i) { - sz += evts[i].required_size(); - } - return sz; - } - - private void testEvt_(String ys, ExpectedEvent[] expected) - { - boolean dbglog = false; - Rapidyaml rapidyaml = new Rapidyaml(); - try { - int[] actual = new int[2 * required_size_(expected)]; - byte[] src = ys.getBytes(StandardCharsets.UTF_8); - int numEvts = rapidyaml.parseYsToEvt(src, actual); - assertTrue(numEvts < actual.length); - try { - int ia = 0; - int ie = 0; - int status = 1; - while(true) { - if((ia < numEvts) != (ie < expected.length)) { - status = 0; - break; - } - if(ia >= numEvts) - break; - if(ie >= expected.length) - break; - int cmp = 1; - if(dbglog) - System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); - cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; - if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { - cmp &= (ia + 2 < numEvts) ? 1 : 0; - if(cmp != 0) { - cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; - cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; - if(dbglog) - System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); - if(cmp != 0) { - cmp &= (actual[ia + 1] >= 0) ? 1 : 0; - cmp &= (actual[ia + 2] >= 0) ? 1 : 0; - cmp &= (actual[ia + 1] + actual[ia + 2] <= src.length) ? 1 : 0; - if(cmp != 0) { - String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); - cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; - if(dbglog) - System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); - } - else { - if(dbglog) - System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); - } - } - } - } - if(dbglog) - System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); - status &= cmp; - ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; - ++ie; - } - if(required_size_(expected) != numEvts) - status = 0; - assertEquals(1, status); - } - catch (Exception e) { - System.err.println("expected:"); - System.err.println(expected); - System.err.println("actual"); - System.err.println(actual); - throw e; - } - } - catch (YamlParseErrorException e) { - fail("parse error:\n" + e.getMessage()); - } - } - - ExpectedEvent mkev(int flags) - { - return new ExpectedEvent(flags); - } - - ExpectedEvent mkev(int flags, int offs, int len, String ref) - { - return new ExpectedEvent(flags, offs, len, ref); - } - public void testPlainMap() { String ys = "a: 1"; @@ -429,13 +291,13 @@ public void testFilterCase() testEvt_(ys, expected); } - public void testFailure() + public void testFailure() throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); String ys = ": : : :"; boolean gotit = false; try { - rapidyaml.parseYsToEdn(ys); + callEdn(ys); } catch(YamlParseErrorException e) { gotit = true; @@ -451,6 +313,325 @@ public void testFailure() catch(Exception e) { fail("wrong exception type"); } + //catch(Throwable e) { + // throw e; + // //fail("wrong exception type"); + //} assertTrue(gotit); } + + + private void testEdn_(String ys, String expected) + { + String actual; + try { + actual = callEdn(ys); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = ""; + } + try { + cmpEdn_(actual, expected); + } + catch (Exception e) { + System.err.printf("error: edn (no buf)"); + } + //------ + try { + actual = callEdnBuf(ys); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = ""; + } + try { + cmpEdn_(actual, expected); + } + catch (Exception e) { + System.err.printf("error: ednbuf"); + } + } + + private void testEvt_(String ys, ExpectedEvent[] expected) + { + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + byte[] srcbuf = new byte[src.length]; + int[] actual; + try { + actual = callEvt(src, srcbuf); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = new int[1]; + } + try { + cmpEvt_(ys, srcbuf, actual, expected); + } + catch (Exception e) { + System.err.printf("error: evt (no buf)"); + throw e; + } + //------ + src = ys.getBytes(StandardCharsets.UTF_8); + ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); + bbuf.put(src); + IntBuffer buf; + try { + buf = callEvtBuf(src, bbuf); + actual = buf2arr(buf); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = new int[1]; + } + try { + cmpEvt_(ys, srcbuf, actual, expected); + } + catch (Exception e) { + System.err.printf("error: evtbuf"); + throw e; + } + } + + private void cmpEdn_(String actual, String expected) throws Exception + { + try { + assertEquals(expected, actual); + assertEquals(expected.length(), actual.length()); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + + boolean dbglog = true; + private void cmpEvt_(String ys, byte[] src, int[] actual, ExpectedEvent[] expected) + { + if(dbglog) { + System.out.printf("----------------------\n~~~\n%s\n~~~\n", ys); + } + int numEvts = actual.length; + try { + int ia = 0; + int ie = 0; + int status = 1; + while(true) { + if((ia < numEvts) != (ie < expected.length)) { + System.out.printf("status=%d szActual=%d szExpected=%d\n", status, numEvts, ExpectedEvent.required_size_(expected)); + status = 0; + break; + } + if(ia >= numEvts) + break; + if(ie >= expected.length) + break; + int cmp = 1; + if(dbglog) + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; + if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { + cmp &= (ia + 2 < numEvts) ? 1 : 0; + if(cmp != 0) { + cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; + cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; + if(dbglog) + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + if(cmp != 0) { + cmp &= (actual[ia + 1] >= 0) ? 1 : 0; + cmp &= (actual[ia + 2] >= 0) ? 1 : 0; + cmp &= (actual[ia + 1] + actual[ia + 2] <= src.length) ? 1 : 0; + if(cmp != 0) { + String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); + cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; + if(dbglog) + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + } + else { + if(dbglog) + System.out.printf(" BAD RANGE len=%d", src.length); + } + } + } + } + if(dbglog) + System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); + status &= cmp; + ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; + ++ie; + } + if(ExpectedEvent.required_size_(expected) != numEvts) + status = 0; + assertEquals(1, status); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + + public static String buf2str(ByteBuffer edn) + { + int size = edn.position(); + size = size > 0 ? size - 1 : 0; + edn.position(0); + edn.limit(size); + return StandardCharsets.UTF_8.decode(edn).toString(); + } + + public static int[] buf2arr(IntBuffer evt) + { + int[] ret = new int[evt.position()]; + for(int i = 0; i < evt.position(); ++i) { + ret[i] = evt.get(i); + } + return ret; + } + + static String callEdn(String orig) throws Exception + { + byte[] src = orig.getBytes(StandardCharsets.UTF_8); + byte[] srcbuf = new byte[src.length]; + return callEdn(src, srcbuf); + } + + static String callEdnBuf(String orig) throws Exception + { + byte[] src = orig.getBytes(StandardCharsets.UTF_8); + ByteBuffer srcbuf = ByteBuffer.allocateDirect(src.length); + ByteBuffer edn = callEdnBuf(src, srcbuf); + return buf2str(edn); + } + + static String callEdn(byte[] src, byte[] srcbuf) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + System.arraycopy(src, 0, srcbuf, 0, src.length); + byte[] edn = new byte[10 * src.length]; + int reqsize = rapidyaml.parseYsToEdn(srcbuf, edn); + if(reqsize > edn.length) { + edn = new byte[reqsize]; + System.arraycopy(src, 0, srcbuf, 0, src.length); + int reqsize2 = rapidyaml.parseYsToEdn(srcbuf, edn); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + String ret = new String(edn, 0, reqsize-1, StandardCharsets.UTF_8); + return ret; + } + + static ByteBuffer callEdnBuf(byte[] src, ByteBuffer srcbuf) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + srcbuf.position(0); + srcbuf.put(src); + ByteBuffer edn = ByteBuffer.allocateDirect(10000); + int reqsize = rapidyaml.parseYsToEdnBuf(srcbuf, edn); + if(reqsize > edn.capacity()) { + edn = ByteBuffer.allocateDirect(reqsize); + srcbuf.position(0); + srcbuf.put(src); + int reqsize2 = rapidyaml.parseYsToEdnBuf(srcbuf, edn); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + edn.position(reqsize); + return edn; + } + + static int[] callEvt(byte[] src, byte[] srcbuf) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + System.arraycopy(src, 0, srcbuf, 0, src.length); + int[] evt = new int[10000]; + int reqsize = rapidyaml.parseYsToEvt(srcbuf, evt); + if(reqsize > evt.length) { + evt = new int[reqsize]; + System.arraycopy(src, 0, srcbuf, 0, src.length); + int reqsize2 = rapidyaml.parseYsToEvt(srcbuf, evt); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + return evt; + } + int[] ret = new int[reqsize]; + System.arraycopy(evt, 0, ret, 0, reqsize); + return ret; + } + + static IntBuffer callEvtBuf(byte[] src, ByteBuffer srcbuf) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + srcbuf.position(0); + srcbuf.put(src); + IntBuffer evt = Rapidyaml.mkIntBuffer(10000); + int reqsize = rapidyaml.parseYsToEvtBuf(srcbuf, evt); + if(reqsize > evt.capacity()) { + evt = Rapidyaml.mkIntBuffer(reqsize); + srcbuf.position(0); + srcbuf.put(src); + int reqsize2 = rapidyaml.parseYsToEvtBuf(srcbuf, evt); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + evt.position(reqsize); + return evt; + } + + ExpectedEvent mkev(int flags) + { + return new ExpectedEvent(flags); + } + + ExpectedEvent mkev(int flags, int offs, int len, String ref) + { + return new ExpectedEvent(flags, offs, len, ref); + } } + +// the result is an array of integers, but we use this to simplify +// running the tests +class ExpectedEvent +{ + int flags; + int str_start; + int str_len; + String str; + ExpectedEvent(int flags) + { + this.flags = flags; + this.str_start = 0; + this.str_len = 0; + this.str = ""; + } + ExpectedEvent(int flags, int str_start, int str_len, String str) + { + this.flags = flags; + this.str_start = str_start; + this.str_len = str_len; + this.str = str; + } + int required_size() + { + return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; + } + + public static int required_size_(ExpectedEvent[] evts) + { + int sz = 0; + for(int i = 0; i < evts.length; ++i) { + sz += evts[i].required_size(); + } + return sz; + } +}; From 5a4bcbdc3be19c8f376b191c4482d17e6fdab43a Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 23 Feb 2025 23:47:52 +0000 Subject: [PATCH 17/21] rapidyaml: Remove edn parsing --- .github/workflows/rapidyaml.yml | 6 +- core/src/yamlscript/parser.clj | 5 +- rapidyaml/native/CMakeLists.txt | 4 - rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 117 --- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 32 - rapidyaml/native/ysparse_edn.cpp | 81 -- rapidyaml/native/ysparse_edn.hpp | 56 -- rapidyaml/native/ysparse_edn_handler.cpp | 80 -- rapidyaml/native/ysparse_edn_handler.hpp | 696 ------------------ rapidyaml/native/ysparse_test.cpp | 253 +------ .../main/java/org/rapidyaml/Rapidyaml.java | 52 -- .../cmp/{CmpEdnEvt.java => CmpEvt.java} | 68 +- .../main/java/org/rapidyaml/cmp/manifest.mf | 2 +- .../src/main/java/org/rapidyaml/cmp/run.sh | 10 +- .../java/org/rapidyaml/RapidyamlTest.java | 210 +----- 15 files changed, 53 insertions(+), 1619 deletions(-) delete mode 100644 rapidyaml/native/ysparse_edn.cpp delete mode 100644 rapidyaml/native/ysparse_edn.hpp delete mode 100644 rapidyaml/native/ysparse_edn_handler.cpp delete mode 100644 rapidyaml/native/ysparse_edn_handler.hpp rename rapidyaml/src/main/java/org/rapidyaml/cmp/{CmpEdnEvt.java => CmpEvt.java} (53%) diff --git a/.github/workflows/rapidyaml.yml b/.github/workflows/rapidyaml.yml index 8b9ceb0c8..6348fa3cd 100644 --- a/.github/workflows/rapidyaml.yml +++ b/.github/workflows/rapidyaml.yml @@ -130,10 +130,8 @@ jobs: fail-fast: false matrix: include: - - {ysparser: ryml-evt, bt: Debug} - - {ysparser: ryml-evt, bt: Release} - - {ysparser: ryml-edn, bt: Debug} - - {ysparser: ryml-edn, bt: Release} + - {ysparser: ryml, bt: Debug} + - {ysparser: ryml, bt: Release} steps: - name: checkout (action) uses: actions/checkout@v4 diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 8cfdd70a2..96be8683e 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -233,8 +233,11 @@ "" parse-snakeyaml "snake" parse-snakeyaml "rapid" parse-rapidyaml + "ryml" parse-rapidyaml + ; TODO: + ;"rapid-buf" parse-rapidyaml-buf + ;"ryml-buf" parse-rapidyaml-buf (die "Unknown YS_PARSER value: " parser-name)) - ; parse-rapidyaml-evt parse-snakeyaml)) (comment diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 4f8524e97..aae90d5b5 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -23,10 +23,6 @@ add_library(${libname} # # ysparse files ysparse_common.hpp - ysparse_edn_handler.hpp - ysparse_edn_handler.cpp - ysparse_edn.hpp - ysparse_edn.cpp ysparse_evt_handler.hpp ysparse_evt_handler.cpp ysparse_evt.hpp diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 34730c6cc..02da00d71 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -1,7 +1,5 @@ #include -#include "ysparse_edn.hpp" #include "ysparse_evt.hpp" -#include #ifdef __cplusplus extern "C" { @@ -18,25 +16,12 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) return (jlong)obj; } -JNIEXPORT jlong JNICALL -Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) -{ - Ryml2Edn *obj = ys2edn_init(); - return (jlong)obj; -} - JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) { ys2evt_destroy((Ryml2Evt*)obj); } -JNIEXPORT void JNICALL -Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) -{ - ys2edn_destroy((Ryml2Edn*)obj); -} - JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, @@ -151,108 +136,6 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, } -JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, - jlong obj, jstring jfilename, - jbyteArray src, jint src_len, - jbyteArray dst, jint dst_len) -{ - TIMED_SECTION("jni_ys2edn_parse", (size_type)src_len); - jbyte* src_ = nullptr; - jbyte* dst_ = nullptr; - jboolean src_is_copy = false; - jboolean dst_is_copy = false; - const char *filename = nullptr; - { - TIMED_SECTION("jni_ys2edn_parse/get_jni_data", (size_type)src_len); - { - TIMED_SECTION("jni_ys2edn_parse/GetByteArray(src)"); - src_ = env->GetByteArrayElements(src, &src_is_copy); - } - { - TIMED_SECTION("jni_ys2edn_parse/GetByteArray(dst)"); - dst_ = env->GetByteArrayElements(dst, &dst_is_copy); - } - { - TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); - filename = env->GetStringUTFChars(jfilename, 0); - } - } - int rc = 0; - try - { - rc = ys2edn_parse((Ryml2Edn*)obj, filename, - (char*)src_, src_len, - (char*)dst_, dst_len); - } - catch (YsParseError const& exc) - { - throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); - } - { - TIMED_SECTION("jni_ys2edn_parse/release"); - env->ReleaseByteArrayElements(src, src_, 0); - env->ReleaseByteArrayElements(dst, dst_, 0); - env->ReleaseStringUTFChars(jfilename, filename); - } - return rc; -} - - -JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2edn_1parse_1buf(JNIEnv *env, jobject, - jlong obj, jstring jfilename, - jobject src, jint src_len, - jobject dst, jint dst_len) -{ - TIMED_SECTION("jni_ys2edn_parse", (size_type)src_len); - char* src_ = nullptr; - char* dst_ = nullptr; - const char *filename = nullptr; - { - TIMED_SECTION("jni_ys2edn_parse/get_jni_data", (size_type)src_len); - src_ = (char*)env->GetDirectBufferAddress(src); - dst_ = (char*)env->GetDirectBufferAddress(dst); - filename = env->GetStringUTFChars(jfilename, 0); - if(!src_) - throw_runtime_exception(env, "null pointer: src"); - if(!dst_) - throw_runtime_exception(env, "null pointer: dst"); - } - { - TIMED_SECTION("jni_ys2edn_parse/call_parse", (size_type)src_len); - try - { - return ys2edn_parse((Ryml2Edn*)obj, filename, src_, src_len, dst_, dst_len); - } - catch (YsParseError const& exc) - { - throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); - } - catch (std::exception const& exc) - { - throw_runtime_exception(env, exc.what()); - } - } - return 0; // this is executed even if there is an exception -} - - -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, - jlong obj, - jobject dst, jint dst_len) -{ - char* dst_ = nullptr; - { - TIMED_SECTION("jni_ys2evt_retry_get/get_dst"); - dst_ = (char*)env->GetDirectBufferAddress(dst); - printf("edn: aqui 1 %p\n", dst_); - } - int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); - return rc; -} - - //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index 74c7a5eb4..d3124b552 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -15,14 +15,6 @@ extern "C" { JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init (JNIEnv *, jobject); -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_init - * Signature: ()J - */ -JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init - (JNIEnv *, jobject); - /* * Class: org_rapidyaml_Rapidyaml * Method: ys2evt_destroy @@ -31,14 +23,6 @@ JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy (JNIEnv *, jobject, jlong); -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_destroy - * Signature: (J)V - */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy - (JNIEnv *, jobject, jlong); - /* * Class: org_rapidyaml_Rapidyaml * Method: ys2evt_parse @@ -55,22 +39,6 @@ JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_parse - * Signature: (JLjava/lang/String;[BI[BI)I - */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse - (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint); - -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_parse_buf - * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/ByteBuffer;I)I - */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse_1buf - (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); - #ifdef __cplusplus } #endif diff --git a/rapidyaml/native/ysparse_edn.cpp b/rapidyaml/native/ysparse_edn.cpp deleted file mode 100644 index 586c3a79e..000000000 --- a/rapidyaml/native/ysparse_edn.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include "ysparse_edn.hpp" - -using namespace ryml; - -#if defined(__cplusplus) -extern "C" { -#endif -// see -// https://stackoverflow.com/questions/230689/best-way-to-throw-exceptions-in-jni-code -// https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method - -namespace { -C4_NORETURN void ys2edn_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) -{ - YsParseError exc; - exc.location = location; - exc.msg.assign(msg, msg_len); - throw exc; -} -} // anon namespace - -RYML_EXPORT Ryml2Edn *ys2edn_init() -{ - TIMED_SECTION("ys2edn_init"); - Callbacks cb = {}; - cb.m_error = &ys2edn_parse_error; - set_callbacks(cb); - Ryml2Edn *ryml2edn = _RYML_CB_ALLOC(get_callbacks(), Ryml2Edn, 1); - _RYML_CB_CHECK(get_callbacks(), ryml2edn != nullptr); - new ((void*)ryml2edn) Ryml2Edn(); - return ryml2edn; -} - -RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn) -{ - TIMED_SECTION("ys2edn_destroy"); - ryml2edn->~Ryml2Edn(); - _RYML_CB_FREE(get_callbacks(), ryml2edn, Ryml2Edn, 1); -} - -RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, - const char *filename, - char *ys, size_type ys_size, - char *edn, size_type edn_size) -{ - TIMED_SECTION("ys2edn_parse", ys_size); - csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; - substr ys_(ys, (size_t)ys_size); - { - TIMED_SECTION("reset + reserve"); - ryml2edn->reset(); - ryml2edn->m_handler.reserve(ys_size > edn_size ? 3 * ys_size : edn_size, 256u); - } - { - TIMED_SECTION("parse_in_place", ys_size); - ryml2edn->m_parser.parse_in_place_ev(filename_, ys_); - } - return ys2edn_retry_get(ryml2edn, edn, edn_size); -} - -RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, - char *edn, size_type edn_size) -{ - TIMED_SECTION("ys2edn_retry_get"); - csubstr result = to_csubstr(ryml2edn->m_sink.result); - size_type required_size = 1 + (size_type)result.len; - if(required_size <= edn_size) - { - memcpy(edn, result.str, (size_t)result.len); - edn[result.len] = '\0'; - } - else if(edn_size > 0) - { - edn[0] = '\0'; - } - return required_size; -} - -#if defined(__cplusplus) -} -#endif diff --git a/rapidyaml/native/ysparse_edn.hpp b/rapidyaml/native/ysparse_edn.hpp deleted file mode 100644 index 3dfe73606..000000000 --- a/rapidyaml/native/ysparse_edn.hpp +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once -#ifndef YSPARSE_EDN_HPP_ -#define YSPARSE_EDN_HPP_ - -#include "ysparse_edn_handler.hpp" -#include "ysparse_common.hpp" - -#if defined(__cplusplus) -extern "C" { -#endif - -struct RYML_EXPORT Ryml2Edn -{ - ys::EventHandlerEdn::EventSink m_sink; - ys::EventHandlerEdn m_handler; - c4::yml::ParseEngine m_parser; - Ryml2Edn() - : m_sink() - , m_handler(&m_sink) - , m_parser(&m_handler) - { - } - void reset() - { - m_sink.reset(); - m_handler.reset(); - } -}; - - -//----------------------------------------------------------------------------- - -/** Initialize the resources */ -RYML_EXPORT Ryml2Edn *ys2edn_init(); - -/** Destroy the resources */ -RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn); - -/** Parse YAML, and return corresponding EDN. Return the number of - * characters needed for edn. The caller must check if the returned - * size is larger than edn_size. If it is, call ys_retry_get() can be - * called afterwards to extract the EDN. */ -RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, - const char *filename, - char *ys, size_type ys_size, - char *edn, size_type edn_size); - -/** Get the edn from the previous call to ys2edn_parse(). */ -RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, - char *edn, size_type edn_size); - -#if defined(__cplusplus) -} -#endif - -#endif /* YSPARSE_EDN_HPP_ */ diff --git a/rapidyaml/native/ysparse_edn_handler.cpp b/rapidyaml/native/ysparse_edn_handler.cpp deleted file mode 100644 index ab295234e..000000000 --- a/rapidyaml/native/ysparse_edn_handler.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include "ysparse_edn_handler.hpp" -#include -#include - - -// instantiate the template -namespace c4 { -namespace yml { -template class ParseEngine; -} // namespace yml -} // namespace c4 - - -namespace ys { -void EventHandlerEdn::EventSink::append_escaped(csubstr val) -{ - #define _c4flush_use_instead(repl, skip) \ - do { \ - this->append(val.range(prev, i)); \ - this->append(repl); \ - prev = i + skip; \ - } \ - while(0) - uint8_t const* C4_RESTRICT s = reinterpret_cast(val.str); - size_t prev = 0; - for(size_t i = 0; i < val.len; ++i) - { - switch(s[i]) - { - case UINT8_C(0x0a): // \n - _c4flush_use_instead("\\n", 1); break; - case UINT8_C(0x5c): // '\\' - _c4flush_use_instead("\\\\", 1); break; - case UINT8_C(0x22): // \" - _c4flush_use_instead("\\\"", 1); break; - case UINT8_C(0x09): // \t - _c4flush_use_instead("\\t", 1); break; - case UINT8_C(0x0d): // \r - _c4flush_use_instead("\\r", 1); break; - case UINT8_C(0x00): // \0 - _c4flush_use_instead("\\0", 1); break; - case UINT8_C(0x0c): // \f (form feed) - _c4flush_use_instead("\\f", 1); break; - case UINT8_C(0x08): // \b (backspace) - _c4flush_use_instead("\\b", 1); break; - case UINT8_C(0x07): // \a (bell) - _c4flush_use_instead("\\a", 1); break; - case UINT8_C(0x0b): // \v (vertical tab) - _c4flush_use_instead("\\v", 1); break; - case UINT8_C(0x1b): // \e (escape) - _c4flush_use_instead("\\e", 1); break; - case UINT8_C(0xc2): - if(i+1 < val.len) - { - const uint8_t np1 = s[i+1]; - if(np1 == UINT8_C(0xa0)) - _c4flush_use_instead("\\_", 2); - else if(np1 == UINT8_C(0x85)) - _c4flush_use_instead("\\N", 2); - } - break; - case UINT8_C(0xe2): - if(i+2 < val.len) - { - if(s[i+1] == UINT8_C(0x80)) - { - if(s[i+2] == UINT8_C(0xa8)) - _c4flush_use_instead("\\L", 3); - else if(s[i+2] == UINT8_C(0xa9)) - _c4flush_use_instead("\\P", 3); - } - } - break; - } - } - // flush the rest - this->append(val.sub(prev)); - #undef _c4flush_use_instead -} -} // namespace ys diff --git a/rapidyaml/native/ysparse_edn_handler.hpp b/rapidyaml/native/ysparse_edn_handler.hpp deleted file mode 100644 index 89c36813f..000000000 --- a/rapidyaml/native/ysparse_edn_handler.hpp +++ /dev/null @@ -1,696 +0,0 @@ -#ifndef _YSPARSE_EDN_HANDLER_HPP_ -#define _YSPARSE_EDN_HANDLER_HPP_ - -#include -#include -#include -#include -#include -#include - -#include - -C4_SUPPRESS_WARNING_GCC_CLANG_PUSH -C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") -C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") - - -namespace ys { - -using c4::csubstr; -using c4::substr; -using c4::to_substr; -using c4::to_csubstr; -using c4::yml::id_type; -using c4::yml::NodeType_e; -using c4::yml::type_bits; -#ifdef RYML_DBG -using c4::_dbg_printf; -#endif - -struct EventHandlerEdnState : public c4::yml::ParserState -{ - c4::yml::NodeData ev_data; -}; - - -struct EventHandlerEdn : public c4::yml::EventHandlerStack -{ - - /** @name types - * @{ */ - - // our internal state must inherit from parser state - using state = EventHandlerEdnState; - - struct EventSink - { - std::string result; - void reset() noexcept { result.clear(); } - void append(csubstr s) noexcept { result.append(s.str, s.len); } - void append(char c) noexcept { result += c; } - void insert(csubstr s, size_t pos) noexcept { result.insert(pos, s.str, s.len); } - void insert(char c, size_t pos) noexcept { result.insert(pos, 1, c); } - csubstr get() const { return csubstr(&result[0], result.size()); } - substr get() { return substr(&result[0], result.size()); } - size_t find_last(csubstr s) const { return result.rfind(s.str, std::string::npos, s.len); } - void append_escaped(csubstr val); - }; - - /** @} */ - -public: - - /** @cond dev */ - EventSink *C4_RESTRICT m_sink; - std::vector m_val_buffers; - char m_key_tag_buf[256]; - char m_val_tag_buf[256]; - std::string m_arena; - bool m_first_doc; - - // undefined at the end - #define _enable_(bits) _enable__() - #define _disable_(bits) _disable__() - #define _has_any_(bits) _has_any__() - /** @endcond */ - -public: - - /** @name construction and resetting - * @{ */ - - EventHandlerEdn(EventSink *sink, c4::yml::Callbacks const& cb) - : EventHandlerStack(cb), m_sink(sink), m_val_buffers(), m_first_doc() - { - reset(); - } - EventHandlerEdn(EventSink *sink) - : EventHandlerEdn(sink, c4::yml::get_callbacks()) - { - } - - void reset() - { - _stack_reset_root(); - m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; - m_val_buffers.resize((size_t)m_stack.size()); - m_arena.clear(); - m_first_doc = true; - } - - void reserve(int edn_size, int arena_size) - { - if(m_val_buffers.empty()) - m_val_buffers.resize((size_t)m_stack.size()); - if(m_sink) - m_sink->result.reserve(edn_size); - for(size_t i = 0; i < m_val_buffers.size(); ++i) - { - int sz = edn_size / (int(1) << (uint32_t)i); - sz = sz >= 128 ? sz : 128; - m_val_buffers[i].result.reserve((size_t)sz); - } - m_arena.reserve(arena_size); - } - - /** @} */ - -public: - - /** @name parse events - * @{ */ - - void start_parse(const char* filename, c4::yml::detail::pfn_relocate_arena relocate_arena, void *relocate_arena_data) - { - this->_stack_start_parse(filename, relocate_arena, relocate_arena_data); - } - - void finish_parse() - { - this->_stack_finish_parse(); - } - - void cancel_parse() - { - while(m_stack.size() > 1) - _pop(); - _buf_flush_(); - } - - /** @} */ - -public: - - /** @name YAML stream events */ - /** @{ */ - - void begin_stream() - { - _send_("(\n"); - } - - void end_stream() - { - _send_(")\n"); - _buf_flush_(); - } - - /** @} */ - -public: - - /** @name YAML document events */ - /** @{ */ - - /** implicit doc start (without ---) */ - void begin_doc() - { - _c4dbgp("begin_doc"); - if(_stack_should_push_on_begin_doc()) - { - _c4dbgp("push!"); - _push(); - _enable_(c4::yml::DOC); - } - m_first_doc = false; - } - /** implicit doc end (without ...) */ - void end_doc() - { - _c4dbgp("end_doc"); - _send_("{:+ \"-DOC\"}\n"); - if(_stack_should_pop_on_end_doc()) - { - _c4dbgp("pop!"); - _pop(); - } - } - - /** explicit doc start, with --- */ - void begin_doc_expl() - { - _c4dbgp("begin_doc_expl"); - if(_stack_should_push_on_begin_doc()) - { - _c4dbgp("push!"); - _push(); - } - if (m_first_doc) - m_first_doc = false; - else - _send_("{:+ \"+DOC\"}\n"); - _enable_(c4::yml::DOC); - } - /** explicit doc end, with ... */ - void end_doc_expl() - { - _c4dbgp("end_doc_expl"); - _send_("{:+ \"-DOC\"}\n"); - if(_stack_should_pop_on_end_doc()) - { - _c4dbgp("pop!"); - _pop(); - } - } - - /** @} */ - -public: - - /** @name YAML map functions */ - /** @{ */ - - void begin_map_key_flow() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - void begin_map_key_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - void begin_map_val_flow() - { - _send_("{:+ \"+MAP\""); - _send_val_props_(); - _send_(", :flow true}\n"); - _mark_parent_with_children_(); - _enable_(c4::yml::MAP|c4::yml::FLOW_SL); - _push(); - } - void begin_map_val_block() - { - _send_("{:+ \"+MAP\""); - _send_val_props_(); - _send_("}\n"); - _mark_parent_with_children_(); - _enable_(c4::yml::MAP|c4::yml::BLOCK); - _push(); - } - - void end_map() - { - _pop(); - _send_("{:+ \"-MAP\"}\n"); - } - - /** @} */ - -public: - - /** @name YAML seq events */ - /** @{ */ - - void begin_seq_key_flow() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - void begin_seq_key_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - void begin_seq_val_flow() - { - _send_("{:+ \"+SEQ\""); - _send_val_props_(); - _send_(", :flow true}\n"); - _mark_parent_with_children_(); - _enable_(c4::yml::SEQ|c4::yml::FLOW_SL); - _push(); - } - void begin_seq_val_block() - { - _send_("{:+ \"+SEQ\""); - _send_val_props_(); - _send_("}\n"); - _mark_parent_with_children_(); - _enable_(c4::yml::SEQ|c4::yml::BLOCK); - _push(); - } - - void end_seq() - { - _pop(); - _send_("{:+ \"-SEQ\"}\n"); - } - - /** @} */ - -public: - - /** @name YAML structure events */ - /** @{ */ - - void add_sibling() - { - _RYML_CB_ASSERT(m_stack.m_callbacks, m_parent); - _buf_flush_to_(m_curr->level, m_parent->level); - m_curr->ev_data = {}; - } - - /** set the previous val as the first key of a new map, with flow style. - * - * See the documentation for @ref doc_event_handlers, which has - * important notes about this event. - */ - void actually_val_is_first_key_of_new_map_flow() - { - // ensure we have a temporary buffer to save the current val - const id_type tmp = m_curr->level + id_type(2); - _buf_ensure_(tmp + id_type(2)); - // save the current val to the temporary buffer - _buf_flush_to_(m_curr->level, tmp); - // create the map. - // this will push a new level, and tmp is one further - begin_map_val_flow(); - _RYML_CB_ASSERT(m_stack.m_callbacks, tmp != m_curr->level); - // now move the saved val as the first key - _buf_flush_to_(tmp, m_curr->level); - } - - void actually_val_is_first_key_of_new_map_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - /** @} */ - -public: - - /** @name YAML scalar events */ - /** @{ */ - - - C4_ALWAYS_INLINE void set_key_scalar_plain_empty() - { - _send_key_scalar_({}, ':'); - _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN|c4::yml::KEYNIL); - } - C4_ALWAYS_INLINE void set_val_scalar_plain_empty() - { - _send_val_scalar_({}, ':'); - _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN|c4::yml::VALNIL); - } - - - C4_ALWAYS_INLINE void set_key_scalar_plain(csubstr scalar) - { - _send_key_scalar_(scalar, '='); - _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN); - } - C4_ALWAYS_INLINE void set_val_scalar_plain(csubstr scalar) - { - _send_val_scalar_(scalar, '='); - _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN); - } - - - C4_ALWAYS_INLINE void set_key_scalar_dquoted(csubstr scalar) - { - _send_key_scalar_(scalar, '$'); - _enable_(c4::yml::KEY|c4::yml::KEY_DQUO); - } - C4_ALWAYS_INLINE void set_val_scalar_dquoted(csubstr scalar) - { - _send_val_scalar_(scalar, '$'); - _enable_(c4::yml::VAL|c4::yml::VAL_DQUO); - } - - - C4_ALWAYS_INLINE void set_key_scalar_squoted(csubstr scalar) - { - _send_key_scalar_(scalar, '\''); - _enable_(c4::yml::KEY|c4::yml::KEY_SQUO); - } - C4_ALWAYS_INLINE void set_val_scalar_squoted(csubstr scalar) - { - _send_val_scalar_(scalar, '\''); - _enable_(c4::yml::VAL|c4::yml::VAL_SQUO); - } - - - C4_ALWAYS_INLINE void set_key_scalar_literal(csubstr scalar) - { - _send_key_scalar_(scalar, '|'); - _enable_(c4::yml::KEY|c4::yml::KEY_LITERAL); - } - C4_ALWAYS_INLINE void set_val_scalar_literal(csubstr scalar) - { - _send_val_scalar_(scalar, '|'); - _enable_(c4::yml::VAL|c4::yml::VAL_LITERAL); - } - - - C4_ALWAYS_INLINE void set_key_scalar_folded(csubstr scalar) - { - _send_key_scalar_(scalar, '>'); - _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); - } - C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) - { - _send_val_scalar_(scalar, '>'); - _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); - } - - - C4_ALWAYS_INLINE void mark_key_scalar_unfiltered() - { - _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); - } - C4_ALWAYS_INLINE void mark_val_scalar_unfiltered() - { - _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); - } - - /** @} */ - -public: - - /** @name YAML anchor/reference events */ - /** @{ */ - - void set_key_anchor(csubstr anchor) - { - _enable_(c4::yml::KEYANCH); - m_curr->ev_data.m_key.anchor = anchor; - } - void set_val_anchor(csubstr anchor) - { - _enable_(c4::yml::VALANCH); - m_curr->ev_data.m_val.anchor = anchor; - } - - void set_key_ref(csubstr ref) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); - _enable_(c4::yml::KEY|c4::yml::KEYREF); - _send_("{:+ \"=ALI\" :* \""); - _send_(ref.sub(1)); - _send_("\"}\n"); - } - void set_val_ref(csubstr ref) - { - _enable_(c4::yml::VAL|c4::yml::VALREF); - _send_("{:+ \"=ALI\" :* \""); - _send_(ref.sub(1)); - _send_("\"}\n"); - } - - /** @} */ - -public: - - /** @name YAML tag events */ - /** @{ */ - - void set_key_tag(csubstr tag) - { - _enable_(c4::yml::KEYTAG); - m_curr->ev_data.m_key.tag = _transform_directive(tag, m_key_tag_buf); - } - void set_val_tag(csubstr tag) - { - _enable_(c4::yml::VALTAG); - m_curr->ev_data.m_val.tag = _transform_directive(tag, m_val_tag_buf); - } - - /** @} */ - -public: - - /** @name YAML directive events */ - /** @{ */ - - void add_directive(csubstr directive) - { - _RYML_CB_ERR(m_stack.m_callbacks, "tag directives not supported"); - } - - /** @} */ - -public: - - /** @name YAML arena events */ - /** @{ */ - - substr alloc_arena(size_t len) - { - const size_t sz = m_arena.size(); - csubstr prev = to_csubstr(m_arena); - m_arena.resize(sz + len); - substr out = to_substr(m_arena).sub(sz); - substr curr = to_substr(m_arena); - if(curr.str != prev.str) - _stack_relocate_to_new_arena(prev, curr); - return out; - } - - substr alloc_arena(size_t len, substr *relocated) - { - csubstr prev = to_csubstr(m_arena); - if(!prev.is_super(*relocated)) - return alloc_arena(len); - substr out = alloc_arena(len); - substr curr = to_substr(m_arena); - if(curr.str != prev.str) - *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); - return out; - } - - /** @} */ - -public: - - /** push a new parent, add a child to the new parent, and set the - * child as the current node */ - void _push() - { - _stack_push(); - _buf_ensure_(m_stack.size() + id_type(1)); - _buf_().reset(); - m_curr->ev_data = {}; - } - - /** end the current scope */ - void _pop() - { - _buf_flush_to_(m_curr->level, m_parent->level); - _stack_pop(); - } - - template C4_ALWAYS_INLINE void _enable__() noexcept - { - m_curr->ev_data.m_type.type = static_cast(m_curr->ev_data.m_type.type | bits); - } - template C4_ALWAYS_INLINE void _disable__() noexcept - { - m_curr->ev_data.m_type.type = static_cast(m_curr->ev_data.m_type.type & (~bits)); - } - template C4_ALWAYS_INLINE bool _has_any__() const noexcept - { - return (m_curr->ev_data.m_type.type & bits) != 0; - } - - void _mark_parent_with_children_() - { - if(m_parent) - m_parent->has_children = true; - } - - EventSink& _buf_() noexcept - { - _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)m_curr->level < m_val_buffers.size()); - return m_val_buffers[(size_t)m_curr->level]; - } - - EventSink& _buf_(id_type level) noexcept - { - _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)level < m_val_buffers.size()); - return m_val_buffers[(size_t)level]; - } - - EventSink const& _buf_(id_type level) const noexcept - { - _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)level < m_val_buffers.size()); - return m_val_buffers[(size_t)level]; - } - - static void _buf_flush_to_(EventSink &C4_RESTRICT src, EventSink &C4_RESTRICT dst) noexcept - { - dst.append(src.get()); - src.reset(); - } - - void _buf_flush_to_(id_type level_src, id_type level_dst) noexcept - { - auto &src = _buf_(level_src); - auto &dst = _buf_(level_dst); - _buf_flush_to_(src, dst); - } - - void _buf_flush_() noexcept - { - _buf_flush_to_(_buf_(), *m_sink); - } - - void _buf_ensure_(id_type size_needed) noexcept - { - if((size_t)size_needed > m_val_buffers.size()) - m_val_buffers.resize((size_t)size_needed); - } - - C4_ALWAYS_INLINE void _send_(csubstr s) noexcept { _buf_().append(s); } - C4_ALWAYS_INLINE void _send_(char c) noexcept { _buf_().append(c); } - - void _send_key_scalar_(csubstr scalar, char scalar_type_code) - { - _send_("{:+ \"=VAL\""); - _send_key_props_(); - _send_(", :"); - _send_(scalar_type_code); - _send_(" \""); - _buf_().append_escaped(scalar); - _send_("\"}\n"); - } - void _send_val_scalar_(csubstr scalar, char scalar_type_code) - { - _send_("{:+ \"=VAL\""); - _send_val_props_(); - _send_(", :"); - _send_(scalar_type_code); - _send_(" \""); - _buf_().append_escaped(scalar); - _send_("\"}\n"); - } - - void _send_key_props_() - { - if(_has_any_(c4::yml::KEYANCH|c4::yml::KEYREF)) - { - _send_(", :& \""); - _send_(m_curr->ev_data.m_key.anchor); - _send_('\"'); - } - if(_has_any_(c4::yml::KEYTAG)) - { - _send_(", :! \""); - _send_tag_(m_curr->ev_data.m_key.tag); - _send_('\"'); - } - m_curr->ev_data.m_key = {}; - _disable_(c4::yml::KEYANCH|c4::yml::KEYREF|c4::yml::KEYTAG); - } - void _send_val_props_() - { - if(_has_any_(c4::yml::VALANCH|c4::yml::VALREF)) - { - _send_(", :& \""); - _send_(m_curr->ev_data.m_val.anchor); - _send_('\"'); - } - if(m_curr->ev_data.m_type.type & c4::yml::VALTAG) - { - _send_(", :! \""); - _send_tag_(m_curr->ev_data.m_val.tag); - _send_('\"'); - } - m_curr->ev_data.m_val = {}; - _disable_(c4::yml::VALANCH|c4::yml::VALREF|c4::yml::VALTAG); - } - void _send_tag_(csubstr tag) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, !tag.empty()); - if(tag.begins_with('!')) - tag = tag.sub(1); - _send_(tag); - } - - csubstr _transform_directive(csubstr tag, substr output) - { - if(tag.begins_with('!')) - { - if(c4::yml::is_custom_tag(tag)) - { - _RYML_CB_ERR_(m_stack.m_callbacks, "tag not found", m_curr->pos); - } - } - csubstr result = c4::yml::normalize_tag_long(tag, output); - if(result.begins_with('<') && result.ends_with('>')) - result = result.offs(1, 1); - _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); - _RYML_CB_CHECK(m_stack.m_callbacks, result.str); - return result; - } -#undef _enable_ -#undef _disable_ -#undef _has_any_ - -}; - -} // namespace ys - -C4_SUPPRESS_WARNING_GCC_POP - -#endif /* _YSPARSE_EDN_HANDLER_HPP_ */ diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 5688d1ebf..644668b26 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -1,6 +1,6 @@ -#include #include #include +#include using c4::csubstr; using c4::substr; @@ -43,12 +43,6 @@ c4::EnumSymbols const esyms() //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- -struct Ys2EdnScoped -{ - Ryml2Edn *ryml2edn; - Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} - ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } -}; struct Ys2EvtScoped { Ryml2Evt *ryml2evt; @@ -103,7 +97,6 @@ size_t expected_size(std::vector const& evt) struct TestCase { csubstr ys; - csubstr edn; std::vector evt; public: @@ -134,15 +127,9 @@ struct TestCase } \ } while(0) - TestResult test(Ryml2Edn *ryml2edn, Ryml2Evt *ryml2evt) const + TestResult test(Ryml2Evt *ryml2evt) const { TestResult tr = {}; - _runtest(test_edn_large_enough, ); - _runtest(test_edn_too_small, ); - _runtest(test_edn_nullptr, ); - _runtest(test_edn_large_enough_reuse, ryml2edn); - _runtest(test_edn_too_small_reuse, ryml2edn); - _runtest(test_edn_nullptr_reuse, ryml2edn); _runtest(test_evt_large_enough, ); _runtest(test_evt_too_small, ); _runtest(test_evt_nullptr, ); @@ -153,22 +140,6 @@ struct TestCase } // happy path: large-enough destination string - TestResult test_edn_large_enough_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output; - output.resize(2 * edn.len); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - &output[0], (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } TestResult test_evt_large_enough_reuse(Ryml2Evt *ryml2evt) const { if(evt.empty()) return {}; @@ -186,11 +157,6 @@ struct TestCase CHECK(testeq(output, input)); return tr; } - TestResult test_edn_large_enough() const - { - Ys2EdnScoped lib; - return test_edn_large_enough_reuse(lib.ryml2edn); - } TestResult test_evt_large_enough() const { Ys2EvtScoped lib; @@ -198,25 +164,6 @@ struct TestCase } // less-happy path: destination string not large enough - TestResult test_edn_too_small_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output = "?"; - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - CHECK(output != "?"); - output.resize(reqsize); - size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); - CHECK(getsize == reqsize); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } TestResult test_evt_too_small_reuse(Ryml2Evt *ryml2evt) const { TestResult tr = {}; @@ -240,11 +187,6 @@ struct TestCase CHECK(testeq(output, input)); return tr; } - TestResult test_edn_too_small() const - { - Ys2EdnScoped lib; - return test_edn_too_small_reuse(lib.ryml2edn); - } TestResult test_evt_too_small() const { Ys2EvtScoped lib; @@ -252,18 +194,6 @@ struct TestCase } // safe calling with nullptr - TestResult test_edn_nullptr_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - nullptr, 0); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - return tr; - } TestResult test_evt_nullptr_reuse(Ryml2Evt *ryml2evt) const { TestResult tr = {}; @@ -286,11 +216,6 @@ struct TestCase CHECK(testeq(output, input)); return tr; } - TestResult test_edn_nullptr() const - { - Ys2EdnScoped lib; - return test_edn_nullptr_reuse(lib.ryml2edn); - } TestResult test_evt_nullptr() const { Ys2EvtScoped lib; @@ -299,21 +224,6 @@ struct TestCase public: - bool testeq(csubstr actual) const - { - const bool status = (actual == edn); - if(!status) - printf("------\n" - "FAIL:\n" - "input:[%zu]~~~%.*s~~~\n" - "expected:[%zu]~~~%.*s~~~\n" - "actual:[%zu]~~~%.*s~~~\n", - ys.len, (int)ys.len, ys.str, - edn.len, (int)edn.len, edn.str, - actual.len, (int)actual.len, actual.str); - return status; - } - bool testeq(std::vector const& actual, csubstr parsed_source) const { int status = true; @@ -392,21 +302,13 @@ struct TestCase namespace { // make the declarations shorter -#define tc(ys, edn, ...) {ys, edn, std::vector(__VA_ARGS__)} +#define tc(ys, ...) {ys, std::vector(__VA_ARGS__)} #define e(...) EvtWithScalar{__VA_ARGS__} using namespace evt; inline constexpr bool needs_filter = true; const TestCase test_cases[] = { // case ------------------------------------------------- tc("a: 1", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "1"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -419,14 +321,6 @@ const TestCase test_cases[] = { }), // case ------------------------------------------------- tc("say: 2 + 2", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "say"} -{:+ "=VAL", := "2 + 2"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -439,14 +333,6 @@ const TestCase test_cases[] = { }), // case ------------------------------------------------- tc("𝄞: ✅", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "𝄞"} -{:+ "=VAL", := "✅"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -459,15 +345,6 @@ const TestCase test_cases[] = { }), // case ------------------------------------------------- tc("[a, b, c]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "=VAL", := "c"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -481,16 +358,6 @@ const TestCase test_cases[] = { }), // case ------------------------------ tc("[a: b]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "-MAP"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -519,42 +386,6 @@ foo: ! - &anchor-1 !tag-1 foobar --- another: doc -)", - R"(( -{:+ "+MAP", :! "yamlscript/v0"} -{:+ "=VAL", := "foo"} -{:+ "+SEQ", :! ""} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-MAP"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-SEQ"} -{:+ "=VAL", := "foo"} -{:+ "=VAL", :' "foo"} -{:+ "=VAL", :$ "foo"} -{:+ "=VAL", :| "foo\n"} -{:+ "=VAL", :> "foo\n"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "1"} -{:+ "=VAL", := "2"} -{:+ "=VAL", := "true"} -{:+ "=VAL", := "false"} -{:+ "=VAL", := "null"} -{:+ "-SEQ"} -{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} -{:+ "-SEQ"} -{:+ "-MAP"} -{:+ "-DOC"} -{:+ "+DOC"} -{:+ "+MAP"} -{:+ "=VAL", := "another"} -{:+ "=VAL", := "doc"} -{:+ "-MAP"} -{:+ "-DOC"} -) )", { e(BSTR), @@ -613,22 +444,6 @@ fold: > U V W -)", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "plain"} -{:+ "=VAL", := "well a b c"} -{:+ "=VAL", := "squo"} -{:+ "=VAL", :' "single'quote"} -{:+ "=VAL", := "dquo"} -{:+ "=VAL", :$ "x\t\ny"} -{:+ "=VAL", := "lit"} -{:+ "=VAL", :| "X\nY\nZ\n"} -{:+ "=VAL", := "fold"} -{:+ "=VAL", :> "U V W\n"} -{:+ "-MAP"} -{:+ "-DOC"} -) )", { e(BSTR), @@ -650,14 +465,6 @@ fold: > }), // case ------------------------------------------------- tc("- !!seq []", - R"(( -{:+ "+SEQ"} -{:+ "+SEQ", :! "tag:yaml.org,2002:seq", :flow true} -{:+ "-SEQ"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -676,20 +483,6 @@ fold: > Q: $(orig-prompt:trim) A ($api-model): $(answer:trim) -)_", - R"_(( -{:+ "+MAP"} -{:+ "=VAL", := "defn run(prompt session=nil)"} -{:+ "+MAP"} -{:+ "=VAL", := "when session"} -{:+ "+MAP"} -{:+ "=VAL", := "write session _ :append true"} -{:+ "=VAL", :| "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "-DOC"} -) )_", { e(BSTR), @@ -734,43 +527,6 @@ defn run(prompt session=nil): A ($api-model): $(answer:trim) -)_", - R"_(( -{:+ "+MAP"} -{:+ "=VAL", := "defn run(prompt session=nil)"} -{:+ "+MAP"} -{:+ "=VAL", := "session-text ="} -{:+ "+MAP"} -{:+ "=VAL", := "when session && session:fs-e"} -{:+ "=VAL", :: ""} -{:+ "-MAP"} -{:+ "=VAL", := "answer ="} -{:+ "+MAP"} -{:+ "=VAL", := "cond"} -{:+ "+MAP"} -{:+ "=VAL", := "api-model =~ /^dall-e/"} -{:+ "=VAL", := "openai-image(prompt).data.0.url"} -{:+ "=VAL", := "api-model.in?(anthropic-models)"} -{:+ "=VAL", := "anthropic(prompt):anthropic-message:format"} -{:+ "=VAL", := "api-model.in?(groq-models)"} -{:+ "=VAL", := "groq(prompt).choices.0.message.content:format"} -{:+ "=VAL", := "api-model.in?(openai-models)"} -{:+ "=VAL", := "openai-chat(prompt).choices.0.message.content:format"} -{:+ "=VAL", := "else"} -{:+ "=VAL", := "die()"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "=VAL", := "say"} -{:+ "=VAL", := "answer"} -{:+ "=VAL", := "when session"} -{:+ "+MAP"} -{:+ "=VAL", := "write session _ :append true"} -{:+ "=VAL", :| "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "-DOC"} -) )_", { e(BSTR), @@ -816,7 +572,6 @@ defn run(prompt session=nil): int main() { - Ys2EdnScoped ys2edn; Ys2EvtScoped ys2evt; TestResult total = {}; size_t failed_cases = {}; @@ -826,7 +581,7 @@ int main() printf("-----------------------------------------\n" "case %zu/%zu ...\n" "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); - const TestResult tr = test_cases[i].test(ys2edn.ryml2edn, ys2evt.ryml2evt); + const TestResult tr = test_cases[i].test(ys2evt.ryml2evt); total.add(tr); failed_cases += (!tr); printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index f77b519d7..a8803482b 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -14,32 +14,20 @@ public class Rapidyaml public static String RAPIDYAML_VERSION = "0.8.0"; private native long ys2evt_init(); - private native long ys2edn_init(); - private native void ys2evt_destroy(long ryml2evt); - private native void ys2edn_destroy(long ryml2edn); - private native int ys2evt_parse(long ryml2evt, String filename, byte[] ys, int ys_length, int[] evt, int evt_length); private native int ys2evt_parse_buf(long ryml2evt, String filename, ByteBuffer ys, int ys_length, IntBuffer evt, int evt_length); - private native int ys2edn_parse(long ryml2edn, String filename, - byte[] ys, int ys_length, - byte[] edn, int edn_length); - private native int ys2edn_parse_buf(long ryml2edn, String filename, - ByteBuffer ys, int ys_length, - ByteBuffer edn, int edn_length); - private final long ryml2edn; private final long ryml2evt; public Rapidyaml() { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); - this.ryml2edn = this.ys2edn_init(); this.ryml2evt = this.ys2evt_init(); // TODO: receive this argument as ctor parameter timingEnabled(System.getenv("YS_RYML_TIMER") != null); @@ -52,7 +40,6 @@ public Rapidyaml() protected void finalize() throws Throwable { try { - this.ys2edn_destroy(this.ryml2edn); this.ys2evt_destroy(this.ryml2evt); } finally { @@ -111,45 +98,6 @@ public static IntBuffer mkIntBuffer(int numInts) } - //------------------------ - // EDN - //------------------------ - - public int parseYsToEdn(byte[] src, byte[] edn) throws Exception - { - return parseYsToEdn("yamlscript", src, edn); - } - - public int parseYsToEdnBuf(ByteBuffer src, ByteBuffer edn) throws Exception - { - return parseYsToEdnBuf("yamlscript", src, edn); - } - - public int parseYsToEdn(String filename, byte[] src, byte[] edn) throws Exception - { - long t = timingStart("ys2edn"); - int ret = ys2edn_parse(this.ryml2edn, filename, src, src.length, edn, edn.length); - timingStop("ys2edn", t, src.length); - return ret; - } - - public int parseYsToEdnBuf(String filename, ByteBuffer src, ByteBuffer edn) throws Exception - { - if(!src.isDirect()) - throw new RuntimeException("src must be direct"); - if(!edn.isDirect()) - throw new RuntimeException("edn must be direct"); - long t = timingStart("ys2ednBuf"); - edn.position(edn.capacity()); - int reqsize = ys2edn_parse_buf(this.ryml2edn, filename, src, src.position(), edn, edn.capacity()); - if(reqsize <= edn.capacity()) { - edn.position(reqsize); - } - timingStop("ys2ednBuf", t, src.position()); - return reqsize; - } - - //------------------------ // TIME //------------------------ diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java similarity index 53% rename from rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java rename to rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java index cd86ae189..19b6fab7c 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java @@ -9,24 +9,24 @@ import java.nio.IntBuffer; // https://stackoverflow.com/questions/804466/how-do-i-create-executable-java-program -public class CmpEdnEvt +public class CmpEvt { public static void main(String[] args) throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); rapidyaml.timingEnabled(true); - compareEdnEvt(rapidyaml, "./yamllm.ys"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); + compareEvt(rapidyaml, "./yamllm.ys"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); } - public static void compareEdnEvt(Rapidyaml rapidyaml, String path) throws Exception + public static void compareEvt(Rapidyaml rapidyaml, String path) throws Exception { String ys_ = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); byte[] ys = ys_.getBytes(StandardCharsets.UTF_8); @@ -37,15 +37,7 @@ public static void compareEdnEvt(Rapidyaml rapidyaml, String path) throws Except System.out.printf("%s\n", path); System.out.printf(" ys.length=%d\n", ys.length); // - long t = timingStart("edn"); - String ednStr = callEdn(rapidyaml, ys, ysarr); - timingStop("edn", t, ys.length); - // - t = timingStart("ednBuf"); - ByteBuffer edn = callEdnBuf(rapidyaml, ys, ysbuf); - timingStop("ednBuf", t, ys.length); - // - t = timingStart("evt"); + long t = timingStart("evt"); int[] evtarr = callEvt(rapidyaml, ys, ysarr); timingStop("evt", t, ys.length); // @@ -54,42 +46,6 @@ public static void compareEdnEvt(Rapidyaml rapidyaml, String path) throws Except timingStop("evtBuf", t, ys.length); } - static String callEdn(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception - { - System.arraycopy(src, 0, srcbuf, 0, src.length); - byte[] edn = new byte[10000000]; - int reqsize = rapidyaml.parseYsToEdn(srcbuf, edn); - if(reqsize > edn.length) { - edn = new byte[reqsize]; - System.arraycopy(src, 0, srcbuf, 0, src.length); - int reqsize2 = rapidyaml.parseYsToEdn(srcbuf, edn); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - String ret = new String(edn, 0, reqsize-1, StandardCharsets.UTF_8); - return ret; - } - - static ByteBuffer callEdnBuf(Rapidyaml rapidyaml, byte[] src, ByteBuffer srcbuf) throws Exception - { - srcbuf.position(0); - srcbuf.put(src); - ByteBuffer edn = ByteBuffer.allocateDirect(10000000); - int reqsize = rapidyaml.parseYsToEdnBuf(srcbuf, edn); - if(reqsize > edn.capacity()) { - edn = ByteBuffer.allocateDirect(reqsize); - srcbuf.position(0); - srcbuf.put(src); - int reqsize2 = rapidyaml.parseYsToEdnBuf(srcbuf, edn); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - edn.position(reqsize); - return edn; - } - static int[] callEvt(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception { System.arraycopy(src, 0, srcbuf, 0, src.length); diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf index 97704e895..aa44b6579 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf @@ -1 +1 @@ -Main-class: cmp.CmpEdnEvt +Main-class: cmp.CmpEvt diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh index da951ab24..4f655cfe2 100755 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh @@ -11,10 +11,12 @@ make -C $rymldir test RAPIDYAML_TIMED=1 cd $thisdir -wget https://raw.githubusercontent.com/yaml/yamllm/refs/heads/main/bin/yamllm.ys +if [ ! -f yamllm.ys ] ; then + wget https://raw.githubusercontent.com/yaml/yamllm/refs/heads/main/bin/yamllm.ys +fi ls -lFhp jd=${jd:-/usr/lib/jvm/java-23-openjdk/bin} $jd/javac -d . ../*.java -$jd/javac -d . -cp . CmpEdnEvt.java -$jd/jar -cmf manifest.mf CmpEdnEvt.jar cmp org -$jd/java -jar -Djava.library.path=$nativedir CmpEdnEvt.jar +$jd/javac -d . -cp . CmpEvt.java +$jd/jar -cmf manifest.mf CmpEvt.jar cmp org +$jd/java -jar -Djava.library.path=$nativedir CmpEvt.jar diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index 174dde7aa..7a2af04d7 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -39,15 +39,6 @@ public static Test suite() public void testPlainMap() { String ys = "a: 1"; - testEdn_(ys, - "(\n" + - "{:+ \"+MAP\"}\n" + - "{:+ \"=VAL\", := \"a\"}\n" + - "{:+ \"=VAL\", := \"1\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -64,15 +55,6 @@ public void testPlainMap() public void testUtf8() { String ys = "𝄞: ✅"; - testEdn_(ys, - "(\n" + - "{:+ \"+MAP\"}\n" + - "{:+ \"=VAL\", := \"𝄞\"}\n" + - "{:+ \"=VAL\", := \"✅\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -89,14 +71,6 @@ public void testUtf8() public void testTaggedInt() { String ys = "- !!int 42"; - testEdn_(ys, - "(\n" + - "{:+ \"+SEQ\"}\n" + - "{:+ \"=VAL\", :! \"tag:yaml.org,2002:int\", := \"42\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -113,15 +87,6 @@ public void testTaggedInt() public void testTaggedSeq() { String ys = "- !!seq []"; - testEdn_(ys, - "(\n" + - "{:+ \"+SEQ\"}\n" + - "{:+ \"+SEQ\", :! \"tag:yaml.org,2002:seq\", :flow true}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -155,43 +120,6 @@ public void testLargeCase() "- &anchor-1 !tag-1 foobar\n" + "---\n" + "another: doc\n"; - testEdn_(ys, - "(\n" + - "{:+ \"+MAP\", :! \"yamlscript/v0\"}\n" + - "{:+ \"=VAL\", := \"foo\"}\n" + - "{:+ \"+SEQ\", :! \"\"}\n" + - "{:+ \"+MAP\", :flow true}\n" + - "{:+ \"=VAL\", := \"x\"}\n" + - "{:+ \"=VAL\", := \"y\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"+SEQ\", :flow true}\n" + - "{:+ \"=VAL\", := \"x\"}\n" + - "{:+ \"=VAL\", := \"y\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"=VAL\", := \"foo\"}\n" + - "{:+ \"=VAL\", :' \"foo\"}\n" + - "{:+ \"=VAL\", :$ \"foo\"}\n" + - "{:+ \"=VAL\", :| \"foo\\nliteral\\n\"}\n" + - "{:+ \"=VAL\", :> \"foo folded\\n\"}\n" + - "{:+ \"+SEQ\", :flow true}\n" + - "{:+ \"=VAL\", := \"1\"}\n" + - "{:+ \"=VAL\", := \"2\"}\n" + - "{:+ \"=VAL\", := \"true\"}\n" + - "{:+ \"=VAL\", := \"false\"}\n" + - "{:+ \"=VAL\", := \"null\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"=VAL\", :& \"anchor-1\", :! \"tag-1\", := \"foobar\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - "{:+ \"+DOC\"}\n" + - "{:+ \"+MAP\"}\n" + - "{:+ \"=VAL\", := \"another\"}\n" + - "{:+ \"=VAL\", := \"doc\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC|Evt.EXPL), @@ -254,22 +182,6 @@ public void testFilterCase() " U\n" + " V\n" + " W\n"; - testEdn_(ys, - "(\n" + - "{:+ \"+MAP\"}\n" + - "{:+ \"=VAL\", := \"plain\"}\n" + - "{:+ \"=VAL\", := \"well a b c\"}\n" + - "{:+ \"=VAL\", := \"squo\"}\n" + - "{:+ \"=VAL\", :' \"single'quote\"}\n" + - "{:+ \"=VAL\", := \"dquo\"}\n" + - "{:+ \"=VAL\", :$ \"x\\t\\ny\"}\n" + - "{:+ \"=VAL\", := \"lit\"}\n" + - "{:+ \"=VAL\", :| \"X\\nY\\nZ\\n\"}\n" + - "{:+ \"=VAL\", := \"fold\"}\n" + - "{:+ \"=VAL\", :> \"U V W\\n\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n"); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -295,9 +207,11 @@ public void testFailure() throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); String ys = ": : : :"; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + byte[] srcbuf = new byte[src.length]; boolean gotit = false; try { - callEdn(ys); + callEvt(src, srcbuf); } catch(YamlParseErrorException e) { gotit = true; @@ -313,46 +227,38 @@ public void testFailure() throws Exception catch(Exception e) { fail("wrong exception type"); } - //catch(Throwable e) { - // throw e; - // //fail("wrong exception type"); - //} assertTrue(gotit); } - - private void testEdn_(String ys, String expected) + public void testFailureBuf() throws Exception { - String actual; - try { - actual = callEdn(ys); - } - catch (Exception e) { - fail("parse error:\n" + e.getMessage()); - actual = ""; - } - try { - cmpEdn_(actual, expected); - } - catch (Exception e) { - System.err.printf("error: edn (no buf)"); - } - //------ + Rapidyaml rapidyaml = new Rapidyaml(); + String ys = ": : : :"; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); + bbuf.put(src); + boolean gotit = false; try { - actual = callEdnBuf(ys); + callEvtBuf(src, bbuf); } - catch (Exception e) { - fail("parse error:\n" + e.getMessage()); - actual = ""; + catch(YamlParseErrorException e) { + gotit = true; + assertEquals(2, e.offset); + assertEquals(1, e.line); + assertEquals(3, e.column); + assertTrue(e.getMessage() != null); + assertFalse(e.getMessage().isEmpty()); } - try { - cmpEdn_(actual, expected); + catch(RuntimeException e) { + fail("wrong exception type"); } - catch (Exception e) { - System.err.printf("error: ednbuf"); + catch(Exception e) { + fail("wrong exception type"); } + assertTrue(gotit); } + private void testEvt_(String ys, ExpectedEvent[] expected) { byte[] src = ys.getBytes(StandardCharsets.UTF_8); @@ -394,21 +300,6 @@ private void testEvt_(String ys, ExpectedEvent[] expected) } } - private void cmpEdn_(String actual, String expected) throws Exception - { - try { - assertEquals(expected, actual); - assertEquals(expected.length(), actual.length()); - } - catch (Exception e) { - System.err.println("expected:"); - System.err.println(expected); - System.err.println("actual"); - System.err.println(actual); - throw e; - } - } - boolean dbglog = true; private void cmpEvt_(String ys, byte[] src, int[] actual, ExpectedEvent[] expected) { @@ -495,59 +386,6 @@ public static int[] buf2arr(IntBuffer evt) return ret; } - static String callEdn(String orig) throws Exception - { - byte[] src = orig.getBytes(StandardCharsets.UTF_8); - byte[] srcbuf = new byte[src.length]; - return callEdn(src, srcbuf); - } - - static String callEdnBuf(String orig) throws Exception - { - byte[] src = orig.getBytes(StandardCharsets.UTF_8); - ByteBuffer srcbuf = ByteBuffer.allocateDirect(src.length); - ByteBuffer edn = callEdnBuf(src, srcbuf); - return buf2str(edn); - } - - static String callEdn(byte[] src, byte[] srcbuf) throws Exception - { - Rapidyaml rapidyaml = new Rapidyaml(); - System.arraycopy(src, 0, srcbuf, 0, src.length); - byte[] edn = new byte[10 * src.length]; - int reqsize = rapidyaml.parseYsToEdn(srcbuf, edn); - if(reqsize > edn.length) { - edn = new byte[reqsize]; - System.arraycopy(src, 0, srcbuf, 0, src.length); - int reqsize2 = rapidyaml.parseYsToEdn(srcbuf, edn); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - String ret = new String(edn, 0, reqsize-1, StandardCharsets.UTF_8); - return ret; - } - - static ByteBuffer callEdnBuf(byte[] src, ByteBuffer srcbuf) throws Exception - { - Rapidyaml rapidyaml = new Rapidyaml(); - srcbuf.position(0); - srcbuf.put(src); - ByteBuffer edn = ByteBuffer.allocateDirect(10000); - int reqsize = rapidyaml.parseYsToEdnBuf(srcbuf, edn); - if(reqsize > edn.capacity()) { - edn = ByteBuffer.allocateDirect(reqsize); - srcbuf.position(0); - srcbuf.put(src); - int reqsize2 = rapidyaml.parseYsToEdnBuf(srcbuf, edn); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - edn.position(reqsize); - return edn; - } - static int[] callEvt(byte[] src, byte[] srcbuf) throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); From 07e6489bed7275214cb858a86973a43e2e5eb1cb Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 24 Feb 2025 00:14:41 +0000 Subject: [PATCH 18/21] rapidyaml: Add run-time timing toggle --- .github/workflows/rapidyaml.yml | 60 ++++++++++--------- rapidyaml/native/CMakeLists.txt | 5 ++ rapidyaml/native/Makefile | 10 +++- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 18 ++++++ rapidyaml/native/org_rapidyaml_Rapidyaml.h | 8 +++ rapidyaml/native/ysparse_common.hpp | 42 +++++++++---- rapidyaml/native/ysparse_test.cpp | 8 ++- .../main/java/org/rapidyaml/Rapidyaml.java | 5 +- 8 files changed, 111 insertions(+), 45 deletions(-) diff --git a/.github/workflows/rapidyaml.yml b/.github/workflows/rapidyaml.yml index 6348fa3cd..8cfa4103a 100644 --- a/.github/workflows/rapidyaml.yml +++ b/.github/workflows/rapidyaml.yml @@ -65,40 +65,34 @@ jobs: - name: get rapidyaml run: | make -C rapidyaml/native rapidyaml - - name: c++ tests, no timing --------------------------------------------------- + - name: run c++ tests, static ----------------------------- run: echo - - name: cfg c++ tests, no timing + - name: cfg c++, static run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - RAPIDYAML_TIMED=0 \ - make -C rapidyaml/native cfg - - name: build c++ tests, no timing + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-static + - name: build c++ tests, static run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - RAPIDYAML_TIMED=0 \ - make -C rapidyaml/native build - - name: run c++ tests, no timing + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-static + - name: run c++ tests, static run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - RAPIDYAML_TIMED=0 \ - make -C rapidyaml/native test - - name: c++ tests, with timing --------------------------------------------------- + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static + - name: run c++ tests, static with timing + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static-timing + - name: run c++ tests, shared ----------------------------- run: echo - - name: cfg c++ tests, with timing + - name: cfg c++, shared run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - RAPIDYAML_TIMED=1 \ - make -C rapidyaml/native cfg - - name: build c++ tests, with timing + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-shared + - name: build c++ tests, shared run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - RAPIDYAML_TIMED=1 \ - make -C rapidyaml/native build - - name: run c++ tests, with timing + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-shared + - name: run c++ tests, shared run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - RAPIDYAML_TIMED=1 \ - make -C rapidyaml/native test + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared + - name: run c++ tests, shared with timing + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared-timing # run the java tests, also in Debug to test with assertions java: @@ -115,12 +109,20 @@ jobs: - name: checkout (action) uses: actions/checkout@v4 with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: build lib + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml build + - name: build jar + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml jar + - name: install + run: | + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml install - name: run java tests run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - make -C rapidyaml test + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml test - # run core and ys tests, also in Debug to test with assertions + # run ys tests, also in Debug to test with assertions ys: name: ys/${{matrix.ysparser}}/${{matrix.bt}} runs-on: ubuntu-24.04 diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index aae90d5b5..968d7cb9d 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -64,3 +64,8 @@ add_custom_target(${libname}-test-run COMMAND $ COMMENT "running C++ tests" ) +add_custom_target(${libname}-test-run-timing + DEPENDS ${libname}-test + COMMAND $ --timing + COMMENT "running C++ tests, with timing" +) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index ec0037293..e656be03e 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -63,17 +63,21 @@ cfg-static: rapidyaml $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF test-static: build-static $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run +test-static-timing: build-static + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run-timing $(RAPIDYAML_LIB): cfg-static $(RAPIDYAML_DEPS) - $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --target rapidyaml --parallel --verbose + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --parallel --target rapidyaml cp -fv $(BDIR)-static/*.a $@ build-shared: $(RAPIDYAML_SO) cfg-shared: rapidyaml $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON test-shared: build-shared - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --target rapidyaml-test-run --verbose + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run +test-shared-timing: build-shared + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run-timing $(RAPIDYAML_SO): cfg-shared $(RAPIDYAML_DEPS) - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --target rapidyaml --parallel --verbose + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --parallel --target rapidyaml cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 02da00d71..0a337082f 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -5,10 +5,17 @@ extern "C" { #endif + static C4_NO_INLINE void throw_runtime_exception(JNIEnv * env, const char* msg); static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg); +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set(JNIEnv *, jobject, jboolean yes) +{ + ysparse_timing_set(yes); +} + JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) { @@ -16,6 +23,7 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) return (jlong)obj; } + JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) { @@ -140,6 +148,16 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- +static bool s_timing_enabled = false; +RYML_EXPORT bool ysparse_timing_get() +{ + return s_timing_enabled; +} +RYML_EXPORT void ysparse_timing_set(bool yes) +{ + s_timing_enabled = yes; +} + static C4_NO_INLINE void throw_java_exception(JNIEnv * env, const char* type, const char* msg) { jclass clazz = env->FindClass(type); diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index d3124b552..54c2aad02 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -7,6 +7,14 @@ #ifdef __cplusplus extern "C" { #endif +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_timing_set + * Signature: (Z)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set + (JNIEnv *, jobject, jboolean); + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2evt_init diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp index ce3c3d1b4..15bb2caf6 100644 --- a/rapidyaml/native/ysparse_common.hpp +++ b/rapidyaml/native/ysparse_common.hpp @@ -3,6 +3,7 @@ #define YSPARSE_COMMON_HPP_ #include +#include namespace ryml { using namespace c4; @@ -18,6 +19,19 @@ struct YsParseError : public std::exception const char* what() const noexcept override { return msg.c_str(); } }; + +//----------------------------------------------------------------------------- +// timing + +#ifdef __cplusplus +extern "C" { +#endif +RYML_EXPORT bool ysparse_timing_get(); +RYML_EXPORT void ysparse_timing_set(bool yes); +#ifdef __cplusplus +} +#endif + #ifndef YSPARSE_TIMED #define TIMED_SECTION(...) #error @@ -28,22 +42,28 @@ struct YsParseError : public std::exception struct timed_section { using myclock = std::chrono::steady_clock; - ryml::csubstr name; + const char* name; size_type len; myclock::time_point start; - timed_section(ryml::csubstr n, size_type len_=0) - : name(n) - , len(len_) - , start(myclock::now()) + C4_NO_INLINE timed_section(const char* n, size_type len_=0) { + if(ysparse_timing_get()) + { + name = n; + len = len_; + start = myclock::now(); + } } - ~timed_section() + C4_NO_INLINE ~timed_section() { - const std::chrono::duration t = myclock::now() - start; - fprintf(stderr, "%.6fms: %.*s", t.count(), (int)name.len, name.str); - if(len) - fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); - fprintf(stderr, "\n"); + if(ysparse_timing_get()) + { + const std::chrono::duration t = myclock::now() - start; + fprintf(stderr, "%.6fms: %s", t.count(), name); + if(len) + fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); + fprintf(stderr, "\n"); + } } }; #endif // YSPARSE_TIMED diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 644668b26..77f31c5e2 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -570,8 +570,14 @@ defn run(prompt session=nil): }; } // namespace -int main() +int main(int argc, const char *argv[]) { + for(int i = 1; i < argc; ++i) + { + csubstr arg = ryml::to_csubstr(argv[i]); + if(arg == "--timing" || arg == "-t") + ysparse_timing_set(true); + } Ys2EvtScoped ys2evt; TestResult total = {}; size_t failed_cases = {}; diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index a8803482b..4c7bc2fd4 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -13,6 +13,8 @@ public class Rapidyaml { public static String RAPIDYAML_VERSION = "0.8.0"; + private native void ysparse_timing_set(boolean yes); + // TODO: rename these to ysparse_init() etc private native long ys2evt_init(); private native void ys2evt_destroy(long ryml2evt); private native int ys2evt_parse(long ryml2evt, String filename, @@ -64,7 +66,7 @@ public int parseYsToEvtBuf(ByteBuffer src, IntBuffer evt) throws Exception public int parseYsToEvt(String filename, byte[] src, int[] evts) throws Exception { - long t = timingStart("ys2evtBuf"); + long t = timingStart("ys2evt"); int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); timingStop("ys2evt", t, src.length); return required_size; @@ -107,6 +109,7 @@ public static IntBuffer mkIntBuffer(int numInts) public void timingEnabled(boolean yes) { showTiming = yes; + ysparse_timing_set(yes); } private long timingStart(String name) From 0b9f8cb4daa30dcdbea34f2d374672f316812286 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 24 Feb 2025 00:59:52 +0000 Subject: [PATCH 19/21] rapidyaml: Rename ryml2evt/ys2evt to ysparse --- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 56 +++++++++---------- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 16 +++--- rapidyaml/native/ysparse_common.hpp | 3 +- rapidyaml/native/ysparse_evt.cpp | 42 +++++++------- rapidyaml/native/ysparse_evt.hpp | 16 +++--- rapidyaml/native/ysparse_test.cpp | 24 ++++---- .../main/java/org/rapidyaml/Rapidyaml.java | 34 +++++------ 7 files changed, 93 insertions(+), 98 deletions(-) diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 0a337082f..050c75aad 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -17,62 +17,56 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set(JNIEnv *, jobject, jboolean ye } JNIEXPORT jlong JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) +Java_org_rapidyaml_Rapidyaml_ysparse_1init(JNIEnv *env, jobject) { - Ryml2Evt *obj = ys2evt_init(); + ysparse *obj = ysparse_init(); return (jlong)obj; } JNIEXPORT void JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) +Java_org_rapidyaml_Rapidyaml_ysparse_1destroy(JNIEnv *, jobject, jlong obj) { - ys2evt_destroy((Ryml2Evt*)obj); + ysparse_destroy((ysparse*)obj); } JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, +Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject, jlong obj, jstring jfilename, jbyteArray src, jint src_len, jintArray dst, jint dst_len) { - TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + TIMED_SECTION("jni:ysparse", (size_type)src_len); jbyte* src_ = nullptr; int* dst_ = nullptr; const char *filename = nullptr; jboolean dst_is_copy = false; jboolean src_is_copy = false; { - TIMED_SECTION("jni_ys2evt_parse/get_jni", (size_type)src_len); + TIMED_SECTION("jni:ysparse/get_jni", (size_type)src_len); + // this is __S__L__O__W__ + // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time + // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string { - TIMED_SECTION("jni_ys2evt_parse/GetByteArray(src)"); + TIMED_SECTION("jni:ysparse/GetByteArray(src)"); src_ = env->GetByteArrayElements(src, &src_is_copy); } { - // TODO this is __S__L__O__W__ - // - // the problem is with GetIntArrayElements(). we should - // use GetDirectBufferAddress(), but that requires a ByteBuffer->jobject - // instead of a int[]->jintArray - // - // see: - // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time - // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string - TIMED_SECTION("jni_ys2evt_parse/GetIntArray(dst)"); + TIMED_SECTION("jni:ysparse/GetIntArray(dst)"); dst_ = env->GetIntArrayElements(dst, &dst_is_copy); } { - TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); + TIMED_SECTION("jni:ysparse/GetStringUTFChars()"); filename = env->GetStringUTFChars(jfilename, 0); } } int rc = 0; { - TIMED_SECTION("jni_ys2evt_parse/call_parse", (size_type)src_len); + TIMED_SECTION("jni:ysparse/parse", (size_type)src_len); try { - rc = ys2evt_parse((Ryml2Evt*)obj, filename, + rc = ysparse_parse((ysparse*)obj, filename, (char*)src_, src_len, dst_, dst_len); } @@ -86,18 +80,18 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, } } { - TIMED_SECTION("jni_ys2evt_parse/release"); + TIMED_SECTION("jni:ysparse/release"); + // __S__L__O__W__ { - TIMED_SECTION("jni_ys2evt_parse/ReleaseByteArray(src)"); + TIMED_SECTION("jni:ysparse/ReleaseByteArray(src)"); env->ReleaseByteArrayElements(src, src_, 0); } { - // TODO __S__L__O__W__ - TIMED_SECTION("jni_ys2evt_parse/ReleaseIntArray(dst)"); + TIMED_SECTION("jni:ysparse/ReleaseIntArray(dst)"); env->ReleaseIntArrayElements(dst, dst_, 0); } { - TIMED_SECTION("jni_ys2evt_parse/ReleaseStringUTFChars()"); + TIMED_SECTION("jni:ysparse/ReleaseStringUTFChars()"); env->ReleaseStringUTFChars(jfilename, filename); } } @@ -106,17 +100,17 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, +Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf(JNIEnv *env, jobject, jlong obj, jstring jfilename, jobject src, jint src_len, jobject dst, jint dst_len) { - TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + TIMED_SECTION("jni:ysparse_buf", (size_type)src_len); char* src_ = nullptr; int* dst_ = nullptr; const char *filename = nullptr; { - TIMED_SECTION("jni_ys2evt_parse/get_jni", (size_type)src_len); + TIMED_SECTION("jni:ysparse_buf/get_jni", (size_type)src_len); src_ = (char*)env->GetDirectBufferAddress(src); dst_ = (int*)env->GetDirectBufferAddress(dst); filename = env->GetStringUTFChars(jfilename, 0); @@ -126,10 +120,10 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, throw_runtime_exception(env, "null pointer: dst"); } { - TIMED_SECTION("jni_ys2evt_parse/call_parse", (size_type)src_len); + TIMED_SECTION("jni:ysparse_buf/parse", (size_type)src_len); try { - return ys2evt_parse((Ryml2Evt*)obj, filename, src_, src_len, dst_, dst_len); + return ysparse_parse((ysparse*)obj, filename, src_, src_len, dst_, dst_len); } catch (YsParseError const& exc) { diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index 54c2aad02..021ccff69 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -17,34 +17,34 @@ JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_init + * Method: ysparse_init * Signature: ()J */ -JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1init (JNIEnv *, jobject); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_destroy + * Method: ysparse_destroy * Signature: (J)V */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1destroy (JNIEnv *, jobject, jlong); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_parse + * Method: ysparse_parse * Signature: (JLjava/lang/String;[BI[II)I */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_parse_buf + * Method: ysparse_parse_buf * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/IntBuffer;I)I */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); #ifdef __cplusplus diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp index 15bb2caf6..48e4dc090 100644 --- a/rapidyaml/native/ysparse_common.hpp +++ b/rapidyaml/native/ysparse_common.hpp @@ -52,6 +52,7 @@ struct timed_section name = n; len = len_; start = myclock::now(); + //fprintf(stderr, "%10s : %s...\n", " ", name); } } C4_NO_INLINE ~timed_section() @@ -59,7 +60,7 @@ struct timed_section if(ysparse_timing_get()) { const std::chrono::duration t = myclock::now() - start; - fprintf(stderr, "%.6fms: %s", t.count(), name); + fprintf(stderr, "%10.6fms: %s", t.count(), name); if(len) fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); fprintf(stderr, "\n"); diff --git a/rapidyaml/native/ysparse_evt.cpp b/rapidyaml/native/ysparse_evt.cpp index 9ef064b3f..0c3497666 100644 --- a/rapidyaml/native/ysparse_evt.cpp +++ b/rapidyaml/native/ysparse_evt.cpp @@ -10,7 +10,7 @@ extern "C" { // https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method namespace { -C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) +C4_NORETURN void ysparse_error(const char* msg, size_t msg_len, Location location, void *user_data) { YsParseError exc; exc.location = location; @@ -19,43 +19,43 @@ C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location lo } } // anon namespace -RYML_EXPORT Ryml2Evt *ys2evt_init() +RYML_EXPORT ysparse *ysparse_init() { - TIMED_SECTION("ys2evt_init"); + TIMED_SECTION("cpp:ysparse_init"); Callbacks cb = {}; - cb.m_error = &ys2evt_parse_error; + cb.m_error = &ysparse_error; set_callbacks(cb); - Ryml2Evt *ryml2evt = _RYML_CB_ALLOC(get_callbacks(), Ryml2Evt, 1); + ysparse *ryml2evt = _RYML_CB_ALLOC(get_callbacks(), ysparse, 1); _RYML_CB_CHECK(get_callbacks(), ryml2evt != nullptr); - new ((void*)ryml2evt) Ryml2Evt(); + new ((void*)ryml2evt) ysparse(); return ryml2evt; } -RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt) +RYML_EXPORT void ysparse_destroy(ysparse *obj) { - TIMED_SECTION("ys2evt_destroy"); - ryml2evt->~Ryml2Evt(); - _RYML_CB_FREE(get_callbacks(), ryml2evt, Ryml2Evt, 1); + TIMED_SECTION("cpp:ysparse_destroy"); + obj->~ysparse(); + _RYML_CB_FREE(get_callbacks(), obj, ysparse, 1); } -RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, - const char *filename, - char *ys, size_type ys_size, - evt::DataType *events, size_type evt_size) +RYML_EXPORT size_type ysparse_parse(ysparse *obj, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *events, size_type evt_size) { - TIMED_SECTION("ys2evt_parse", ys_size); + TIMED_SECTION("cpp:ysparse", ys_size); csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); { - TIMED_SECTION("reset + reserve"); - ryml2evt->reset(ys_, events, evt_size); - ryml2evt->m_handler.reserve(256u); + TIMED_SECTION("cpp:ysparse/reset"); + obj->reset(ys_, events, evt_size); + obj->m_handler.reserve(256u); } { - TIMED_SECTION("parse_in_place", ys_size); - ryml2evt->m_parser.parse_in_place_ev(filename_, ys_); + TIMED_SECTION("cpp:ysparse/parse", ys_size); + obj->m_parser.parse_in_place_ev(filename_, ys_); } - return (size_type)ryml2evt->m_handler.m_evt_curr; + return (size_type)obj->m_handler.m_evt_curr; } #if defined(__cplusplus) diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp index cc231ae1a..9b0c9dbb8 100644 --- a/rapidyaml/native/ysparse_evt.hpp +++ b/rapidyaml/native/ysparse_evt.hpp @@ -9,11 +9,11 @@ extern "C" { #endif -struct RYML_EXPORT Ryml2Evt +struct RYML_EXPORT ysparse { ys::EventHandlerEvt m_handler; c4::yml::ParseEngine m_parser; - Ryml2Evt() + ysparse() : m_handler() , m_parser(&m_handler) { @@ -29,10 +29,10 @@ struct RYML_EXPORT Ryml2Evt //----------------------------------------------------------------------------- /** Initialize the resources */ -RYML_EXPORT Ryml2Evt *ys2evt_init(); +RYML_EXPORT ysparse *ysparse_init(); /** Destroy the resources */ -RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt); +RYML_EXPORT void ysparse_destroy(ysparse *ryml2evt); /** Parse YAML in the string `ys` of size `ys_size`, and write the * result into the array of (integer) events `evt` of size @@ -82,10 +82,10 @@ RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt); * in-place in the input string, and the extra integers will pertain * to the resulting filtered string. */ -RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, - const char *filename, - char *ys, size_type ys_size, - evt::DataType *evt, size_type evt_size); +RYML_EXPORT size_type ysparse_parse(ysparse *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *evt, size_type evt_size); #if defined(__cplusplus) } diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 77f31c5e2..8567e3342 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -45,9 +45,9 @@ c4::EnumSymbols const esyms() struct Ys2EvtScoped { - Ryml2Evt *ryml2evt; - Ys2EvtScoped() : ryml2evt(ys2evt_init()) {} - ~Ys2EvtScoped() { if(ryml2evt) ys2evt_destroy(ryml2evt); } + ysparse *ryml2evt; + Ys2EvtScoped() : ryml2evt(ysparse_init()) {} + ~Ys2EvtScoped() { if(ryml2evt) ysparse_destroy(ryml2evt); } }; @@ -127,7 +127,7 @@ struct TestCase } \ } while(0) - TestResult test(Ryml2Evt *ryml2evt) const + TestResult test(ysparse *ryml2evt) const { TestResult tr = {}; _runtest(test_evt_large_enough, ); @@ -140,7 +140,7 @@ struct TestCase } // happy path: large-enough destination string - TestResult test_evt_large_enough_reuse(Ryml2Evt *ryml2evt) const + TestResult test_evt_large_enough_reuse(ysparse *ryml2evt) const { if(evt.empty()) return {}; TestResult tr = {}; @@ -148,7 +148,7 @@ struct TestCase substr input = c4::to_substr(input_); std::vector output; output.resize(2 * expected_size(evt)); - size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, &output[0], (size_type)output.size()); CHECK_MSG((size_t)reqsize == expected_size(evt), "%d vs %zu", reqsize, expected_size(evt)); @@ -164,14 +164,14 @@ struct TestCase } // less-happy path: destination string not large enough - TestResult test_evt_too_small_reuse(Ryml2Evt *ryml2evt) const + TestResult test_evt_too_small_reuse(ysparse *ryml2evt) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); std::vector output; output.resize(expected_size(evt)); - size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, output.data(), (size_type)output.size()); CHECK(reqsize == expected_size(evt)); @@ -179,7 +179,7 @@ struct TestCase output.resize(reqsize); input_.assign(ys.begin(), ys.end()); // FIXME input = c4::to_substr(input_); - size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize2 = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, output.data(), (size_type)output.size()); CHECK(reqsize2 == reqsize); @@ -194,12 +194,12 @@ struct TestCase } // safe calling with nullptr - TestResult test_evt_nullptr_reuse(Ryml2Evt *ryml2evt) const + TestResult test_evt_nullptr_reuse(ysparse *ryml2evt) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); - size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, nullptr, 0); CHECK(reqsize == expected_size(evt)); @@ -208,7 +208,7 @@ struct TestCase output.resize(reqsize); input_.assign(ys.begin(), ys.end()); // FIXME input = c4::to_substr(input_); - size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize2 = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, output.data(), (size_type)output.size()); CHECK(reqsize2 == reqsize); diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 4c7bc2fd4..f071e1ac9 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -15,22 +15,22 @@ public class Rapidyaml private native void ysparse_timing_set(boolean yes); // TODO: rename these to ysparse_init() etc - private native long ys2evt_init(); - private native void ys2evt_destroy(long ryml2evt); - private native int ys2evt_parse(long ryml2evt, String filename, - byte[] ys, int ys_length, - int[] evt, int evt_length); - private native int ys2evt_parse_buf(long ryml2evt, String filename, - ByteBuffer ys, int ys_length, - IntBuffer evt, int evt_length); + private native long ysparse_init(); + private native void ysparse_destroy(long ysparse); + private native int ysparse_parse(long ysparse, String filename, + byte[] ys, int ys_length, + int[] evt, int evt_length); + private native int ysparse_parse_buf(long ysparse, String filename, + ByteBuffer ys, int ys_length, + IntBuffer evt, int evt_length); - private final long ryml2evt; + private final long ysparse; public Rapidyaml() { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); - this.ryml2evt = this.ys2evt_init(); + this.ysparse = this.ysparse_init(); // TODO: receive this argument as ctor parameter timingEnabled(System.getenv("YS_RYML_TIMER") != null); } @@ -42,7 +42,7 @@ public Rapidyaml() protected void finalize() throws Throwable { try { - this.ys2evt_destroy(this.ryml2evt); + this.ysparse_destroy(this.ysparse); } finally { super.finalize(); @@ -66,9 +66,9 @@ public int parseYsToEvtBuf(ByteBuffer src, IntBuffer evt) throws Exception public int parseYsToEvt(String filename, byte[] src, int[] evts) throws Exception { - long t = timingStart("ys2evt"); - int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); - timingStop("ys2evt", t, src.length); + long t = timingStart("ysparse"); + int required_size = ysparse_parse(this.ysparse, filename, src, src.length, evts, evts.length); + timingStop("ysparse", t, src.length); return required_size; } @@ -82,13 +82,13 @@ public int parseYsToEvtBuf(String filename, ByteBuffer src, IntBuffer evt) throw // but for evt it really does if(evt.order() != ByteOrder.nativeOrder()) throw new RuntimeException("evt byte order must be native"); - long t = timingStart("ys2evtBuf"); + long t = timingStart("ysparseBuf"); evt.position(evt.capacity()); - int reqsize = ys2evt_parse_buf(this.ryml2evt, filename, src, src.position(), evt, evt.capacity()); + int reqsize = ysparse_parse_buf(this.ysparse, filename, src, src.position(), evt, evt.capacity()); if(reqsize <= evt.capacity()) { evt.position(reqsize); } - timingStop("ys2evtBuf", t, src.position()); + timingStop("ysparseBuf", t, src.position()); return reqsize; } From c9cb79b57020d299dd3ac4be16a3943fbd1e0463 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 24 Feb 2025 11:50:40 +0000 Subject: [PATCH 20/21] cicd: Split workflows to ys and rapidyaml --- .github/workflows/rapidyaml.yml | 77 ++++++--------------------------- .github/workflows/ys.yml | 49 +++++++++++++++++++++ 2 files changed, 63 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/ys.yml diff --git a/.github/workflows/rapidyaml.yml b/.github/workflows/rapidyaml.yml index 8cfa4103a..395e8d0a4 100644 --- a/.github/workflows/rapidyaml.yml +++ b/.github/workflows/rapidyaml.yml @@ -60,39 +60,29 @@ jobs: uses: actions/checkout@v4 with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched - name: check jni header up to date - run: | - make -C rapidyaml/native -B jni jnicheck + run: make -C rapidyaml/native -B jni jnicheck - name: get rapidyaml - run: | - make -C rapidyaml/native rapidyaml + run: make -C rapidyaml/native rapidyaml - name: run c++ tests, static ----------------------------- run: echo - name: cfg c++, static - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-static - name: build c++ tests, static - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-static - name: run c++ tests, static - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static - name: run c++ tests, static with timing - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static-timing + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static-timing - name: run c++ tests, shared ----------------------------- run: echo - name: cfg c++, shared - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-shared - name: build c++ tests, shared - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-shared - name: run c++ tests, shared - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared - name: run c++ tests, shared with timing - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared-timing + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared-timing # run the java tests, also in Debug to test with assertions java: @@ -110,49 +100,10 @@ jobs: uses: actions/checkout@v4 with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched - name: build lib - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml build + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml build - name: build jar - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml jar + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml jar - name: install - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml install + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml install - name: run java tests - run: | - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml test - - # run ys tests, also in Debug to test with assertions - ys: - name: ys/${{matrix.ysparser}}/${{matrix.bt}} - runs-on: ubuntu-24.04 - if: always() - continue-on-error: false - strategy: - fail-fast: false - matrix: - include: - - {ysparser: ryml, bt: Debug} - - {ysparser: ryml, bt: Release} - steps: - - name: checkout (action) - uses: actions/checkout@v4 - with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched - - name: run core tests - run: | - . .profile - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - YS_PARSER=${{matrix.ysparser}} \ - make test-core v=1 - - name: run ys tests - run: | - . .profile - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - YS_PARSER=${{matrix.ysparser}} \ - make -C ys test-run v=1 - - name: run ys tests ? - run: | - . .profile - RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ - YS_PARSER=${{matrix.ysparser}} \ - make test-ys v=1 # is this the same as above? + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml test diff --git a/.github/workflows/ys.yml b/.github/workflows/ys.yml new file mode 100644 index 000000000..fd0ec886d --- /dev/null +++ b/.github/workflows/ys.yml @@ -0,0 +1,49 @@ +name: ys + +defaults: + run: + shell: bash -xeo pipefail {0} +'on': + workflow_dispatch: null + push: + branches: + - main + pull_request: + +jobs: + + # run ys tests, also in Debug to test with assertions + ys: + name: ys/${{matrix.ysparser}}/${{matrix.bt}} + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - {v: 1, ysparser: snake} + - {v: 1, ysparser: ryml, bt: Debug} + - {v: 1, ysparser: ryml, bt: Release} + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: run core tests + run: | + . .profile + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + YS_PARSER=${{matrix.ysparser}} \ + make test-core v=${{matrix.v}} + - name: run ys tests + run: | + . .profile + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + YS_PARSER=${{matrix.ysparser}} \ + make -C ys test-run v=${{matrix.v}} + - name: run ys tests ? + run: | + . .profile + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} \ + YS_PARSER=${{matrix.ysparser}} \ + make test-ys v=${{matrix.v}} From 6d2cfde66ea941e9c7551fd517a7290ecda31407 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Fri, 27 Dec 2024 15:18:56 +0000 Subject: [PATCH 21/21] rapidyaml: Add optional static linking via musl This is done via a cmake toolchain file (added here) to the use musl toolchain, which the makefile downloads as appropriate. --- .github/workflows/rapidyaml.yml | 40 +++++++++++++++---------- common/vars.mk | 1 + rapidyaml/native/Makefile | 48 +++++++++++++++++++++++------- rapidyaml/native/musl.x86_64.cmake | 34 +++++++++++++++++++++ 4 files changed, 97 insertions(+), 26 deletions(-) create mode 100644 rapidyaml/native/musl.x86_64.cmake diff --git a/.github/workflows/rapidyaml.yml b/.github/workflows/rapidyaml.yml index 395e8d0a4..f27754ddd 100644 --- a/.github/workflows/rapidyaml.yml +++ b/.github/workflows/rapidyaml.yml @@ -53,8 +53,12 @@ jobs: fail-fast: false matrix: include: - - bt: Debug - - bt: Release + - {bt: Debug, musl: 0} + - {bt: Debug, musl: 1} + - {bt: Release, musl: 0} + - {bt: Release, musl: 1} + env: + MKOPTS: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} RAPIDYAML_MUSL=${{matrix.musl}} steps: - name: checkout (action) uses: actions/checkout@v4 @@ -66,23 +70,23 @@ jobs: - name: run c++ tests, static ----------------------------- run: echo - name: cfg c++, static - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-static + run: $MKOPTS make -C rapidyaml/native cfg-static - name: build c++ tests, static - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-static + run: $MKOPTS make -C rapidyaml/native build-static - name: run c++ tests, static - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static + run: $MKOPTS make -C rapidyaml/native test-static - name: run c++ tests, static with timing - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static-timing + run: $MKOPTS make -C rapidyaml/native test-static-timing - name: run c++ tests, shared ----------------------------- run: echo - name: cfg c++, shared - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-shared + run: $MKOPTS make -C rapidyaml/native cfg-shared - name: build c++ tests, shared - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-shared + run: $MKOPTS make -C rapidyaml/native build-shared - name: run c++ tests, shared - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared + run: $MKOPTS make -C rapidyaml/native test-shared - name: run c++ tests, shared with timing - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared-timing + run: $MKOPTS make -C rapidyaml/native test-shared-timing # run the java tests, also in Debug to test with assertions java: @@ -93,17 +97,21 @@ jobs: fail-fast: false matrix: include: - - bt: Debug - - bt: Release + - {bt: Debug, musl: 0} + - {bt: Debug, musl: 1} + - {bt: Release, musl: 0} + - {bt: Release, musl: 1} + env: + MKOPTS: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} RAPIDYAML_MUSL=${{matrix.musl}} steps: - name: checkout (action) uses: actions/checkout@v4 with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched - name: build lib - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml build + run: $MKOPTS make -C rapidyaml build - name: build jar - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml jar + run: $MKOPTS make -C rapidyaml jar - name: install - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml install + run: $MKOPTS make -C rapidyaml install - name: run java tests - run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml test + run: $MKOPTS make -C rapidyaml test diff --git a/common/vars.mk b/common/vars.mk index ccb0cc1a2..363efe1c4 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -235,6 +235,7 @@ RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml RAPIDYAML_BUILD_TYPE ?= Release RAPIDYAML_DBG ?= 0 RAPIDYAML_TIMED ?= 1 +RAPIDYAML_MUSL ?= 1 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \ diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index e656be03e..97316fa99 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -9,7 +9,9 @@ include $(COMMON)/python.mk # https://stackoverflow.com/questions/24493337/linking-static-library-with-jni THIS_DIR := $(shell pwd) -BDIR := $(THIS_DIR)/_build/$(RAPIDYAML_BUILD_TYPE)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) +BUILD_ROOT := $(THIS_DIR)/_build +MUSL_DIR := $(BUILD_ROOT)/x86_64-linux-musl-cross +BDIR := $(BUILD_ROOT)/$(RAPIDYAML_BUILD_TYPE)-musl$(RAPIDYAML_MUSL)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) RAPIDYAML_DEPS := \ Makefile \ @@ -19,12 +21,23 @@ RAPIDYAML_DEPS := \ $(wildcard ./*pp) \ CMK_ENV ?= -CMK_FLAGS_EXTRA ?= +CMK_ENV_STATIC ?= +CMK_ENV_SHARED ?= CMK_FLAGS := \ -D CMAKE_BUILD_TYPE=$(RAPIDYAML_BUILD_TYPE) \ -D YSPARSE_TIMED=$(RAPIDYAML_TIMED) \ -D YSPARSE_DBG=$(RAPIDYAML_DBG) \ -D CMAKE_EXPORT_COMPILE_COMMANDS=ON +CMK_FLAGS_STATIC ?= +CMK_FLAGS_SHARED ?= +CMK_FLAGS_EXTRA ?= +ifneq ($(RAPIDYAML_MUSL),0) + CMK_ENV += MUSL_DIR=$(MUSL_DIR) + CMK_FLAGS_STATIC += \ + -D CMAKE_TOOLCHAIN_FILE=musl.x86_64.cmake \ + -D CMAKE_C_FLAGS='-static' \ + -D CMAKE_CXX_FLAGS='-static' +endif #------------------------------------------------------------------------------ @@ -60,26 +73,41 @@ jnicheck: jni build-static: $(RAPIDYAML_LIB) cfg-static: rapidyaml - $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF + $(CMK_ENV) $(CMK_ENV_STATIC) $(CMAKE) -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_STATIC) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF test-static: build-static - $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run + $(CMK_ENV) $(CMK_ENV_STATIC) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run test-static-timing: build-static - $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run-timing + $(CMK_ENV) $(CMK_ENV_STATIC) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run-timing $(RAPIDYAML_LIB): cfg-static $(RAPIDYAML_DEPS) - $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --parallel --target rapidyaml + $(CMK_ENV) $(CMK_ENV_STATIC) $(CMAKE) --build $(BDIR)-static --verbose --parallel --target rapidyaml cp -fv $(BDIR)-static/*.a $@ build-shared: $(RAPIDYAML_SO) cfg-shared: rapidyaml - $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON + $(CMK_ENV) $(CMK_ENV_SHARED) $(CMAKE) -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_SHARED) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON test-shared: build-shared - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run + $(CMK_ENV) $(CMK_ENV_SHARED) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run test-shared-timing: build-shared - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run-timing + $(CMK_ENV) $(CMK_ENV_SHARED) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run-timing $(RAPIDYAML_SO): cfg-shared $(RAPIDYAML_DEPS) - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --parallel --target rapidyaml + $(CMK_ENV) $(CMK_ENV_SHARED) $(CMAKE) --build $(BDIR)-shared --verbose --parallel --target rapidyaml cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so +musl: +ifneq ($(RAPIDYAML_MUSL),0) +musl: $(MUSL_DIR) +cfg-static: $(MUSL_DIR) +cfg-shared: $(MUSL_DIR) +$(MUSL_DIR): + mkdir -p `dirname $(MUSL_DIR)` + cd `dirname $(MUSL_DIR)` && \ + wget https://musl.cc/x86_64-linux-musl-cross.tgz && \ + tar xfz x86_64-linux-musl-cross.tgz + ln -fs /usr/lib/ld-musl-x86_64.so.1 $(MUSL_DIR)/bin/ldd + @# musls dynamic linker is an ldd stand-in + @# see https://wiki.musl-libc.org/faq +endif + $(RAPIDYAML_JNI_H): $(JAVAC) $(RAPIDYAML_JAVA) $(JAVAC) -h . $(RAPIDYAML_JAVA) # $^ doesn't work diff --git a/rapidyaml/native/musl.x86_64.cmake b/rapidyaml/native/musl.x86_64.cmake new file mode 100644 index 000000000..f20fb53b0 --- /dev/null +++ b/rapidyaml/native/musl.x86_64.cmake @@ -0,0 +1,34 @@ +set(MUSL_DIR $ENV{MUSL_DIR}) +set(MUSL_TGT x86_64-linux-musl) + +set(MUSL_BIN ${MUSL_DIR}/bin/${MUSL_TGT}-) +set(MUSL_PFX ${MUSL_DIR}/${MUSL_TGT}) +set(MUSL_LIB ${MUSL_DIR}/${MUSL_TGT}/lib) +set(MUSL_INC ${MUSL_DIR}/${MUSL_TGT}/include) + +set(MUSL TRUE) + +set(CMAKE_C_COMPILER ${MUSL_BIN}gcc) +set(CMAKE_CXX_COMPILER ${MUSL_BIN}g++) +set(CMAKE_AR ${MUSL_BIN}ar) +set(CMAKE_C_COMPILER_AR ${MUSL_BIN}ar) +set(CMAKE_CXX_COMPILER_AR ${MUSL_BIN}ar) +set(CMAKE_RANLIB ${MUSL_BIN}ranlib) +set(CMAKE_C_COMPILER_RANLIB ${MUSL_BIN}ranlib) +set(CMAKE_CXX_COMPILER_RANLIB ${MUSL_BIN}ranlib) +set(CMAKE_ADDR2LINE ${MUSL_BIN}addr2line) +set(CMAKE_LINKER ${MUSL_BIN}ld) +set(CMAKE_NM ${MUSL_BIN}nm) +set(CMAKE_OBJCOPY ${MUSL_BIN}objcopy) +set(CMAKE_OBJDUMP ${MUSL_BIN}objdump) +set(CMAKE_READELF ${MUSL_BIN}readelf) +set(CMAKE_STRIP ${MUSL_BIN}strip) + +# set searching rules for cross-compiler +set(CMAKE_SYSTEM_PREFIX_PATH ${MUSL_PFX}) +set(CMAKE_SYSTEM_LIBRARY_PATH ${MUSL_LIB}) +set(CMAKE_SYSTEM_INCLUDE_PATH ${MUSL_INC}) +set(CMAKE_FIND_ROOT_PATH ${MUSL_PFX}) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH) # search also in the host +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH) # search also in the host +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) # search also in the host