From 8e789ba3ff7d60f260449d15879c405fddee882a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sun, 23 Mar 2025 20:40:27 -0700 Subject: [PATCH 01/29] make: Major Makefile refactoring --- .gitignore | 1 + Makefile | 27 +------ clojure/Makefile | 2 +- common/base.mk | 6 +- common/binding.mk | 8 +- common/clojure.mk | 3 +- common/docker-apt-base.dockerfile | 22 ------ common/docker-apt-clojure.dockerfile | 5 -- common/docker-apt-dev.dockerfile | 14 ---- common/docker-apt-node.dockerfile | 12 --- common/docker-apt-perl.dockerfile | 5 -- common/docker-apt-python.dockerfile | 7 -- common/docker-copy-project-clj.dockerfile | 2 - common/docker-copy-project-deps.dockerfile | 2 - common/docker-deps-clojure.dockerfile | 5 -- common/docker-deps-node.dockerfile | 9 --- common/docker-deps-perl.dockerfile | 6 -- common/docker-deps-python.dockerfile | 3 - common/docker-from-ubuntu.dockerfile | 6 -- common/docker-install-graalvm.dockerfile | 8 -- common/docker-useradd.dockerfile | 12 --- common/docker.mk | 38 --------- common/native.mk | 3 +- common/project.clj | 26 ------- common/reflection.json | 68 +++++++++------- common/vars-cli.mk | 32 ++++++++ common/vars-core.mk | 8 ++ common/vars-libys.mk | 33 ++++++++ common/vars.mk | 12 +-- core/Makefile | 25 +++--- go/Makefile | 2 +- java/Makefile | 4 +- julia/Makefile | 2 +- libyamlscript/Makefile | 86 +++++++-------------- libyamlscript/src/libyamlscript/API.java | 2 +- libyamlscript/src/libyamlscript/core.clj | 2 +- nodejs/Makefile | 2 +- perl/Makefile | 10 --- python/Makefile | 22 +----- python/ReadMe.md | 4 +- ruby/ReadMe.md | 4 +- rust/ReadMe.md | 4 +- www/src/install | 4 +- ys/Makefile | 90 ++++++++-------------- 44 files changed, 214 insertions(+), 434 deletions(-) delete mode 100644 common/docker-apt-base.dockerfile delete mode 100644 common/docker-apt-clojure.dockerfile delete mode 100644 common/docker-apt-dev.dockerfile delete mode 100644 common/docker-apt-node.dockerfile delete mode 100644 common/docker-apt-perl.dockerfile delete mode 100644 common/docker-apt-python.dockerfile delete mode 100644 common/docker-copy-project-clj.dockerfile delete mode 100644 common/docker-copy-project-deps.dockerfile delete mode 100644 common/docker-deps-clojure.dockerfile delete mode 100644 common/docker-deps-node.dockerfile delete mode 100644 common/docker-deps-perl.dockerfile delete mode 100644 common/docker-deps-python.dockerfile delete mode 100644 common/docker-from-ubuntu.dockerfile delete mode 100644 common/docker-install-graalvm.dockerfile delete mode 100644 common/docker-useradd.dockerfile delete mode 100644 common/docker.mk delete mode 100644 common/project.clj create mode 100644 common/vars-cli.mk create mode 100644 common/vars-core.mk create mode 100644 common/vars-libys.mk diff --git a/.gitignore b/.gitignore index d5629f25d..cb8881acc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.vimrc /*.clj /*.md *.md.tmp diff --git a/Makefile b/Makefile index 72cc80595..935ebdb6b 100644 --- a/Makefile +++ b/Makefile @@ -49,9 +49,6 @@ PUBLISH := $(DIRS:%=publish-%) CLEAN := $(DIRS:%=clean-%) REALCLEAN := $(DIRS:%=realclean-%) DISTCLEAN := $(DIRS:%=distclean-%) -DOCKER_BUILD := $(DIRS:%=docker-build-%) -DOCKER_TEST := $(DIRS:%=docker-test-%) -DOCKER_SHELL := $(DIRS:%=docker-shell-%) export HEAD := $(shell git rev-parse HEAD) @@ -142,7 +139,7 @@ test: $(TEST) test-core: $(MAKE) -C core test v=$v test-ys: - $(MAKE) -C ys test-all v=$v GRAALVM_O=b + $(MAKE) -C ys test v=$v GRAALVM_O=b test-%: % $(MAKE) -C $< test v=$v GRAALVM_O=b test-unit: @@ -262,7 +259,9 @@ bump: $(BUILD_BIN_YS) $(CLEAN): clean: $(CLEAN) - $(RM) -r libyamlscript/lib ys/bin $(MAVEN_REPOSITORY)/yamlscript + $(RM) -r $(MAVEN_REPOSITORY)/yamlscript + $(RM) -r $(MAVEN_REPOSITORY)/org/yamlscript + $(RM) -r libyamlscript/lib ys/bin $(RM) -r libyamlscript-0* ys-0* yamlscript.cli-*.jar $(RM) -r sample/advent/hearsay-rust/target/ $(RM) -r homebrew-yamlscript @@ -289,24 +288,6 @@ distclean-%: % $(MAKE) -C $< distclean $(RM) -r .calva/ .clj-kondo/.cache .lsp/ -# XXX Limit removing ~/.m2 to ingy until we can get ~/.m2 to not be used sysclean: realclean $(RM) -r $(YS_TMP) $(RM) -r /tmp/yamlscript-* /tmp/ys-local -ifeq (ingy,$(USER)) - $(RM) -r $(HOME)/.m2 -endif - -$(DOCKER_BUILD): -docker-build: $(DOCKER_BUILD) -docker-build-%: % - $(MAKE) -C $< docker-build - -$(DOCKER_TEST): -docker-test: $(DOCKER_TEST) -docker-test-%: % - $(MAKE) -C $< docker-test v=$v - -$(DOCKER_SHELL): -docker-shell-%: % - $(MAKE) -C $< docker-shell v=$v diff --git a/clojure/Makefile b/clojure/Makefile index 70ace4810..3c6921e51 100644 --- a/clojure/Makefile +++ b/clojure/Makefile @@ -8,7 +8,7 @@ export CLOJARS_PASSWORD ?= $(shell util/get-setting password) #------------------------------------------------------------------------------ -test install deploy:: $(LEIN) $(YAMLSCRIPT_JAVA_INSTALLED) +test install deploy:: $(LEIN) $< $@ release: deploy diff --git a/common/base.mk b/common/base.mk index 06158290b..94a126511 100644 --- a/common/base.mk +++ b/common/base.mk @@ -20,7 +20,7 @@ export YSLANG := $(SUBDIR) #------------------------------------------------------------------------------ .SECONDEXPANSION: -.DELETE_ON_ERROR: +# .DELETE_ON_ERROR: .PHONY: test @@ -41,10 +41,6 @@ chown:: clean-all:: $(MAKE) -C $(ROOT) $@ -docker-build:: - -docker-test:: - always: env:: diff --git a/common/binding.mk b/common/binding.mk index bbd6b15ba..cc8cd1f64 100644 --- a/common/binding.mk +++ b/common/binding.mk @@ -1,12 +1,10 @@ -include $(COMMON)/docker.mk +include $(COMMON)/vars-libys.mk -test:: $(LIBYAMLSCRIPT_SO_FQNP) +test:: $(LIBYS_SO_FQNP) -$(LIBYAMLSCRIPT_SO_FQNP): | $(ROOT)/libyamlscript +$(LIBYS_SO_FQNP): | $(ROOT)/libyamlscript $(MAKE) -C $(ROOT)/libyamlscript build -export PATH := $(BUILD_BIN):$(PATH) - build-doc:: build-bin build-bin: diff --git a/common/clojure.mk b/common/clojure.mk index 158fc0c37..d75792763 100644 --- a/common/clojure.mk +++ b/common/clojure.mk @@ -45,7 +45,6 @@ LEIN_REPL_OPTIONS := \ #------------------------------------------------------------------------------ clean:: - $(RM) Dockerfile $(RM) -r .lein-* $(RM) -r reports/ target/ @@ -128,7 +127,7 @@ ifdef PORT repl-port := :port $(PORT) endif -.nrepl-pid: $(LEIN) +.nrepl-pid: $(LEIN) repl-deps ( \ $< $(LEIN_REPL_OPTIONS) repl :headless $(repl-port) & \ echo $$! > $@ \ diff --git a/common/docker-apt-base.dockerfile b/common/docker-apt-base.dockerfile deleted file mode 100644 index a27315f9d..000000000 --- a/common/docker-apt-base.dockerfile +++ /dev/null @@ -1,22 +0,0 @@ - -RUN set -x \ - && apt-get update \ - && apt-get install -y \ - apt-file \ - apt-transport-https \ - autoconf \ - automake \ - bash \ - build-essential \ - cpio \ - curl \ - dialog \ - git \ - iputils-ping \ - libtool \ - pkgconf \ - unzip \ - wget \ - zip \ - zlib1g-dev \ - && true diff --git a/common/docker-apt-clojure.dockerfile b/common/docker-apt-clojure.dockerfile deleted file mode 100644 index 5e408a67d..000000000 --- a/common/docker-apt-clojure.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ - -RUN set -x \ - && sudo apt-get install -y \ - leiningen \ - && true diff --git a/common/docker-apt-dev.dockerfile b/common/docker-apt-dev.dockerfile deleted file mode 100644 index e6e8e3d07..000000000 --- a/common/docker-apt-dev.dockerfile +++ /dev/null @@ -1,14 +0,0 @@ - -RUN set -x \ - && sudo apt-get install -y \ - bash-completion \ - less \ - locales \ - silversearcher-ag \ - tig \ - tmate \ - tmux \ - vim \ - zsh \ - zsh-common \ - && true diff --git a/common/docker-apt-node.dockerfile b/common/docker-apt-node.dockerfile deleted file mode 100644 index ef832fd8b..000000000 --- a/common/docker-apt-node.dockerfile +++ /dev/null @@ -1,12 +0,0 @@ - -RUN set +x \ - && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh \ - | bash \ - && export NVM_DIR=$HOME/.nvm \ - && ls -la $HOME \ - && source "$NVM_DIR/nvm.sh" \ - && nvm install --lts \ - && npm install -g yarn \ - && echo "source $NVM_DIR/nvm.sh" >> $HOME/.bashrc \ - && echo "source $HOME/.bashrc" >> $HOME/.bash_profile \ - && true diff --git a/common/docker-apt-perl.dockerfile b/common/docker-apt-perl.dockerfile deleted file mode 100644 index 5212db2e1..000000000 --- a/common/docker-apt-perl.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ - -RUN set -x \ - && sudo apt-get install -y \ - cpanminus \ - && true diff --git a/common/docker-apt-python.dockerfile b/common/docker-apt-python.dockerfile deleted file mode 100644 index 97d6623fb..000000000 --- a/common/docker-apt-python.dockerfile +++ /dev/null @@ -1,7 +0,0 @@ - -RUN set -x \ - && sudo apt-get install -y \ - python3 \ - python3-pytest \ - python3-venv \ - && true diff --git a/common/docker-copy-project-clj.dockerfile b/common/docker-copy-project-clj.dockerfile deleted file mode 100644 index 422ecbef8..000000000 --- a/common/docker-copy-project-clj.dockerfile +++ /dev/null @@ -1,2 +0,0 @@ - -COPY project.clj /tmp/project.clj diff --git a/common/docker-copy-project-deps.dockerfile b/common/docker-copy-project-deps.dockerfile deleted file mode 100644 index f76e81f00..000000000 --- a/common/docker-copy-project-deps.dockerfile +++ /dev/null @@ -1,2 +0,0 @@ - -COPY .project.clj /tmp/project.clj diff --git a/common/docker-deps-clojure.dockerfile b/common/docker-deps-clojure.dockerfile deleted file mode 100644 index 2141e7ea7..000000000 --- a/common/docker-deps-clojure.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ - -RUN set -x \ - && cd /tmp \ - && lein deps \ - && true diff --git a/common/docker-deps-node.dockerfile b/common/docker-deps-node.dockerfile deleted file mode 100644 index a00baafff..000000000 --- a/common/docker-deps-node.dockerfile +++ /dev/null @@ -1,9 +0,0 @@ - -COPY package.json yarn.lock /tmp/ - -RUN set +x \ - && source $HOME/.nvm/nvm.sh \ - && cd /tmp \ - && yarn install \ - && sudo mv node_modules / \ - && true diff --git a/common/docker-deps-perl.dockerfile b/common/docker-deps-perl.dockerfile deleted file mode 100644 index b487c3f33..000000000 --- a/common/docker-deps-perl.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ - -RUN set -x \ - && sudo cpanm -n \ - Lingy \ - Regexp::Common \ - && true diff --git a/common/docker-deps-python.dockerfile b/common/docker-deps-python.dockerfile deleted file mode 100644 index e98d23521..000000000 --- a/common/docker-deps-python.dockerfile +++ /dev/null @@ -1,3 +0,0 @@ - -RUN python3 -m venv /tmp/venv \ - && true diff --git a/common/docker-from-ubuntu.dockerfile b/common/docker-from-ubuntu.dockerfile deleted file mode 100644 index 09d44eac2..000000000 --- a/common/docker-from-ubuntu.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG UBUNTU_VERSION -FROM ubuntu:$UBUNTU_VERSION -ARG DOCKER_UID=1000 -ARG DOCKER_GID=1000 -ARG GRAALVM_TAR=graalvm-jdk-21_linux-x64_bin.tar.gz -SHELL ["/bin/bash", "-c"] diff --git a/common/docker-install-graalvm.dockerfile b/common/docker-install-graalvm.dockerfile deleted file mode 100644 index a0a0abd70..000000000 --- a/common/docker-install-graalvm.dockerfile +++ /dev/null @@ -1,8 +0,0 @@ - -RUN set -x \ - && cd /tmp \ - && curl -o $GRAALVM_TAR \ - https://download.oracle.com/graalvm/21/latest/$GRAALVM_TAR \ - && tar xzf $GRAALVM_TAR \ - && mv graalvm-jdk-21.* graalvm-oracle-21 \ - && true diff --git a/common/docker-useradd.dockerfile b/common/docker-useradd.dockerfile deleted file mode 100644 index 0d1e71c9b..000000000 --- a/common/docker-useradd.dockerfile +++ /dev/null @@ -1,12 +0,0 @@ - -RUN set -x \ - && apt-get install -y sudo \ - && groupadd -g $DOCKER_GID user \ - && useradd -u $DOCKER_UID -g $DOCKER_GID user \ - && adduser user sudo \ - && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \ - && mkdir -p /home/user \ - && chown -R user:user /home/user \ - && true - -USER user diff --git a/common/docker.mk b/common/docker.mk deleted file mode 100644 index e0f67304f..000000000 --- a/common/docker.mk +++ /dev/null @@ -1,38 +0,0 @@ -DOCKER_NAME := yamlscript-$(SUBDIR) -DOCKER_IMAGE := $(DOCKER_NAME):latest -DOCKER_HISTORY := $(YS_TMP)/$(DOCKER_NAME)-bash-history -export DOCKER_UID := $(shell id -u) -export DOCKER_GID := $(shell id -g) - -ifneq (,$(wildcard /.dockerenv)) - DOCKERENV := 1 -endif - -docker-build:: Dockerfile - cp $(COMMON)/project.clj .project.clj - docker build \ - --build-arg UBUNTU_VERSION=22.04 \ - --build-arg DOCKER_UID=$(DOCKER_UID) \ - --build-arg DOCKER_GID=$(DOCKER_GID) \ - --tag $(DOCKER_IMAGE) . - $(RM) $< .project.clj - -docker-test:: docker-build - docker run --rm -it \ - --volume $(ROOT):/host \ - --workdir /host/$(SUBDIR) \ - -u $$DOCKER_UID:$$DOCKER_GID \ - $(DOCKER_IMAGE) \ - make test v=$v - -docker-shell:: docker-build - touch $(DOCKER_HISTORY) - docker run --rm -it \ - --volume $(ROOT):/host \ - --volume $(DOCKER_HISTORY):/home/user/.bash_history \ - --workdir /host/$(SUBDIR) \ - --entrypoint /bin/bash \ - $(DOCKER_IMAGE) - -clean:: - $(RM) .project.clj diff --git a/common/native.mk b/common/native.mk index 87b48eb29..34c300961 100644 --- a/common/native.mk +++ b/common/native.mk @@ -6,10 +6,11 @@ NATIVE_OPTS := \ --initialize-at-build-time \ --enable-preview \ --enable-url-protocols=https \ + --emit=build-report \ -march=compatibility \ + -H:IncludeResources=SCI_VERSION \ -H:ReflectionConfigurationFiles=reflection.json \ -H:+ReportExceptionStackTraces \ - -H:IncludeResources=SCI_VERSION \ -H:Log=registerResource: \ -J-Dclojure.spec.skip-macros=true \ -J-Dclojure.compiler.direct-linking=true \ diff --git a/common/project.clj b/common/project.clj deleted file mode 100644 index 4eec95986..000000000 --- a/common/project.clj +++ /dev/null @@ -1,26 +0,0 @@ -;; This code is licensed under MIT license (See License for details) -;; Copyright 2023-2025 Ingy dot Net - -(defproject yamlscript/docker "0.1.95" - :description "Program in YAML — Code is Data" - :dependencies - [#__ - [borkdude/edamame "1.3.23"] - [clj-commons/clj-yaml "1.0.27"] - [commons-codec/commons-codec "1.11"] - [dev.weavejester/lein-cljfmt "0.11.2"] - [io.github.borkdude/lein-lein2deps "0.1.0"] - [lein-exec "0.3.7"] - [org.apache.httpcomponents/httpclient "4.5.13"] - [org.apache.httpcomponents/httpcore "4.4.15"] - [org.babashka/sci "0.8.41"] - [org.clojure/clojure "1.12.0"] - [org.clojure/data.json "2.4.0"] - [org.clojure/tools.gitlibs "1.0.100"] - [org.clojure/tools.reader "1.3.6"] - [org.flatland/ordered "1.15.11"] - [org.slf4j/slf4j-api "1.7.36"] - [org.snakeyaml/snakeyaml-engine "2.6"] - [pjstadig/humane-test-output "0.11.0"] - [reifyhealth/lein-git-down "0.4.1"] - #__]) diff --git a/common/reflection.json b/common/reflection.json index 3b64a1f6b..fc799578b 100644 --- a/common/reflection.json +++ b/common/reflection.json @@ -1,115 +1,125 @@ [ { - "name": "java.lang.AssertionError", + "name": "clojure.lang.Delay", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name": "java.lang.Class", - "allDeclaredConstructors": true, + "name": "clojure.lang.ExceptionInfo", "allPublicConstructors": true, - "allDeclaredMethods": true, + "allPublicFields": true, "allPublicMethods": true }, { - "name": "java.lang.Exception", + "name": "clojure.lang.LineNumberingPushbackReader", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name": "clojure.lang.Delay", + "name": "clojure.lang.RT", + "methods": [ + {"name": "aget"}, + {"name": "aset"}, + {"name": "aclone"} + ] + }, + { + "name": "java.io.StringReader", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name": "clojure.lang.ExceptionInfo", + "name": "java.io.StringWriter", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"clojure.lang.LineNumberingPushbackReader", + "name": "java.lang.AssertionError", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.String", + "name": "java.lang.Class", + "allDeclaredConstructors": true, + "allDeclaredMethods": true, "allPublicConstructors": true, - "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.io.StringReader", + "name": "java.lang.ClassLoader", + "allDeclaredConstructors": true, + "allDeclaredFields": true, + "allDeclaredMethods": true, "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.io.StringWriter", + "name": "java.lang.Double", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Double", + "name": "java.lang.Exception", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Integer", + "name": "java.lang.IllegalArgumentException", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Number", + "name": "java.lang.Integer", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.IllegalArgumentException", + "name": "java.lang.Math", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.System", + "name": "java.lang.Number", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Math", + "name": "java.lang.String", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Thread", + "name": "java.lang.System", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, - { - "name": "java.lang.reflect.AccessibleObject", - "methods": [{"name":"canAccess"}] + "name": "java.lang.Thread", + "allPublicConstructors": true, + "allPublicFields": true, + "allPublicMethods": true }, - { - "name": "clojure.lang.RT", + "name": "java.lang.reflect.AccessibleObject", "methods": [ - {"name": "aget"}, - {"name": "aset"}, - {"name": "aclone"} + {"name": "canAccess"} ] }, - - {"name": "[Ljava.lang.Double;"} + { + "name": "[Ljava.lang.Double;" + } ] diff --git a/common/vars-cli.mk b/common/vars-cli.mk new file mode 100644 index 000000000..f4c8728de --- /dev/null +++ b/common/vars-cli.mk @@ -0,0 +1,32 @@ +include $(COMMON)/vars-core.mk + +CLI_BIN := bin/ys-$(YS_VERSION) +CLI_SRC := \ + src/yamlscript/cli.clj \ + +CLI_BIN_BASH := bin/ys-sh-$(YS_VERSION) +CLI_BIN_BASH_SRC := share/ys-0.bash + +CLI_JAR := \ + target/uberjar/yamlscript.cli-$(YS_VERSION)-SNAPSHOT-standalone.jar + +CLI_JAR_DEPS := \ + $(LEIN) \ + $(CORE_INSTALLED) \ + $(CLI_SRC) \ + +CLI_DEPS := \ + $(CLI_BIN) \ + $(CLI_BIN_BASH) \ + +ifdef YS_NATIVE_BUILD_STATIC +ifeq (true,$(IS_LINUX)) +ifeq (true,$(IS_INTEL)) +CLI_DEPS := $(MUSL_GCC) $(CLI_DEPS) +NATIVE_OPTS += \ + -H:CCompilerOption=-Wl,-z,stack-size=2097152 \ + --static \ + --libc=musl +endif +endif +endif diff --git a/common/vars-core.mk b/common/vars-core.mk new file mode 100644 index 000000000..c89b28902 --- /dev/null +++ b/common/vars-core.mk @@ -0,0 +1,8 @@ +CORE_DIR := $(ROOT)/core +CORE_JAR := $(CORE_DIR)/target/core-$(YAMLSCRIPT_VERSION)-standalone.jar +CORE_DEPS := $(LEIN) + +CORE_INSTALLED := \ + $(MAVEN_REPOSITORY)/yamlscript/core/$(YAMLSCRIPT_VERSION) +CORE_INSTALLED := \ + $(CORE_INSTALLED)/core-$(YAMLSCRIPT_VERSION).jar diff --git a/common/vars-libys.mk b/common/vars-libys.mk new file mode 100644 index 000000000..399d587e1 --- /dev/null +++ b/common/vars-libys.mk @@ -0,0 +1,33 @@ +LIBYS_DIR := $(ROOT)/libyamlscript +LIBYS_LIB := $(LIBYS_DIR)/lib + +export LD_LIBRARY_PATH := $(LIBYS_LIB):$(LD_LIBRARY_PATH) +export $(DY)LD_LIBRARY_PATH := $(LD_LIBRARY_PATH) + +LIBYS_SO_NAME := $(LIBYS_LIB)/libyamlscript +LIBYS_SO_FQNP := $(LIBYS_SO_NAME).$(SO).$(YAMLSCRIPT_VERSION) +LIBYS_SO_BASE := $(LIBYS_LIB)/libyamlscript.$(SO) +LIBYS_SO_APIP := $(LIBYS_SO_BASE).$(API_VERSION) +LIBYS_SO_VERS := $(LIBYS_LIB)/libyamlscript.$(YAMLSCRIPT_VERSION).$(SO) + +LIBYS_DEPS := \ + $(LIBYS_SO_FQNP) \ + +LIBYS_JAR := \ + $(LIBYS_LIB)/target/libyamlscript-$(YAMLSCRIPT_VERSION)-standalone.jar + +LIBYS_INSTALLED := \ + $(MAVEN_REPOSITORY)/org/yamlscript/yamlscript/$(YAMLSCRIPT_VERSION) +LIBYS_INSTALLED := \ + $(LIBYS_INSTALLED)/yamlscript-$(YAMLSCRIPT_VERSION).jar + +LIBYS_JAR_PATH := \ + target/libyamlscript-$(YAMLSCRIPT_VERSION)-standalone.jar + +LIBYS_SOURCES := \ + src/libyamlscript/core.clj \ + src/libyamlscript/API.java \ + +LIBYS_HEADERS := \ + $(LIBYS_LIB)/graal_isolate.h \ + $(LIBYS_SO_NAME).$(YAMLSCRIPT_VERSION).h \ diff --git a/common/vars.mk b/common/vars.mk index 154eab278..10eb73b17 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -27,6 +27,7 @@ export YAMLSCRIPT_ROOT ?= $(ROOT) export API_VERSION := 0 export YS_VERSION := $(shell grep '^version:' $(ROOT)/Meta | cut -d' ' -f2) +YAMLSCRIPT_VERSION := $(YS_VERSION) ifdef v export TEST_VERBOSE := 1 @@ -82,16 +83,6 @@ CURL := $(shell command -v curl) TIME := time -p -LIBYAMLSCRIPT_DIR := $(ROOT)/libyamlscript/lib -LIBRARY_PATH := $(LIBYAMLSCRIPT_DIR) -export $(DY)LD_LIBRARY_PATH := $(LIBRARY_PATH) -export LD_LIBRARY_PATH := $(LIBRARY_PATH) -LIBYAMLSCRIPT_SO_NAME := $(LIBYAMLSCRIPT_DIR)/libyamlscript -LIBYAMLSCRIPT_SO_FQNP := $(LIBYAMLSCRIPT_SO_NAME).$(SO).$(YS_VERSION) -LIBYAMLSCRIPT_SO_BASE := $(LIBYAMLSCRIPT_DIR)/libyamlscript.$(SO) -LIBYAMLSCRIPT_SO_APIP := $(LIBYAMLSCRIPT_SO_BASE).$(API_VERSION) -LIBYAMLSCRIPT_SO_VERS := $(LIBYAMLSCRIPT_DIR)/libyamlscript.$(YS_VERSION).$(SO) - ifeq (true,$(IS_ROOT)) PREFIX ?= /usr/local else @@ -165,6 +156,7 @@ GRAALVM_DOWNLOAD := $(YS_TMP)/$(GRAALVM_TAR) GRAALVM_INSTALLED := $(GRAALVM_HOME)/release GRAALVM_O ?= 1 +# qbm is Quick Build Mode ifdef qbm GRAALVM_O := b endif diff --git a/core/Makefile b/core/Makefile index a6d28f544..fea294d67 100644 --- a/core/Makefile +++ b/core/Makefile @@ -1,26 +1,25 @@ include ../common/base.mk include $(COMMON)/java.mk include $(COMMON)/clojure.mk -include $(COMMON)/docker.mk +include $(COMMON)/vars-core.mk export PATH := $(ROOT)/core/bin:$(PATH) #------------------------------------------------------------------------------ build:: -install test:: $(LEIN) build - $< $@ +test:: $(CORE_DEPS) + $(LEIN) $@ -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ +install: $(CORE_INSTALLED) + +$(CORE_INSTALLED): $(CORE_JAR) + $(LEIN) install + touch $@ + +$(CORE_JAR): $(CORE_DEPS) + $(LEIN) uberjar + touch $@ clean:: $(RM) pom.xml diff --git a/go/Makefile b/go/Makefile index eb35bbcbc..3b673ca02 100644 --- a/go/Makefile +++ b/go/Makefile @@ -42,7 +42,7 @@ export PATH := $(GOBIN):$(PATH) #------------------------------------------------------------------------------ -build:: build-doc $(LIBYAMLSCRIPT_SO_FQNP) $(GO_INSTALLED) +build:: build-doc $(LIBYS_SO_FQNP) $(GO_INSTALLED) build-doc:: ReadMe.md diff --git a/java/Makefile b/java/Makefile index 1a265fe08..ed27e36a5 100644 --- a/java/Makefile +++ b/java/Makefile @@ -13,10 +13,10 @@ MVN_COMMANDS := \ #------------------------------------------------------------------------------ -$(MVN_COMMANDS):: $(JAVA_INSTALLED) $(LIBYAMLSCRIPT_SO_FQNP) +$(MVN_COMMANDS):: $(JAVA_INSTALLED) $(LIBYS_SO_FQNP) mvn $@ -release: $(JAVA_INSTALLED) $(LIBYAMLSCRIPT_SO_FQNP) test package +release: $(JAVA_INSTALLED) $(LIBYS_SO_FQNP) test package ifndef n $(error 'make $@' needs the n variable set to the new version) endif diff --git a/julia/Makefile b/julia/Makefile index 513a2a139..ca1b9715c 100644 --- a/julia/Makefile +++ b/julia/Makefile @@ -11,7 +11,7 @@ build:: build-doc build-doc:: ReadMe.md ifdef JULIA -test:: $(LIBYAMLSCRIPT_SO_FQNP) deps +test:: $(LIBYS_SO_FQNP) deps $(JULIA) --project=. test/runtests.jl deps:: diff --git a/libyamlscript/Makefile b/libyamlscript/Makefile index 2b58ba6da..46e66453e 100644 --- a/libyamlscript/Makefile +++ b/libyamlscript/Makefile @@ -1,54 +1,34 @@ include ../common/base.mk include $(COMMON)/java.mk include $(COMMON)/clojure.mk -include $(COMMON)/docker.mk include $(COMMON)/native.mk - -LIBYAMLSCRIPT_JAR_PATH := target/libyamlscript-$(YS_VERSION)-standalone.jar - -LYS_SOURCES := \ - src/libyamlscript/core.clj \ - src/libyamlscript/API.java \ - -HEADERS := \ - $(LIBYAMLSCRIPT_DIR)/graal_isolate.h \ - $(LIBYAMLSCRIPT_SO_NAME).$(YS_VERSION).h \ - -BUILD_TARGETS := \ - $(LIBYAMLSCRIPT_SO_FQNP) \ - $(HEADERS) \ - -# Avoid rebuild in ephemerally created Docker container. -ifdef DOCKERENV -ifneq (,$(wildcard $(LIBYAMLSCRIPT_SO_FQNP))) -LIBYAMLSCRIPT_JAR_PATH := -endif -endif +include $(COMMON)/vars-core.mk +include $(COMMON)/vars-libys.mk #------------------------------------------------------------------------------ -build:: $(BUILD_TARGETS) +build:: $(LIBYS_DEPS) -jar: $(LIBYAMLSCRIPT_JAR_PATH) +# jar: $(LIBYS_JAR_PATH) -install:: $(BUILD_TARGETS) +install:: $(LIBYS_DEPS) mkdir -p $(PREFIX)/include/ - install -m 644 $(HEADERS) $(PREFIX)/include/ + install -m 644 $(LIBYS_HEADERS) $(PREFIX)/include/ mkdir -p $(PREFIX)/lib/ install -m 644 $< $(PREFIX)/lib/ - ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYAMLSCRIPT_SO_BASE)) - ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYAMLSCRIPT_SO_APIP)) - ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYAMLSCRIPT_SO_VERS)) + ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYS_SO_BASE)) + ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYS_SO_APIP)) + ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYS_SO_VERS)) realclean:: $(RM) -r lib/ $(RM) src/libyamlscript/Core.class -test:: $(LIBYAMLSCRIPT_SO_FQNP) +test:: $(LIBYS_SO_FQNP) ls -lh lib -repl-deps:: $(LIBYAMLSCRIPT_JAR_PATH) +repl-deps:: $(LIBYS_JAR_PATH) -$(LIBYAMLSCRIPT_SO_FQNP): $(LIBYAMLSCRIPT_JAR_PATH) +$(LIBYS_SO_FQNP): $(LIBYS_JAR_PATH) ifneq (true,$(LIBZ)) $(error *** The 'libz.$(SO)' library is required by native-image but not installed) endif @@ -60,31 +40,17 @@ endif \ --shared \ -jar $< \ - -o $(LIBYAMLSCRIPT_SO_NAME) - mv $(LIBYAMLSCRIPT_SO_BASE) $@ - mv $(LIBYAMLSCRIPT_SO_NAME).h $(LIBYAMLSCRIPT_SO_NAME).$(YS_VERSION).h - ln -fs $(notdir $@) $(LIBYAMLSCRIPT_SO_APIP) - ln -fs $(notdir $@) $(LIBYAMLSCRIPT_SO_VERS) - ln -fs $(notdir $@) $(LIBYAMLSCRIPT_SO_BASE) - -ifeq (true,$(IS_ROOT)) -$(LIBYAMLSCRIPT_JAR_PATH): -ifeq (true,$(IS_ROOT)) - $(error *** YS 'make build' must be run as non-root user) -endif -else -$(LIBYAMLSCRIPT_JAR_PATH): $(LEIN) $(JAVA_INSTALLED) $(YAMLSCRIPT_CORE_INSTALLED) $(LYS_SOURCES) - $< uberjar -endif - -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ + -o $(LIBYS_SO_NAME) + mv $(LIBYS_SO_BASE) $@ + mv $(LIBYS_SO_NAME).h $(LIBYS_SO_NAME).$(YAMLSCRIPT_VERSION).h + ln -fs $(notdir $@) $(LIBYS_SO_APIP) + ln -fs $(notdir $@) $(LIBYS_SO_VERS) + ln -fs $(notdir $@) $(LIBYS_SO_BASE) + touch $@ + +$(LIBYS_JAR_PATH): $(CORE_INSTALLED) + $(LEIN) uberjar + touch $@ + +$(CORE_INSTALLED): + $(MAKE) -C $(CORE_DIR) $@ diff --git a/libyamlscript/src/libyamlscript/API.java b/libyamlscript/src/libyamlscript/API.java index 008adf8ef..060c99f33 100644 --- a/libyamlscript/src/libyamlscript/API.java +++ b/libyamlscript/src/libyamlscript/API.java @@ -33,7 +33,7 @@ public final class API { } public static void debug(String s) { - if (System.getenv("LIBYAMLSCRIPT_DEBUG") != null) { + if (System.getenv("YS_DEBUG") != null) { System.err.println(s); } } diff --git a/libyamlscript/src/libyamlscript/core.clj b/libyamlscript/src/libyamlscript/core.clj index 156a4f084..a5f5f17c3 100644 --- a/libyamlscript/src/libyamlscript/core.clj +++ b/libyamlscript/src/libyamlscript/core.clj @@ -49,7 +49,7 @@ :trace (get-in err [:trace])}})) (defn debug [& msg] - (when (System/getenv "LIBYAMLSCRIPT_DEBUG") + (when (System/getenv "YS_DEBUG") (binding [*out* *err*] (apply println msg)))) diff --git a/nodejs/Makefile b/nodejs/Makefile index d95110448..4ad6032c6 100644 --- a/nodejs/Makefile +++ b/nodejs/Makefile @@ -27,7 +27,7 @@ build:: build-doc build-doc:: ReadMe.md -test:: update $(LIBYAMLSCRIPT_SO_FQNP) +test:: update $(LIBYS_SO_FQNP) @printf '%.0s-' {1..80}; echo $(strace) node test/test.js diff --git a/perl/Makefile b/perl/Makefile index 29e9710f5..caf9fdc58 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -55,13 +55,3 @@ $(ALIEN_DIST_DIR): $(LOCAL_LIB): cpanm -L $@ -n $(CPAN_DEPS) - -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ diff --git a/python/Makefile b/python/Makefile index 249694c72..9e28964b3 100644 --- a/python/Makefile +++ b/python/Makefile @@ -2,11 +2,7 @@ include ../common/base.mk include $(COMMON)/binding.mk include $(COMMON)/python.mk -ifdef DOCKERENV - PYTHON_VENV := $(YS_TMP)/venv -else - PYTHON_VENV := $(ROOT)/python/.venv -endif +PYTHON_VENV := $(ROOT)/python/.venv ACTIVATE := source $(PYTHON_VENV)/bin/activate @@ -33,7 +29,7 @@ test-pytest: $(PYTHON_VENV) $(ACTIVATE) && \ pytest $${TEST_VERBOSE:+'-v'} test/*.py -test-ffi: $(LIBYAMLSCRIPT_SO_FQNP) +test-ffi: $(LIBYS_SO_FQNP) $(ACTIVATE) && \ $(PYTHON) \ -c 'import yamlscript; \ @@ -94,17 +90,3 @@ MANIFEST.in: grep -B999 '## Features' | \ head -n-2 \ > $@ - -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ diff --git a/python/ReadMe.md b/python/ReadMe.md index 2677f1b95..c5eead29a 100644 --- a/python/ReadMe.md +++ b/python/ReadMe.md @@ -126,7 +126,7 @@ $ python prog.py You can install this module like any other Python module: ```bash -$ pip install yamlscript +pip install yamlscript ``` but you will need to have a system install of `libyamlscript.so`. @@ -134,7 +134,7 @@ but you will need to have a system install of `libyamlscript.so`. One simple way to do that is with: ```bash -$ curl https://yamlscript.org/install | bash +curl https://yamlscript.org/install | bash ``` > Note: The above command will install the latest version of the YAMLScript diff --git a/ruby/ReadMe.md b/ruby/ReadMe.md index 512b7ed15..f25a1947b 100644 --- a/ruby/ReadMe.md +++ b/ruby/ReadMe.md @@ -126,7 +126,7 @@ $ ruby prog.rb You can install this module like any other Ruby module: ```bash -$ gem install yamlscript +gem install yamlscript ``` but you will need to have a system install of `libyamlscript.so`. @@ -134,7 +134,7 @@ but you will need to have a system install of `libyamlscript.so`. One simple way to do that is with: ```bash -$ curl https://yamlscript.org/install | bash +curl https://yamlscript.org/install | bash ``` > Note: The above command will install the latest version of the YAMLScript diff --git a/rust/ReadMe.md b/rust/ReadMe.md index 01ac18f8b..be61bd845 100644 --- a/rust/ReadMe.md +++ b/rust/ReadMe.md @@ -146,7 +146,7 @@ Object {"bar": Object {"oh": String("Hello")}, "baz": String("Hello, World!"), " You can install this module like any other Rust module: ```bash -$ cargo add yamlscript +cargo add yamlscript ``` but you will need to have a system install of `libyamlscript.so`. @@ -154,7 +154,7 @@ but you will need to have a system install of `libyamlscript.so`. One simple way to do that is with: ```bash -$ curl https://yamlscript.org/install | bash +curl https://yamlscript.org/install | bash ``` > Note: The above command will install the latest version of the YAMLScript diff --git a/www/src/install b/www/src/install index 818f1a884..9a3ebf00b 100644 --- a/www/src/install +++ b/www/src/install @@ -9,8 +9,8 @@ set -e -u -o pipefail -LIBYAMLSCRIPT_VERSION=0.1.95 -VERSION=${VERSION:-$LIBYAMLSCRIPT_VERSION} +YAMLSCRIPT_VERSION=0.1.95 +VERSION=${VERSION:-$YAMLSCRIPT_VERSION} export VERSION main() ( diff --git a/ys/Makefile b/ys/Makefile index 8044f4cbf..778c0a231 100644 --- a/ys/Makefile +++ b/ys/Makefile @@ -1,25 +1,8 @@ include ../common/base.mk include $(COMMON)/java.mk include $(COMMON)/clojure.mk -include $(COMMON)/docker.mk include $(COMMON)/native.mk - -YAMLSCRIPT_CLI_BIN := bin/ys-$(YS_VERSION) -YAMLSCRIPT_CLI_SRC := \ - src/yamlscript/cli.clj \ - -YAMLSCRIPT_CLI_BIN_BASH := bin/ys-sh-$(YS_VERSION) -YAMLSCRIPT_CLI_BIN_BASH_SRC := share/ys-0.bash - -YAMLSCRIPT_CLI_JAR_PATH := \ - target/uberjar/yamlscript.cli-$(YS_VERSION)-SNAPSHOT-standalone.jar - -# Avoid rebuild in ephemerally created Docker container. -ifdef DOCKERENV -ifneq (,$(wildcard $(YAMLSCRIPT_CLI_BIN))) -YAMLSCRIPT_CLI_JAR_PATH := -endif -endif +include $(COMMON)/vars-cli.mk BPAN_LOCAL := .bpan BPAN_REPO_URL := https://github.com/bpan-org/bpan @@ -42,36 +25,42 @@ endif #------------------------------------------------------------------------------ -build:: $(YS_BUILD_DEPS) +build:: $(CLI_DEPS) -install: build +install: $(CLI_DEPS) mkdir -p $(PREFIX)/bin - install -m 755 $(YAMLSCRIPT_CLI_BIN) $(PREFIX)/bin/ - ln -fs $(notdir $(YAMLSCRIPT_CLI_BIN)) \ - $(PREFIX)/$(YAMLSCRIPT_CLI_BIN:%-$(YS_VERSION)=%-$(API_VERSION)) - ln -fs $(notdir $(YAMLSCRIPT_CLI_BIN)) \ - $(PREFIX)/$(YAMLSCRIPT_CLI_BIN:%-$(YS_VERSION)=%) - install -m 755 $(YAMLSCRIPT_CLI_BIN_BASH) \ + install -m 755 $(CLI_BIN) $(PREFIX)/bin/ + ln -fs $(notdir $(CLI_BIN)) \ + $(PREFIX)/$(CLI_BIN:%-$(YS_VERSION)=%-$(API_VERSION)) + ln -fs $(notdir $(CLI_BIN)) \ + $(PREFIX)/$(CLI_BIN:%-$(YS_VERSION)=%) + install -m 755 $(CLI_BIN_BASH) \ $(PREFIX)/bin/ -jar: $(YAMLSCRIPT_CLI_JAR_PATH) - @: +# jar: $(CLI_JAR) -test: test-unit +test: test-run test-all: test-unit test-run -test-unit: $(LEIN) $(YAMLSCRIPT_CORE_INSTALLED) - $< test $(lein-test) +test-unit: $(CORE_INSTALLED) + $(LEIN) test $(lein-test) -test-run: $(BPAN_LOCAL) build +test-run: $(CLI_DEPS) $(BPAN_LOCAL) prove $${TEST_VERBOSE:+'-v'} $(test) +repl-deps:: $(CORE_INSTALLED) + +clean:: + $(RM) -r .cpcache/ + +realclean:: + $(RM) -r bin lib $(BPAN_LOCAL) + $(BPAN_LOCAL): git clone --depth=1 $(BPAN_REPO_URL) $@ -$(YAMLSCRIPT_CLI_BIN): $(YAMLSCRIPT_CLI_JAR_PATH) -ifndef YS_BIN_DIR +$(CLI_BIN): $(CLI_JAR) ifneq (true,$(LIBZ)) $(error *** \ The 'libz.$(SO)' library is required by native-image but not installed) @@ -86,34 +75,19 @@ endif -o $@ ln -fs $(notdir $@) $(@:%-$(YS_VERSION)=%-$(API_VERSION)) ln -fs $(notdir $@) $(@:%-$(YS_VERSION)=%) -endif + touch $@ -$(YAMLSCRIPT_CLI_BIN_BASH): $(YAMLSCRIPT_CLI_BIN_BASH_SRC) +$(CLI_BIN_BASH): $(CLI_BIN_BASH_SRC) mkdir -p $(dir $@) cp $< $@ chmod 755 $@ -clean:: - $(RM) -r .cpcache/ - -realclean:: - $(RM) -r bin lib $(BPAN_LOCAL) +$(CLI_JAR): $(CLI_JAR_DEPS) + $(LEIN) uberjar + touch $@ -ifeq (true,$(IS_ROOT)) -$(YAMLSCRIPT_CLI_JAR_PATH): -else -$(YAMLSCRIPT_CLI_JAR_PATH): $(LEIN) $(YAMLSCRIPT_CORE_INSTALLED) $(YAMLSCRIPT_CLI_SRC) - $< uberjar -endif +$(CORE_INSTALLED): + $(MAKE) -C $(CORE_DIR) $@ -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ +$(CORE_JAR): + $(MAKE) -C $(CORE_DIR) $@ From 25740faa77f3a35833e94307f498abc659856404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sun, 23 Mar 2025 22:16:42 -0700 Subject: [PATCH 02/29] make: Uncomment .DELETE_ON_ERROR --- common/base.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/base.mk b/common/base.mk index 94a126511..756004e0a 100644 --- a/common/base.mk +++ b/common/base.mk @@ -20,7 +20,7 @@ export YSLANG := $(SUBDIR) #------------------------------------------------------------------------------ .SECONDEXPANSION: -# .DELETE_ON_ERROR: +.DELETE_ON_ERROR: .PHONY: test From 8e00bcaf9aef512884380b0960410e900bf9e819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 28 Aug 2024 13:22:18 -0700 Subject: [PATCH 03/29] core: Add rapidyaml parser support --- Makefile | 2 + common/vars.mk | 15 + core/Makefile | 2 +- core/deps.edn | 1 + core/project.clj | 1 + core/src/yamlscript/parser.clj | 57 +- libyamlscript/Makefile | 4 + rapidyaml/Makefile | 67 ++ rapidyaml/native/.gitignore | 4 + rapidyaml/native/CMakeLists.txt | 40 ++ rapidyaml/native/Makefile | 71 ++ rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 115 +++ rapidyaml/native/org_rapidyaml_Rapidyaml.h | 45 ++ rapidyaml/native/rapidyaml_edn.cpp | 108 +++ rapidyaml/native/rapidyaml_edn.hpp | 71 ++ rapidyaml/native/rapidyaml_edn_handler.cpp | 83 +++ rapidyaml/native/rapidyaml_edn_handler.hpp | 673 ++++++++++++++++++ rapidyaml/native/rapidyaml_test.cpp | 305 ++++++++ rapidyaml/pom.xml | 115 +++ rapidyaml/rapidyaml.mk | 5 + rapidyaml/src/README.md | 33 + .../main/java/org/rapidyaml/Rapidyaml.java | 61 ++ .../rapidyaml/YamlParseErrorException.java | 16 + rapidyaml/src/site/site.xml | 26 + .../java/org/rapidyaml/RapidyamlTest.java | 163 +++++ util/RYS | 13 + ys/config/reflect-config.json | 1 + ys/config/resource-config.json | 6 + 28 files changed, 2084 insertions(+), 19 deletions(-) create mode 100644 rapidyaml/Makefile create mode 100644 rapidyaml/native/.gitignore create mode 100644 rapidyaml/native/CMakeLists.txt create mode 100644 rapidyaml/native/Makefile create mode 100644 rapidyaml/native/org_rapidyaml_Rapidyaml.cpp create mode 100644 rapidyaml/native/org_rapidyaml_Rapidyaml.h create mode 100644 rapidyaml/native/rapidyaml_edn.cpp create mode 100644 rapidyaml/native/rapidyaml_edn.hpp create mode 100644 rapidyaml/native/rapidyaml_edn_handler.cpp create mode 100644 rapidyaml/native/rapidyaml_edn_handler.hpp create mode 100644 rapidyaml/native/rapidyaml_test.cpp create mode 100644 rapidyaml/pom.xml create mode 100644 rapidyaml/rapidyaml.mk create mode 100644 rapidyaml/src/README.md create mode 100644 rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java create mode 100644 rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java create mode 100644 rapidyaml/src/site/site.xml create mode 100644 rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java create mode 100755 util/RYS create mode 120000 ys/config/reflect-config.json create mode 100644 ys/config/resource-config.json diff --git a/Makefile b/Makefile index 935ebdb6b..6af0b89bf 100644 --- a/Makefile +++ b/Makefile @@ -22,12 +22,14 @@ BINDINGS := \ rust \ DIRS := \ + rapidyaml \ core \ libyamlscript \ $(BINDINGS) \ ys \ BUILD_DIRS := \ + rapidyaml \ libyamlscript \ go \ nodejs \ diff --git a/common/vars.mk b/common/vars.mk index 10eb73b17..afa4ce58d 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -213,6 +213,21 @@ RELEASE_LYS_NAME := libyamlscript-$(YS_VERSION)-$(GRAALVM_ARCH) RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz +#------------------------------------------------------------------------------ +# RapidYAML variables: +#------------------------------------------------------------------------------ +RAPIDYAML := $(ROOT)/rapidyaml + +RAPIDYAML_VERSION := 0.7.2 +RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar +RAPIDYAML_SO := \ + $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_INSTALLED_DIR := \ + $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)/ +RAPIDYAML_INSTALLED := \ + $(RAPIDYAML_INSTALLED_DIR)/rapidyaml-$(RAPIDYAML_VERSION).jar + + #------------------------------------------------------------------------------ default:: diff --git a/core/Makefile b/core/Makefile index fea294d67..c001566df 100644 --- a/core/Makefile +++ b/core/Makefile @@ -6,7 +6,7 @@ include $(COMMON)/vars-core.mk export PATH := $(ROOT)/core/bin:$(PATH) #------------------------------------------------------------------------------ -build:: +build:: $(RAPIDYAML_SO) $(RAPIDYAML_INSTALLED) test:: $(CORE_DEPS) $(LEIN) $@ diff --git a/core/deps.edn b/core/deps.edn index cbb5af22d..5637484a0 100644 --- a/core/deps.edn +++ b/core/deps.edn @@ -5,6 +5,7 @@ org.clojure/data.json {:mvn/version "2.4.0"}, clj-commons/clj-yaml {:mvn/version "1.0.27"}, org.flatland/ordered {:mvn/version "1.15.11"}, + org.rapidyaml/rapidyaml {:mvn/version "0.7.2"}, org.snakeyaml/snakeyaml-engine {:mvn/version "2.7"}, babashka/babashka.pods {:mvn/version "0.2.0"}, babashka/fs {:mvn/version "0.5.20"}, diff --git a/core/project.clj b/core/project.clj index ed58e1860..dbefdf0fd 100644 --- a/core/project.clj +++ b/core/project.clj @@ -22,6 +22,7 @@ [org.clojure/data.json "2.4.0"] [clj-commons/clj-yaml "1.0.27"] [org.flatland/ordered "1.15.11"] + [org.rapidyaml/rapidyaml "0.7.2"] [org.snakeyaml/snakeyaml-engine "2.7"] [babashka/babashka.pods "0.2.0"] [babashka/fs "0.5.20"] diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 00e2af5b4..ae9d390a5 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -9,9 +9,11 @@ (ns yamlscript.parser (:require + [clojure.string :as str] [yamlscript.common]) (:import (java.util Optional) + (org.rapidyaml Rapidyaml) (org.snakeyaml.engine.v2.api LoadSettings) (org.snakeyaml.engine.v2.api.lowlevel Parse) (org.snakeyaml.engine.v2.exceptions Mark) @@ -29,21 +31,20 @@ SequenceEndEvent)) (:refer-clojure)) -(declare ys-event) +(declare + parse-snakeyaml + parse-rapidyaml) (def shebang-ys #"^#!.*/env ys-0(?:\.d+\.\d+)?\n") (def shebang-bash #"^#!.*[/ ]bash\n+source +<\(") (defn parse "Parse a YAML string into a sequence of event objects." [yaml-string] - (let [parser (new Parse (.build (LoadSettings/builder))) - has-code-mode-shebang (or (re-find shebang-ys yaml-string) + (let [has-code-mode-shebang (or (re-find shebang-ys yaml-string) (re-find shebang-bash yaml-string)) - events (->> yaml-string - (.parseString parser) - (map ys-event) - (remove nil?) - rest) + events (if (System/getenv "YS_PARSER_RAPIDYAML") + (parse-rapidyaml yaml-string) + (parse-snakeyaml yaml-string)) [first-event & rest-events] events first-event-tag (:! first-event) first-event (if (and has-code-mode-shebang @@ -56,6 +57,26 @@ events (cons first-event rest-events)] (remove nil? events))) +(declare snake-event) + +;; TODO - Set bigger buffer size in scanner class +(defn parse-snakeyaml [yaml-string] + (let [parser (new Parse (.build (LoadSettings/builder)))] + (->> yaml-string + (.parseString parser) + (map snake-event) + (remove nil?) + rest))) + +(defn parse-rapidyaml [yaml-string] + (let [rapid-parser (new Rapidyaml)] + (->> yaml-string + ( #(.parseYS ^Rapidyaml rapid-parser %1)) + #_(#(do (println %1) %1)) + (#(str/replace %1 #"^\(\n\{:\+\ \"\+DOC\"\}" "(")) + #_(#(do (println %1) %1)) + read-string))) + (defn parse-test-case [yaml-string] (->> yaml-string parse @@ -132,16 +153,16 @@ (let [obj (event-obj event)] (assoc obj :* (str (. event getAlias))))) -(defmulti ys-event class) -(defmethod ys-event DocumentStartEvent [event] (doc-start event)) -(defmethod ys-event DocumentEndEvent [event] (doc-end event)) -(defmethod ys-event MappingStartEvent [event] (map-start event)) -(defmethod ys-event MappingEndEvent [event] (map-end event)) -(defmethod ys-event SequenceStartEvent [event] (seq-start event)) -(defmethod ys-event SequenceEndEvent [event] (seq-end event)) -(defmethod ys-event ScalarEvent [event] (scalar-val event)) -(defmethod ys-event AliasEvent [event] (alias-val event)) -(defmethod ys-event :default [_] nil) +(defmulti snake-event class) +(defmethod snake-event DocumentStartEvent [event] (doc-start event)) +(defmethod snake-event DocumentEndEvent [event] (doc-end event)) +(defmethod snake-event MappingStartEvent [event] (map-start event)) +(defmethod snake-event MappingEndEvent [event] (map-end event)) +(defmethod snake-event SequenceStartEvent [event] (seq-start event)) +(defmethod snake-event SequenceEndEvent [event] (seq-end event)) +(defmethod snake-event ScalarEvent [event] (scalar-val event)) +(defmethod snake-event AliasEvent [event] (alias-val event)) +(defmethod snake-event :default [_] nil) (comment ) diff --git a/libyamlscript/Makefile b/libyamlscript/Makefile index 46e66453e..34c577ca4 100644 --- a/libyamlscript/Makefile +++ b/libyamlscript/Makefile @@ -8,7 +8,11 @@ include $(COMMON)/vars-libys.mk #------------------------------------------------------------------------------ build:: $(LIBYS_DEPS) +<<<<<<< HEAD # jar: $(LIBYS_JAR_PATH) +======= +jar: $(LIBYAMLSCRIPT_JAR_PATH) $(RAPIDYAML_INSTALLED) +>>>>>>> 8a8f78d0 (core: Add rapidyaml parser support) install:: $(LIBYS_DEPS) mkdir -p $(PREFIX)/include/ diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile new file mode 100644 index 000000000..5d455f5c6 --- /dev/null +++ b/rapidyaml/Makefile @@ -0,0 +1,67 @@ +include ../common/base.mk +include $(COMMON)/clojure.mk +include $(COMMON)/java.mk +include ./rapidyaml.mk + +RAPIDYAML_JAR_DEPS := \ + $(wildcard \ + build \ + $(JAVA_INSTALLED) \ + Makefile \ + pom.xml \ + ) + +RAPIDYAML_CLASS := \ + src/main/java/org/rapidyaml/Rapidyaml.class \ + src/main/java/org/rapidyaml/YamlParseErrorException.class + + +#------------------------------------------------------------------------------ +default:: + echo $(RAPIDYAML_SO) + +x: + YS -ce 'foo: var' + +build:: $(RAPIDYAML_SO) + @: + +jar: $(RAPIDYAML_JAR) + +install:: $(RAPIDYAML_INSTALLED) + +test:: build $(JAVA_INSTALLED) + mvn $@ + +test-x: build $(JAVA_INSTALLED) + mvn -X -e test + +clean:: + $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) + $(RM) -r reports/ target/ + $(MAKE) -C native $@ + +realclean:: clean + $(MAKE) -C native $@ + +sysclean:: + $(RM) -r $(MAVEN_REPOSITORY)/org/rapidyaml + + +#------------------------------------------------------------------------------ +$(RAPIDYAML_SO): + $(MAKE) -C native $@ + +$(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED) + mvn install + +$(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED) + jar -cvf $@ $< + +$(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) + # this doesn't work: + #javac $< + # ... but this does: + javac $(RAPIDYAML_JAVA) + +crl: $(RAPIDYAML_CLASS) diff --git a/rapidyaml/native/.gitignore b/rapidyaml/native/.gitignore new file mode 100644 index 000000000..226243a92 --- /dev/null +++ b/rapidyaml/native/.gitignore @@ -0,0 +1,4 @@ +/rapidyaml/ +/rapidyaml-build/ +/rapidyaml_all.hpp +/librapidyaml.* diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt new file mode 100644 index 000000000..2bdd95182 --- /dev/null +++ b/rapidyaml/native/CMakeLists.txt @@ -0,0 +1,40 @@ +cmake_minimum_required(VERSION 3.12) +project(rapidyaml + DESCRIPTION "rapidyaml -> yamlscript" + HOMEPAGE_URL "https://github.com/biojppm/rapidyaml -> https://github.com/yaml/yamlscript" + LANGUAGES CXX) + +find_package(JNI REQUIRED) + +option(YS2EDN_TIMED "add timings to sections" OFF) + +if(UNIX) + set(CMAKE_SHARED_LIBRARY_SUFFIX .so) +endif() + +add_library(rapidyaml + org_rapidyaml_Rapidyaml.h + org_rapidyaml_Rapidyaml.cpp + rapidyaml_all.hpp + rapidyaml_edn_handler.hpp + rapidyaml_edn_handler.cpp + rapidyaml_edn.hpp + rapidyaml_edn.cpp +) +target_include_directories(rapidyaml PUBLIC + ${CMAKE_CURRENT_LIST_DIR} + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/test +) +target_compile_definitions(rapidyaml PUBLIC + RYML_WITH_TAB_TOKENS + RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS + RYML_SINGLE_HEADER + $<$:YS2EDN_TIMED> +) +set_target_properties(rapidyaml PROPERTIES CXX_STANDARD 17) + +# target_link_libraries(rapidyaml PUBLIC JNI::JNI) +target_include_directories(rapidyaml PUBLIC ${JNI_INCLUDE_DIRS}) + +add_executable(rapidyaml-test rapidyaml_test.cpp) +target_link_libraries(rapidyaml-test rapidyaml) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile new file mode 100644 index 000000000..876509c10 --- /dev/null +++ b/rapidyaml/native/Makefile @@ -0,0 +1,71 @@ +include ../../common/base.mk +include $(COMMON)/clojure.mk +include $(COMMON)/java.mk +include $(COMMON)/python.mk +include ../rapidyaml.mk + +RAPIDYAML_H := org_rapidyaml_Rapidyaml.h +RAPIDYAML_HPP := rapidyaml_all.hpp +RAPIDYAML_AMALGAMATE_OPTS := --stl + +RAPIDYAML_SO_DEPS := \ + Makefile \ + CMakeLists.txt \ + $(JAVA_HOME) \ + $(RAPIDYAML_H) \ + $(RAPIDYAML_HPP) \ + $(wildcard ./*pp) \ + +# TODO maybe change to static library! +# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni +SHARED := 1 +TIMED := 0 + +CMK_FLAGS := +CMK_FLAGS += -D YS2EDN_TIMED=$(TIMED) +CMK_FLAGS += -D BUILD_SHARED_LIBS=$(SHARED) + + +#------------------------------------------------------------------------------ +default:: + +build:: $(RAPIDYAML_SO) + +test:: build + cmake --build rapidyaml-build \ + --parallel --target rapidyaml-test + ./rapidyaml-build/rapidyaml-test + +rapidyaml: + mkdir -p $@ + git -C $@ init -q . + git -C $@ remote add origin $(RAPIDYAML_REPO) + git -C $@ fetch origin $(RAPIDYAML_TAG) + git -C $@ reset --hard FETCH_HEAD + git -C $@ submodule update --init --recursive + +clean:: + $(RM) librapidyaml.* + $(RM) $(RAPIDYAML_HPP) + $(RM) -r rapidyaml-build + $(RM) -r rapidyaml-install + +realclean:: clean + $(RM) -r rapidyaml + + +#------------------------------------------------------------------------------ +$(RAPIDYAML_SO): $(RAPIDYAML_SO_DEPS) + cmake -S . -B rapidyaml-build \ + $(CMK_FLAGS) + cmake --build rapidyaml-build \ + --parallel --verbose --target rapidyaml + mv rapidyaml-build/*.so $@ + ln -fs $@ librapidyaml.so + +$(RAPIDYAML_H): $(RAPIDYAML_JAVA) + javac -h . $^ + +$(RAPIDYAML_HPP): rapidyaml + $(PYTHON) rapidyaml/tools/amalgamate.py \ + $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp new file mode 100644 index 000000000..4e57b87d8 --- /dev/null +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -0,0 +1,115 @@ +#include +#include "./rapidyaml_edn.hpp" +#include + +#ifndef _Included_org_rapidyaml_Rapidyaml +#define _Included_org_rapidyaml_Rapidyaml +#ifdef __cplusplus +extern "C" { +#endif + +void throw_java_exception(JNIEnv * env, + const char* type, + const char* msg) +{ + jclass clazz = env->FindClass(type); + if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown + env->ThrowNew(clazz, msg); +} + +void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg) +{ + // see https://stackoverflow.com/questions/55013243/jni-custom-exceptions-with-more-than-one-parameter + jclass clazz = env->FindClass("org/rapidyaml/YamlParseErrorException"); + if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown + { + jstring jmsg = env->NewStringUTF(msg); + jint joffset = (jint)offset; + jint jline = (jint)line; + jint jcol = (jint)column; + // see https://www.rgagnon.com/javadetails/java-0286.html + // about the proper signature. + // we want (int, int, int, String): + const char * const signature = "(IIILjava/lang/String;)V"; + jmethodID ctor = env->GetMethodID(clazz, "", signature); + jobject jexc = env->NewObject(clazz, ctor, joffset, jline, jcol, jmsg); + env->Throw((jthrowable)jexc); // https://stackoverflow.com/questions/2455668/jni-cast-between-jobect-and-jthrowable + } +} + + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_init + * Signature: ()Ljava/lang/Object; + */ +JNIEXPORT jlong JNICALL +Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) +{ + Ryml2Edn *obj = ys2edn_init(); + return (jlong)obj; +} + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_destroy + * Signature: (Ljava/lang/Object;)V + */ +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) +{ + ys2edn_destroy((Ryml2Edn*)obj); +} + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_parse + * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I + */ +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jbyteArray src, jint src_len, + jbyteArray dst, jint dst_len) +{ + jboolean src_is_copy, dst_is_copy; + jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); + jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); + const char *filename = env->GetStringUTFChars(jfilename, 0); + int rc = 0; + try + { + rc = ys2edn_parse((Ryml2Edn*)obj, filename, + (char*)src_, src_len, + (char*)dst_, dst_len); + } + catch (Ryml2EdnParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + env->ReleaseByteArrayElements(src, src_, 0); + env->ReleaseByteArrayElements(dst, dst_, 0); + env->ReleaseStringUTFChars(jfilename, filename); + return rc; +} + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_retry_get + * Signature: (Ljava/lang/Object;[BI)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, + jlong obj, + jbyteArray dst, jint dst_len) +{ + jboolean src_is_copy, dst_is_copy; + jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); + int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); + env->ReleaseByteArrayElements(dst, dst_, 0); + return rc; +} + +#ifdef __cplusplus +} +#endif +#endif diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h new file mode 100644 index 000000000..c73497636 --- /dev/null +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_rapidyaml_Rapidyaml */ + +#ifndef _Included_org_rapidyaml_Rapidyaml +#define _Included_org_rapidyaml_Rapidyaml +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init + (JNIEnv *, jobject); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy + (JNIEnv *, jobject, jlong); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_parse + * Signature: (JLjava/lang/String;[BI[BI)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_retry_get + * Signature: (J[BI)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/rapidyaml/native/rapidyaml_edn.cpp b/rapidyaml/native/rapidyaml_edn.cpp new file mode 100644 index 000000000..8f5bcfca9 --- /dev/null +++ b/rapidyaml/native/rapidyaml_edn.cpp @@ -0,0 +1,108 @@ +#define RYML_SINGLE_HDR_DEFINE_NOW +#include +#include "rapidyaml_edn.hpp" +#include + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +using namespace ryml; + +#ifndef YS2EDN_TIMED +#define TIMED_SECTION(name) +#else +#include +#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) +struct timed_section +{ + using myclock = std::chrono::steady_clock; + using fmsecs = std::chrono::duration; + csubstr name; + myclock::time_point start; + fmsecs since() const { return myclock::now() - start; } + timed_section(csubstr n) : name(n), start(myclock::now()) {} + ~timed_section() + { + fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); + } +}; +#endif + + +#if defined(__cplusplus) +extern "C" { +#endif +// see +// https://stackoverflow.com/questions/230689/best-way-to-throw-exceptions-in-jni-code +// https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method + +namespace { +C4_NORETURN void ys2edn_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) +{ + Ryml2EdnParseError exc; + exc.location = location; + exc.msg.assign(msg, msg_len); + throw exc; +} +} // anon namespace + +RYML_EXPORT Ryml2Edn *ys2edn_init() +{ + TIMED_SECTION("ys2edn_init"); + Callbacks cb = {}; + cb.m_error = &ys2edn_parse_error; + set_callbacks(cb); + Ryml2Edn *ryml2edn = _RYML_CB_ALLOC(get_callbacks(), Ryml2Edn, 1); + _RYML_CB_CHECK(get_callbacks(), ryml2edn != nullptr); + new ((void*)ryml2edn) Ryml2Edn(); + return ryml2edn; +} + +RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn) +{ + TIMED_SECTION("ys2edn_destroy"); + ryml2edn->~Ryml2Edn(); +} + +RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, + const char *filename, + char *ys, size_type ys_size, + char *edn, size_type edn_size) +{ + TIMED_SECTION("ys2edn_parse"); + csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; + substr ys_(ys, (size_t)ys_size); + { + TIMED_SECTION("reset"); + ryml2edn->reset(); + } + { + TIMED_SECTION("parse_in_place"); + ryml2edn->m_parser.parse_in_place_ev(filename_, ys_); + } + return ys2edn_retry_get(ryml2edn, edn, edn_size); +} + +RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, + char *edn, size_type edn_size) +{ + TIMED_SECTION("ys2edn_retry_get"); + csubstr result = to_csubstr(ryml2edn->m_sink.result); + size_type required_size = 1 + (size_type)result.len; + if(required_size <= edn_size) + { + memcpy(edn, result.str, (size_t)result.len); + edn[result.len] = '\0'; + } + else if(edn_size > 0) + { + edn[0] = '\0'; + } + return required_size; +} + +#if defined(__cplusplus) +} +#endif diff --git a/rapidyaml/native/rapidyaml_edn.hpp b/rapidyaml/native/rapidyaml_edn.hpp new file mode 100644 index 000000000..69ca2f077 --- /dev/null +++ b/rapidyaml/native/rapidyaml_edn.hpp @@ -0,0 +1,71 @@ +#pragma once +#ifndef RAPIDYAML_EVENTS_H +#define RAPIDYAML_EVENTS_H + +#include +#include +#include "rapidyaml_edn_handler.hpp" + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +#if defined(__cplusplus) +extern "C" { +#endif + +using size_type = int; + +struct RYML_EXPORT Ryml2Edn +{ + c4::yml::EventHandlerEdn::EventSink m_sink; + c4::yml::EventHandlerEdn m_handler; + c4::yml::ParseEngine m_parser; + Ryml2Edn() + : m_sink() + , m_handler(&m_sink) + , m_parser(&m_handler) + { + } + void reset() + { + m_sink.reset(); + m_handler.reset(); + } +}; + +struct RYML_EXPORT Ryml2EdnParseError : public std::exception +{ + c4::yml::Location location; + std::string msg; + const char* what() const noexcept override { return msg.c_str(); } +}; + + +//----------------------------------------------------------------------------- + +/** Initialize the resources */ +RYML_EXPORT Ryml2Edn *ys2edn_init(); + +/** Destroy the resources */ +RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn); + +/** Parse YAML, and return corresponding EDN. Return the number of + * characters needed for edn. Check if the returned size is larger + * than edn_size. If it is, call ys_retry_get() can be called + * afterwards to extract the EDN. */ +RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, + const char *filename, + char *ys, size_type ys_size, + char *edn, size_type edn_size); + +/** Get the edn from the previous call to ys2edn_parse(). */ +RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, + char *edn, size_type edn_size); + +#if defined(__cplusplus) +} +#endif + +#endif /* RAPIDYAML_EVENTS_H */ diff --git a/rapidyaml/native/rapidyaml_edn_handler.cpp b/rapidyaml/native/rapidyaml_edn_handler.cpp new file mode 100644 index 000000000..635f44601 --- /dev/null +++ b/rapidyaml/native/rapidyaml_edn_handler.cpp @@ -0,0 +1,83 @@ +#ifndef RYML_SINGLE_HEADER +#include +#include +#include +#endif +#include "./rapidyaml_edn_handler.hpp" + + +namespace c4 { +namespace yml { + +// instantiate the template +template class ParseEngine; + + +void EventHandlerEdn::EventSink::append_escaped(csubstr val) +{ + #define _c4flush_use_instead(repl, skip) \ + do { \ + this->append(val.range(prev, i)); \ + this->append(repl); \ + prev = i + skip; \ + } \ + while(0) + uint8_t const* C4_RESTRICT s = reinterpret_cast(val.str); + size_t prev = 0; + for(size_t i = 0; i < val.len; ++i) + { + switch(s[i]) + { + case UINT8_C(0x0a): // \n + _c4flush_use_instead("\\n", 1); break; + case UINT8_C(0x5c): // '\\' + _c4flush_use_instead("\\\\", 1); break; + case UINT8_C(0x22): // \" + _c4flush_use_instead("\\\"", 1); break; + case UINT8_C(0x09): // \t + _c4flush_use_instead("\\t", 1); break; + case UINT8_C(0x0d): // \r + _c4flush_use_instead("\\r", 1); break; + case UINT8_C(0x00): // \0 + _c4flush_use_instead("\\0", 1); break; + case UINT8_C(0x0c): // \f (form feed) + _c4flush_use_instead("\\f", 1); break; + case UINT8_C(0x08): // \b (backspace) + _c4flush_use_instead("\\b", 1); break; + case UINT8_C(0x07): // \a (bell) + _c4flush_use_instead("\\a", 1); break; + case UINT8_C(0x0b): // \v (vertical tab) + _c4flush_use_instead("\\v", 1); break; + case UINT8_C(0x1b): // \e (escape) + _c4flush_use_instead("\\e", 1); break; + case UINT8_C(0xc2): + if(i+1 < val.len) + { + const uint8_t np1 = s[i+1]; + if(np1 == UINT8_C(0xa0)) + _c4flush_use_instead("\\_", 2); + else if(np1 == UINT8_C(0x85)) + _c4flush_use_instead("\\N", 2); + } + break; + case UINT8_C(0xe2): + if(i+2 < val.len) + { + if(s[i+1] == UINT8_C(0x80)) + { + if(s[i+2] == UINT8_C(0xa8)) + _c4flush_use_instead("\\L", 3); + else if(s[i+2] == UINT8_C(0xa9)) + _c4flush_use_instead("\\P", 3); + } + } + break; + } + } + // flush the rest + this->append(val.sub(prev)); + #undef _c4flush_use_instead +} + +} // namespace yml +} // namespace c4 diff --git a/rapidyaml/native/rapidyaml_edn_handler.hpp b/rapidyaml/native/rapidyaml_edn_handler.hpp new file mode 100644 index 000000000..973c52f57 --- /dev/null +++ b/rapidyaml/native/rapidyaml_edn_handler.hpp @@ -0,0 +1,673 @@ +#ifndef _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ +#define _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ + +#ifdef RYML_SINGLE_HEADER +#include +#else +#ifndef _C4_YML_EVENT_HANDLER_STACK_HPP_ +#include "c4/yml/event_handler_stack.hpp" +#endif +#ifndef _C4_YML_STD_STRING_HPP_ +#include "c4/yml/std/string.hpp" +#endif +#ifndef _C4_YML_DETAIL_PRINT_HPP_ +#include "c4/yml/detail/print.hpp" +#endif +#endif + +#include + +C4_SUPPRESS_WARNING_GCC_CLANG_PUSH +C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") +C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") + + +namespace c4 { +namespace yml { + + +/** @addtogroup doc_event_handlers + * @{ */ + + +struct EventHandlerEdnState : public ParserState +{ + NodeData ev_data; +}; + + +struct EventHandlerEdn : public EventHandlerStack +{ + + /** @name types + * @{ */ + + // our internal state must inherit from parser state + using state = EventHandlerEdnState; + + struct EventSink + { + std::string result; + void reset() noexcept { result.clear(); } + void append(csubstr s) noexcept { result.append(s.str, s.len); } + void append(char c) noexcept { result += c; } + void insert(csubstr s, size_t pos) noexcept { result.insert(pos, s.str, s.len); } + void insert(char c, size_t pos) noexcept { result.insert(pos, 1, c); } + csubstr get() const { return csubstr(&result[0], result.size()); } + substr get() { return substr(&result[0], result.size()); } + size_t find_last(csubstr s) const { return result.rfind(s.str, std::string::npos, s.len); } + void append_escaped(csubstr val); + }; + + /** @} */ + +public: + + /** @cond dev */ + EventSink *C4_RESTRICT m_sink; + std::vector m_val_buffers; + char m_key_tag_buf[256]; + char m_val_tag_buf[256]; + std::string m_arena; + bool m_first_doc; + + // undefined at the end + #define _enable_(bits) _enable__() + #define _disable_(bits) _disable__() + #define _has_any_(bits) _has_any__() + /** @endcond */ + +public: + + /** @name construction and resetting + * @{ */ + + EventHandlerEdn() : EventHandlerStack(), m_sink(), m_val_buffers(), m_first_doc() {} + EventHandlerEdn(Callbacks const& cb) : EventHandlerStack(cb), m_sink(), m_val_buffers(), m_first_doc() {} + EventHandlerEdn(EventSink *sink, Callbacks const& cb) : EventHandlerStack(cb), m_sink(sink), m_val_buffers(), m_first_doc() + { + reset(); + } + EventHandlerEdn(EventSink *sink) : EventHandlerEdn(sink, get_callbacks()) {} + + void reset() + { + _stack_reset_root(); + m_curr->flags |= RUNK|RTOP; + m_val_buffers.resize((size_t)m_stack.size()); + m_arena.clear(); + m_first_doc = true; + } + + /** @} */ + +public: + + /** @name parse events + * @{ */ + + void start_parse(const char* filename, detail::pfn_relocate_arena relocate_arena, void *relocate_arena_data) + { + this->_stack_start_parse(filename, relocate_arena, relocate_arena_data); + } + + void finish_parse() + { + this->_stack_finish_parse(); + } + + void cancel_parse() + { + while(m_stack.size() > 1) + _pop(); + _buf_flush_(); + } + + /** @} */ + +public: + + /** @name YAML stream events */ + /** @{ */ + + void begin_stream() + { + _send_("(\n"); + } + + void end_stream() + { + _send_(")\n"); + _buf_flush_(); + } + + /** @} */ + +public: + + /** @name YAML document events */ + /** @{ */ + + /** implicit doc start (without ---) */ + void begin_doc() + { + _c4dbgp("begin_doc"); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + _enable_(DOC); + } + m_first_doc = false; + } + /** implicit doc end (without ...) */ + void end_doc() + { + _c4dbgp("end_doc"); + _send_("{:+ \"-DOC\"}\n"); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** explicit doc start, with --- */ + void begin_doc_expl() + { + _c4dbgp("begin_doc_expl"); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + } + if (m_first_doc) + m_first_doc = false; + else + _send_("{:+ \"+DOC\"}\n"); + _enable_(DOC); + } + /** explicit doc end, with ... */ + void end_doc_expl() + { + _c4dbgp("end_doc_expl"); + _send_("{:+ \"-DOC\"}\n"); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** @} */ + +public: + + /** @name YAML map functions */ + /** @{ */ + + void begin_map_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_map_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_map_val_flow() + { + _send_("{:+ \"+MAP\""); + _send_val_props_(); + _send_(", :flow true}\n"); + _mark_parent_with_children_(); + _enable_(MAP|FLOW_SL); + _push(); + } + void begin_map_val_block() + { + _send_("{:+ \"+MAP\""); + _send_val_props_(); + _send_("}\n"); + _mark_parent_with_children_(); + _enable_(MAP|BLOCK); + _push(); + } + + void end_map() + { + _pop(); + _send_("{:+ \"-MAP\"}\n"); + } + + /** @} */ + +public: + + /** @name YAML seq events */ + /** @{ */ + + void begin_seq_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_seq_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_seq_val_flow() + { + _send_("{:+ \"+SEQ\""); + _send_val_props_(); + _send_(", :flow true}\n"); + _mark_parent_with_children_(); + _enable_(SEQ|FLOW_SL); + _push(); + } + void begin_seq_val_block() + { + _send_("{:+ \"+SEQ\""); + _send_val_props_(); + _send_("}\n"); + _mark_parent_with_children_(); + _enable_(SEQ|BLOCK); + _push(); + } + + void end_seq() + { + _pop(); + _send_("{:+ \"-SEQ\"}\n"); + } + + /** @} */ + +public: + + /** @name YAML structure events */ + /** @{ */ + + void add_sibling() + { + _RYML_CB_ASSERT(m_stack.m_callbacks, m_parent); + _buf_flush_to_(m_curr->level, m_parent->level); + m_curr->ev_data = {}; + } + + /** set the previous val as the first key of a new map, with flow style. + * + * See the documentation for @ref doc_event_handlers, which has + * important notes about this event. + */ + void actually_val_is_first_key_of_new_map_flow() + { + // ensure we have a temporary buffer to save the current val + const id_type tmp = m_curr->level + id_type(2); + _buf_ensure_(tmp + id_type(2)); + // save the current val to the temporary buffer + _buf_flush_to_(m_curr->level, tmp); + // create the map. + // this will push a new level, and tmp is one further + begin_map_val_flow(); + _RYML_CB_ASSERT(m_stack.m_callbacks, tmp != m_curr->level); + // now move the saved val as the first key + _buf_flush_to_(tmp, m_curr->level); + } + + void actually_val_is_first_key_of_new_map_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + /** @} */ + +public: + + /** @name YAML scalar events */ + /** @{ */ + + + C4_ALWAYS_INLINE void set_key_scalar_plain(csubstr scalar) + { + _send_key_scalar_(scalar, '='); + _enable_(KEY|KEY_PLAIN); + } + C4_ALWAYS_INLINE void set_val_scalar_plain(csubstr scalar) + { + _send_val_scalar_(scalar, '='); + _enable_(VAL|VAL_PLAIN); + } + + + C4_ALWAYS_INLINE void set_key_scalar_dquoted(csubstr scalar) + { + _send_key_scalar_(scalar, '$'); + _enable_(KEY|KEY_DQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_dquoted(csubstr scalar) + { + _send_val_scalar_(scalar, '$'); + _enable_(VAL|VAL_DQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_squoted(csubstr scalar) + { + _send_key_scalar_(scalar, '\''); + _enable_(KEY|KEY_SQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_squoted(csubstr scalar) + { + _send_val_scalar_(scalar, '\''); + _enable_(VAL|VAL_SQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_literal(csubstr scalar) + { + _send_key_scalar_(scalar, '|'); + _enable_(KEY|KEY_LITERAL); + } + C4_ALWAYS_INLINE void set_val_scalar_literal(csubstr scalar) + { + _send_val_scalar_(scalar, '|'); + _enable_(VAL|VAL_LITERAL); + } + + + C4_ALWAYS_INLINE void set_key_scalar_folded(csubstr scalar) + { + _send_key_scalar_(scalar, '>'); + _enable_(KEY|KEY_FOLDED); + } + C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) + { + _send_val_scalar_(scalar, '>'); + _enable_(VAL|VAL_FOLDED); + } + + + C4_ALWAYS_INLINE void mark_key_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + C4_ALWAYS_INLINE void mark_val_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + + /** @} */ + +public: + + /** @name YAML anchor/reference events */ + /** @{ */ + + void set_key_anchor(csubstr anchor) + { + _enable_(KEYANCH); + m_curr->ev_data.m_key.anchor = anchor; + } + void set_val_anchor(csubstr anchor) + { + _enable_(VALANCH); + m_curr->ev_data.m_val.anchor = anchor; + } + + void set_key_ref(csubstr ref) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); + _enable_(KEY|KEYREF); + _send_("{:+ \"=ALI\" :* \""); + _send_(ref.sub(1)); + _send_("\"}\n"); + } + void set_val_ref(csubstr ref) + { + _enable_(VAL|VALREF); + _send_("{:+ \"=ALI\" :* \""); + _send_(ref.sub(1)); + _send_("\"}\n"); + } + + /** @} */ + +public: + + /** @name YAML tag events */ + /** @{ */ + + void set_key_tag(csubstr tag) + { + _enable_(KEYTAG); + m_curr->ev_data.m_key.tag = _transform_directive(tag, m_key_tag_buf); + } + void set_val_tag(csubstr tag) + { + _enable_(VALTAG); + m_curr->ev_data.m_val.tag = _transform_directive(tag, m_val_tag_buf); + } + + /** @} */ + +public: + + /** @name YAML directive events */ + /** @{ */ + + void add_directive(csubstr directive) + { + _RYML_CB_ERR(m_stack.m_callbacks, "tag directives not supported"); + } + + /** @} */ + +public: + + /** @name YAML arena events */ + /** @{ */ + + substr alloc_arena(size_t len) + { + const size_t sz = m_arena.size(); + csubstr prev = to_csubstr(m_arena); + m_arena.resize(sz + len); + substr out = to_substr(m_arena).sub(sz); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + _stack_relocate_to_new_arena(prev, curr); + return out; + } + + substr alloc_arena(size_t len, substr *relocated) + { + csubstr prev = to_csubstr(m_arena); + if(!prev.is_super(*relocated)) + return alloc_arena(len); + substr out = alloc_arena(len); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); + return out; + } + + /** @} */ + +public: + + /** @cond dev */ + + /** push a new parent, add a child to the new parent, and set the + * child as the current node */ + void _push() + { + _stack_push(); + _buf_ensure_(m_stack.size() + id_type(1)); + _buf_().reset(); + m_curr->ev_data = {}; + } + + /** end the current scope */ + void _pop() + { + _buf_flush_to_(m_curr->level, m_parent->level); + _stack_pop(); + } + + template C4_ALWAYS_INLINE void _enable__() noexcept + { + m_curr->ev_data.m_type.type = static_cast(m_curr->ev_data.m_type.type | bits); + } + template C4_ALWAYS_INLINE void _disable__() noexcept + { + m_curr->ev_data.m_type.type = static_cast(m_curr->ev_data.m_type.type & (~bits)); + } + template C4_ALWAYS_INLINE bool _has_any__() const noexcept + { + return (m_curr->ev_data.m_type.type & bits) != 0; + } + + void _mark_parent_with_children_() + { + if(m_parent) + m_parent->has_children = true; + } + + EventSink& _buf_() noexcept + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)m_curr->level < m_val_buffers.size()); + return m_val_buffers[(size_t)m_curr->level]; + } + + EventSink& _buf_(id_type level) noexcept + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)level < m_val_buffers.size()); + return m_val_buffers[(size_t)level]; + } + + EventSink const& _buf_(id_type level) const noexcept + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)level < m_val_buffers.size()); + return m_val_buffers[(size_t)level]; + } + + static void _buf_flush_to_(EventSink &C4_RESTRICT src, EventSink &C4_RESTRICT dst) noexcept + { + dst.append(src.get()); + src.reset(); + } + + void _buf_flush_to_(id_type level_src, id_type level_dst) noexcept + { + auto &src = _buf_(level_src); + auto &dst = _buf_(level_dst); + _buf_flush_to_(src, dst); + } + + void _buf_flush_() noexcept + { + _buf_flush_to_(_buf_(), *m_sink); + } + + void _buf_ensure_(id_type size_needed) noexcept + { + if((size_t)size_needed > m_val_buffers.size()) + m_val_buffers.resize((size_t)size_needed); + } + + C4_ALWAYS_INLINE void _send_(csubstr s) noexcept { _buf_().append(s); } + C4_ALWAYS_INLINE void _send_(char c) noexcept { _buf_().append(c); } + + void _send_key_scalar_(csubstr scalar, char scalar_type_code) + { + _send_("{:+ \"=VAL\""); + _send_key_props_(); + _send_(", :"); + _send_(scalar_type_code); + _send_(" \""); + _buf_().append_escaped(scalar); + _send_("\"}\n"); + } + void _send_val_scalar_(csubstr scalar, char scalar_type_code) + { + _send_("{:+ \"=VAL\""); + _send_val_props_(); + _send_(", :"); + _send_(scalar_type_code); + _send_(" \""); + _buf_().append_escaped(scalar); + _send_("\"}\n"); + } + + void _send_key_props_() + { + if(_has_any_(KEYANCH|KEYREF)) + { + _send_(", :& \""); + _send_(m_curr->ev_data.m_key.anchor); + _send_('\"'); + } + if(_has_any_(KEYTAG)) + { + _send_(", :! \""); + _send_tag_(m_curr->ev_data.m_key.tag); + _send_('\"'); + } + m_curr->ev_data.m_key = {}; + _disable_(KEYANCH|KEYREF|KEYTAG); + } + void _send_val_props_() + { + if(_has_any_(VALANCH|VALREF)) + { + _send_(", :& \""); + _send_(m_curr->ev_data.m_val.anchor); + _send_('\"'); + } + if(m_curr->ev_data.m_type.type & VALTAG) + { + _send_(", :! \""); + _send_tag_(m_curr->ev_data.m_val.tag); + _send_('\"'); + } + m_curr->ev_data.m_val = {}; + _disable_(VALANCH|VALREF|VALTAG); + } + void _send_tag_(csubstr tag) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, !tag.empty()); + if(tag.begins_with('!')) + tag = tag.sub(1); + _send_(tag); + } + + csubstr _transform_directive(csubstr tag, substr output) + { + if(tag.begins_with('!')) + { + if(is_custom_tag(tag)) + { + _RYML_CB_ERR_(m_stack.m_callbacks, "tag not found", m_curr->pos); + } + } + csubstr result = normalize_tag_long(tag, output); + _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); + _RYML_CB_CHECK(m_stack.m_callbacks, result.str); + return result; + } +#undef _enable_ +#undef _disable_ +#undef _has_any_ + + /** @endcond */ +}; + +/** @} */ + +} // namespace yml +} // namespace c4 + +C4_SUPPRESS_WARNING_GCC_POP + +#endif /* _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ */ diff --git a/rapidyaml/native/rapidyaml_test.cpp b/rapidyaml/native/rapidyaml_test.cpp new file mode 100644 index 000000000..7c2e15f72 --- /dev/null +++ b/rapidyaml/native/rapidyaml_test.cpp @@ -0,0 +1,305 @@ +#include + +using c4::csubstr; +using c4::substr; + + +struct Ys2EdnScoped +{ + Ryml2Edn *ryml2edn; + Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} + ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } +}; + + +struct ResultScoped +{ + char *edn = nullptr; + ~ResultScoped() { if(edn) ys2edn_free(edn); } +}; + +struct TestResult +{ + uint32_t num_assertions; + uint32_t num_tests; + uint32_t num_failed_assertions; + uint32_t num_failed_tests; + operator bool() const { return num_failed_tests == 0; } + void add(TestResult const& that) + { + num_tests += 1 + that.num_tests; + num_assertions += that.num_assertions; + num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; + num_failed_assertions += that.num_failed_assertions; + } +}; + +struct TestCase +{ + csubstr ys; + csubstr edn; + bool testeq(csubstr actual) const + { + const bool status = (actual == edn); + if(!status) + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n" + "expected:~~~%.*s~~~\n" + "actual:~~~%.*s~~~\n", + (int)ys.len, ys.str, + (int)edn.len, edn.str, + (int)actual.len, actual.str); + return status; + } + + #define _runtest(name, ...) \ + do { \ + printf("[RUN ] %s ... \n", #name); \ + TestResult tr_ = name(__VA_ARGS__); \ + tr.add(tr_); \ + printf("[%s] %s\n", tr_?"OK ":"FAIL", #name); \ + } while(0) + #define CHECK(cond) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s", __FILE__, __LINE__, #cond); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + + TestResult test(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + _runtest(test_1_alloc, ); + _runtest(test_2_large_enough, ); + _runtest(test_3_too_small, ); + _runtest(test_4_nullptr, ); + _runtest(test_1_alloc_reuse, ryml2edn); + _runtest(test_2_large_enough_reuse, ryml2edn); + _runtest(test_3_too_small_reuse, ryml2edn); + _runtest(test_4_nullptr_reuse, ryml2edn); + return tr; + } + + // 1. simplest (but wasteful) first: alloc a new string + TestResult test_1_alloc_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + ResultScoped result; // frees when destroying + result.edn = ys2edn_alloc(ryml2edn, "ysfilename", + input.str, (size_type)input.len); + CHECK(testeq(c4::to_csubstr(result.edn))); + return tr; + } + TestResult test_1_alloc() const + { + Ys2EdnScoped lib; + return test_1_alloc_reuse(lib.ryml2edn); + } + + // 2. happy path: large-enough destination string + TestResult test_2_large_enough_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output; + output.resize(2 * edn.len); + size_type reqsize = ys2edn(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_2_large_enough() const + { + Ys2EdnScoped lib; + return test_2_large_enough_reuse(lib.ryml2edn); + } + + // less-happy path: destination string not large enough + TestResult test_3_too_small_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output = "?"; + size_type reqsize = ys2edn(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + CHECK(output != "?"); + output.resize(reqsize); + size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); + CHECK(getsize == reqsize); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_3_too_small() const + { + Ys2EdnScoped lib; + return test_3_too_small_reuse(lib.ryml2edn); + } + + // safe calling with nullptr + TestResult test_4_nullptr_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + size_type reqsize = ys2edn(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + return tr; + } + TestResult test_4_nullptr() const + { + Ys2EdnScoped lib; + return test_4_nullptr_reuse(lib.ryml2edn); + } +}; + + +const TestCase test_cases[] = { + // case ------------------------------ + {"say: 2 + 2", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "say"} +{:+ "=VAL", := "2 + 2"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {"a: 1", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "1"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {"𝄞: ✅", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "𝄞"} +{:+ "=VAL", := "✅"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {"[a, b, c]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "=VAL", := "c"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {"[a: b]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "-MAP"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)"}, + // case ------------------------------ + {R"(--- !yamlscript/v0 +foo: ! +- {x: y} +- [x, y] +- foo +- 'foo' +- "foo" +- | + foo +- > + foo +- [1, 2, true, false, null] +- &anchor-1 !tag-1 foobar +--- +another: doc +)", + R"(( +{:+ "+MAP", :! "yamlscript/v0"} +{:+ "=VAL", := "foo"} +{:+ "+SEQ", :! ""} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-MAP"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-SEQ"} +{:+ "=VAL", := "foo"} +{:+ "=VAL", :' "foo"} +{:+ "=VAL", :$ "foo"} +{:+ "=VAL", :| "foo\n"} +{:+ "=VAL", :> "foo\n"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "1"} +{:+ "=VAL", := "2"} +{:+ "=VAL", := "true"} +{:+ "=VAL", := "false"} +{:+ "=VAL", := "null"} +{:+ "-SEQ"} +{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} +{:+ "-SEQ"} +{:+ "-MAP"} +{:+ "-DOC"} +{:+ "+DOC"} +{:+ "+MAP"} +{:+ "=VAL", := "another"} +{:+ "=VAL", := "doc"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)"} +}; + + +int main() +{ + Ys2EdnScoped ys2edn; + TestResult total = {}; + size_t failed_cases = {}; + size_t num_cases = C4_COUNTOF(test_cases); + for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) + { + printf("-----------------------------------------\n" + "case %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); + const TestResult tr = test_cases[i].test(ys2edn.ryml2edn); + total.add(tr); + failed_cases += (!tr); + printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + } + printf("%u/%u assertions failed\n", total.num_failed_assertions, total.num_assertions); + printf("%u/%u tests failed\n", total.num_failed_tests, total.num_tests); + printf("%zu/%zu cases failed\n", failed_cases, num_cases); + return total ? 0 : -1; +} diff --git a/rapidyaml/pom.xml b/rapidyaml/pom.xml new file mode 100644 index 000000000..3efb8c7c4 --- /dev/null +++ b/rapidyaml/pom.xml @@ -0,0 +1,115 @@ + + + + + rapidyaml + + 0.7.2 + + rapidyaml + + org.rapidyaml + + 4.0.0 + + + rapidyaml is a C++ library to parse and emit YAML, and do it fast. + + + https://rapidyaml.org + + + + MIT License + http://www.opensource.org/licenses/mit-license.php + repo + + + + + + clojars + Clojars repository + https://clojars.org/repo + + + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 3.8.1 + + + org.json + json + 20240205 + + + + + + + + + maven-clean-plugin + 3.1.0 + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + + + + + + maven-project-info-reports-plugin + + + + diff --git a/rapidyaml/rapidyaml.mk b/rapidyaml/rapidyaml.mk new file mode 100644 index 000000000..73e7bb913 --- /dev/null +++ b/rapidyaml/rapidyaml.mk @@ -0,0 +1,5 @@ +RAPIDYAML_TAG ?= v0.7.2 +RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_JAVA := \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java diff --git a/rapidyaml/src/README.md b/rapidyaml/src/README.md new file mode 100644 index 000000000..d920cfa6b --- /dev/null +++ b/rapidyaml/src/README.md @@ -0,0 +1,33 @@ +## Notes on JNI vs JNA +, packaging, loading, etc + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > JNI is almost always the fasted ffi option. I've heard good things about JNR performance. JNA is usually one of the slower ffi options, but can be sped up with direct mapping, https://github.com/java-native-access/jna/blob/master/www/DirectMapping.md. + + +## Notes on how to profile + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > Another tool to try to find the bottleneck is https://github.com/clojure-goes-fast/clj-async-profiler. The flamegraph might show some obvious performance issue (assuming the issue is on the jvm side). + + > Depending on how long a parse takes, I would recommend something like https://github.com/hugoduncan/criterium. time is not a good way to benchmark code unless it's a very slow function call. I would use the profiler to try and figure out where the bottleneck is. + + +## JNI examples + +- [full JNI example](https://github.com/mkowsiak/jnicookbook/tree/master/recipes/recipeNo031) +- [full JNI example with other non-JNI shared libraries](https://github.com/mkowsiak/jnicookbook/tree/master/recipes/recipeNo035) +- [another example linking with more libraries](https://www.dynamsoft.com/codepool/package-jni-shared-library-jar-file.html) + +- https://stackoverflow.com/questions/1611357/how-to-make-a-jar-file-that-includes-dll-files#comment1483970_1611367 + +## Notes on JNI - how to call c++ code from java + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > I'm surprised trying to pass a mutable byte buffer doesn't cause more issues. I think the recommended way to pass a byte buffer to native is with http://java-native-access.github.io/jna/5.13.0/javadoc/com/sun/jna/Memory.html and you can get the string with .getString. + + > > But I was thinking whether it is possible/practical/advisable (in terms of speed) to build the Clojure dictionary directly in the C++ code. Currently the C++ code is providing an EDN markup string that is later parsed in Clojure. Assuming a large dictionary of say ~40k entries, would it be possible to call native clojure/java JNI functions to build the final structure instead of creating the intermediate EDN? Would that be a gain? Would the ~40k calls to JNI end up costing too much? + > This isn't something I've tried before, so take it with a grain of salt, but you have at least a few options with different tradeoffs. All the collection types in clojure are based on protocols/interfaces, so it would be possible to just return a pointer, with no copying, and wrap it proxy that implements all the relevant interfaces for maps/lists/etc. When JVM code asks for a value from a map or element from list, you produce the JVM value for numbers/strings or you return another proxy pointer if it's a collection. You might still have to make a copy to return a string value. If you're returning large values that you expect will only be partially read or read only once, then lazily producing jvm values might be a win. If you expect the large value to be completely read multiple times, then it could be potentially faster to just convert the full data structure to a JVM value. There's also intermediate options where you do some of the work upfront, and do some of the work lazily. Granularity will also affect memory usage. You probably don't want some scenario where someone parses a giant blob and keeps only a small part, but still has to hold the giant value in memory until the small part gets reclaimed.I don't have a good answer for you here. My intuition is that your approach of building the final data structure in c++ is probably a good idea, but I don't really have the experience to say for sure.This type of question might get a better answer in #data-science. I think they similar issues with dealing with large datasets that are partially processed in native code. They've also built deep integrations with python via https://github.com/clj-python/libpython-clj where I think they've run into similar problems. diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java new file mode 100644 index 000000000..ccf7d4370 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -0,0 +1,61 @@ +package org.rapidyaml; + +import org.rapidyaml.YamlParseErrorException; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; + +/** + * Interface with the shared librapidyaml library + * + */ +public class Rapidyaml { + public static String RAPIDYAML_VERSION = "0.7.2"; + + private native long ys2edn_init(); + private native void ys2edn_destroy(long ryml2edn); + private native int ys2edn_parse(long ryml2edn, String filename, + byte[] ys, int ys_length, + byte[] edn, int edn_length); + private native int ys2edn_retry_get( + long ryml2edn, byte[] edn, int edn_size + ); + + private final long ryml2edn; + + public Rapidyaml() { + String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; + System.loadLibrary(library_name); + this.ryml2edn = this.ys2edn_init(); + } + + // Likely bad idea to implement finalize: + // + // https://stackoverflow.com/questions/158174/why-would-you-ever-implement-finalize + // + protected void finalize() throws Throwable { + try { + this.ys2edn_destroy(this.ryml2edn); + } + finally { + super.finalize(); + } + } + + public String parseYS(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + String filename = "yamlscript"; // fixme + byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); + int edn_size = 10 * src.length; + byte[] edn = new byte[edn_size]; + int required_size = ys2edn_parse(this.ryml2edn, filename, src, src.length, edn, edn_size); + if(required_size > edn_size) { + edn_size = required_size; + edn = new byte[edn_size]; + required_size = ys2edn_retry_get(this.ryml2edn, edn, edn_size); + if(required_size != edn_size) { + throw new RuntimeException("inconsistent size"); + } + } + String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); + return ret; + } +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java b/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java new file mode 100644 index 000000000..d5d4730df --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java @@ -0,0 +1,16 @@ +package org.rapidyaml; + +// https://www.baeldung.com/java-new-custom-exception +public class YamlParseErrorException extends Exception +{ + public final int offset; + public final int line; + public final int column; + public YamlParseErrorException(int offset_, int line_, int column_, String msg) + { + super(msg); + offset = offset_; + line = line_; + column = column_; + } +} diff --git a/rapidyaml/src/site/site.xml b/rapidyaml/src/site/site.xml new file mode 100644 index 000000000..27de8ef17 --- /dev/null +++ b/rapidyaml/src/site/site.xml @@ -0,0 +1,26 @@ + + + + + rapidyaml + https://maven.apache.org/images/apache-maven-project.png + https://www.apache.org/ + + + + https://maven.apache.org/images/maven-logo-black-on-white.png + https://maven.apache.org/ + + + + org.apache.maven.skins + maven-fluido-skin + 1.7 + + + + + + + diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java new file mode 100644 index 000000000..48e9eaf83 --- /dev/null +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -0,0 +1,163 @@ +package org.rapidyaml; + +import org.rapidyaml.*; +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class RapidyamlTest extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public RapidyamlTest(String testName) + { + super(testName); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite(RapidyamlTest.class); + } + + private void testEdn_(String ys, String expected) + { + Rapidyaml rapidyaml = new Rapidyaml(); + try { + String actual = rapidyaml.parseYS(ys); + try { + assertEquals(expected.length(), actual.length()); + assertEquals(expected, actual); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + catch (YamlParseErrorException e) { + fail("parse error:\n" + e.getMessage()); + } + } + + public void testPlainMap() + { + testEdn_( + "a: 1" + , + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"a\"}\n" + + "{:+ \"=VAL\", := \"1\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + } + + public void testUtf8() + { + testEdn_( + "𝄞: ✅" + , + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"𝄞\"}\n" + + "{:+ \"=VAL\", := \"✅\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + } + + public void testLargeCase() + { + testEdn_( + "foo: !\n" + + "- {x: y}\n" + + "- [x, y]\n" + + "- foo\n" + + "- 'foo'\n" + + "- \"foo\"\n" + + "- |\n" + + " foo\n" + + "- >\n" + + " foo\n" + + "- [1, 2, true, false, null]\n" + + "- &anchor-1 !tag-1 foobar\n" + + "---\n" + + "another: doc\n" + , + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"foo\"}\n" + + "{:+ \"+SEQ\", :! \"\"}\n" + + "{:+ \"+MAP\", :flow true}\n" + + "{:+ \"=VAL\", := \"x\"}\n" + + "{:+ \"=VAL\", := \"y\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"+SEQ\", :flow true}\n" + + "{:+ \"=VAL\", := \"x\"}\n" + + "{:+ \"=VAL\", := \"y\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"=VAL\", := \"foo\"}\n" + + "{:+ \"=VAL\", :' \"foo\"}\n" + + "{:+ \"=VAL\", :$ \"foo\"}\n" + + "{:+ \"=VAL\", :| \"foo\\n\"}\n" + + "{:+ \"=VAL\", :> \"foo\\n\"}\n" + + "{:+ \"+SEQ\", :flow true}\n" + + "{:+ \"=VAL\", := \"1\"}\n" + + "{:+ \"=VAL\", := \"2\"}\n" + + "{:+ \"=VAL\", := \"true\"}\n" + + "{:+ \"=VAL\", := \"false\"}\n" + + "{:+ \"=VAL\", := \"null\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"=VAL\", :& \"anchor-1\", :! \"tag-1\", := \"foobar\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + "{:+ \"+DOC\"}\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"another\"}\n" + + "{:+ \"=VAL\", := \"doc\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + } + + public void testFailure() + { + Rapidyaml rapidyaml = new Rapidyaml(); + String ys = ": : : :"; + boolean gotit = false; + try { + rapidyaml.parseYS(ys); + } + catch(YamlParseErrorException e) { + gotit = true; + assertEquals(2, e.offset); + assertEquals(1, e.line); + assertEquals(3, e.column); + assertTrue(e.getMessage() != null); + assertFalse(e.getMessage().isEmpty()); + } + catch(RuntimeException e) { + fail("wrong exception type"); + } + catch(Exception e) { + fail("wrong exception type"); + } + assertTrue(gotit); + } +} diff --git a/util/RYS b/util/RYS new file mode 100755 index 000000000..1d53e01a0 --- /dev/null +++ b/util/RYS @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +( + root=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd -P) + root_ry=$root/rapidyaml + make --no-print-directory -C "$root_ry" build + export YS_PARSER_RAPIDYAML=1 + export LD_LIBRARY_PATH=$root_ry/native + bin=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd -P) + exec "$bin/YS" "$@" +) diff --git a/ys/config/reflect-config.json b/ys/config/reflect-config.json new file mode 120000 index 000000000..c0189b67e --- /dev/null +++ b/ys/config/reflect-config.json @@ -0,0 +1 @@ +../../common/reflection.json \ No newline at end of file diff --git a/ys/config/resource-config.json b/ys/config/resource-config.json new file mode 100644 index 000000000..0bc3aa5c7 --- /dev/null +++ b/ys/config/resource-config.json @@ -0,0 +1,6 @@ +{ + "resources": [ + {"pattern": "librapidyaml.so"}, + {"pattern": "librapibyaml.dylib"} + ] +} From 8efaa8ed5115d15b179f5c1968817a774b090426 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 29 Aug 2024 09:41:49 +0100 Subject: [PATCH 04/29] rapidyaml: Enable C++ tests, clean native/Makefile --- rapidyaml/Makefile | 2 + rapidyaml/native/CMakeLists.txt | 5 ++ rapidyaml/native/Makefile | 22 ++++---- rapidyaml/native/rapidyaml_edn.hpp | 6 +-- rapidyaml/native/rapidyaml_test.cpp | 83 +++++++++++------------------ 5 files changed, 51 insertions(+), 67 deletions(-) diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index 5d455f5c6..ce5fb92a1 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -31,9 +31,11 @@ jar: $(RAPIDYAML_JAR) install:: $(RAPIDYAML_INSTALLED) test:: build $(JAVA_INSTALLED) + $(MAKE) -C native $@ mvn $@ test-x: build $(JAVA_INSTALLED) + $(MAKE) -C native $@ mvn -X -e test clean:: diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 2bdd95182..877eeb4fd 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -38,3 +38,8 @@ target_include_directories(rapidyaml PUBLIC ${JNI_INCLUDE_DIRS}) add_executable(rapidyaml-test rapidyaml_test.cpp) target_link_libraries(rapidyaml-test rapidyaml) +add_custom_target(rapidyaml-test-run + DEPENDS rapidyaml-test + COMMAND $ + COMMENT "running C++ tests" +) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index 876509c10..d1767be59 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -25,6 +25,10 @@ CMK_FLAGS := CMK_FLAGS += -D YS2EDN_TIMED=$(TIMED) CMK_FLAGS += -D BUILD_SHARED_LIBS=$(SHARED) +BDIR := rapidyaml-build + +CMK_BUILD := cmake --build $(BDIR) --parallel --verbose + #------------------------------------------------------------------------------ default:: @@ -32,9 +36,8 @@ default:: build:: $(RAPIDYAML_SO) test:: build - cmake --build rapidyaml-build \ - --parallel --target rapidyaml-test - ./rapidyaml-build/rapidyaml-test + $(CMK_BUILD) --target rapidyaml-test-run + ./$(BDIR)/rapidyaml-test rapidyaml: mkdir -p $@ @@ -47,7 +50,7 @@ rapidyaml: clean:: $(RM) librapidyaml.* $(RM) $(RAPIDYAML_HPP) - $(RM) -r rapidyaml-build + $(RM) -r $(BDIR) $(RM) -r rapidyaml-install realclean:: clean @@ -56,16 +59,13 @@ realclean:: clean #------------------------------------------------------------------------------ $(RAPIDYAML_SO): $(RAPIDYAML_SO_DEPS) - cmake -S . -B rapidyaml-build \ - $(CMK_FLAGS) - cmake --build rapidyaml-build \ - --parallel --verbose --target rapidyaml - mv rapidyaml-build/*.so $@ + cmake -S . -B $(BDIR) $(CMK_FLAGS) + $(CMK_BUILD) --target rapidyaml + mv $(BDIR)/*.so $@ ln -fs $@ librapidyaml.so $(RAPIDYAML_H): $(RAPIDYAML_JAVA) javac -h . $^ $(RAPIDYAML_HPP): rapidyaml - $(PYTHON) rapidyaml/tools/amalgamate.py \ - $(RAPIDYAML_AMALGAMATE_OPTS) > $@ + $(PYTHON) rapidyaml/tools/amalgamate.py $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/native/rapidyaml_edn.hpp b/rapidyaml/native/rapidyaml_edn.hpp index 69ca2f077..28ab0d370 100644 --- a/rapidyaml/native/rapidyaml_edn.hpp +++ b/rapidyaml/native/rapidyaml_edn.hpp @@ -52,9 +52,9 @@ RYML_EXPORT Ryml2Edn *ys2edn_init(); RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn); /** Parse YAML, and return corresponding EDN. Return the number of - * characters needed for edn. Check if the returned size is larger - * than edn_size. If it is, call ys_retry_get() can be called - * afterwards to extract the EDN. */ + * characters needed for edn. The caller must check if the returned + * size is larger than edn_size. If it is, call ys_retry_get() can be + * called afterwards to extract the EDN. */ RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, const char *filename, char *ys, size_type ys_size, diff --git a/rapidyaml/native/rapidyaml_test.cpp b/rapidyaml/native/rapidyaml_test.cpp index 7c2e15f72..50ca878de 100644 --- a/rapidyaml/native/rapidyaml_test.cpp +++ b/rapidyaml/native/rapidyaml_test.cpp @@ -12,12 +12,6 @@ struct Ys2EdnScoped }; -struct ResultScoped -{ - char *edn = nullptr; - ~ResultScoped() { if(edn) ys2edn_free(edn); } -}; - struct TestResult { uint32_t num_assertions; @@ -34,6 +28,7 @@ struct TestResult } }; + struct TestCase { csubstr ys; @@ -55,10 +50,10 @@ struct TestCase #define _runtest(name, ...) \ do { \ - printf("[RUN ] %s ... \n", #name); \ + printf("[ RUN ] %s ... \n", #name); \ TestResult tr_ = name(__VA_ARGS__); \ tr.add(tr_); \ - printf("[%s] %s\n", tr_?"OK ":"FAIL", #name); \ + printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ } while(0) #define CHECK(cond) \ do { \ @@ -73,68 +68,48 @@ struct TestCase TestResult test(Ryml2Edn *ryml2edn) const { TestResult tr = {}; - _runtest(test_1_alloc, ); - _runtest(test_2_large_enough, ); - _runtest(test_3_too_small, ); - _runtest(test_4_nullptr, ); - _runtest(test_1_alloc_reuse, ryml2edn); - _runtest(test_2_large_enough_reuse, ryml2edn); - _runtest(test_3_too_small_reuse, ryml2edn); - _runtest(test_4_nullptr_reuse, ryml2edn); + _runtest(test_large_enough, ); + _runtest(test_too_small, ); + _runtest(test_nullptr, ); + _runtest(test_large_enough_reuse, ryml2edn); + _runtest(test_too_small_reuse, ryml2edn); + _runtest(test_nullptr_reuse, ryml2edn); return tr; } - // 1. simplest (but wasteful) first: alloc a new string - TestResult test_1_alloc_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - ResultScoped result; // frees when destroying - result.edn = ys2edn_alloc(ryml2edn, "ysfilename", - input.str, (size_type)input.len); - CHECK(testeq(c4::to_csubstr(result.edn))); - return tr; - } - TestResult test_1_alloc() const - { - Ys2EdnScoped lib; - return test_1_alloc_reuse(lib.ryml2edn); - } - - // 2. happy path: large-enough destination string - TestResult test_2_large_enough_reuse(Ryml2Edn *ryml2edn) const + // happy path: large-enough destination string + TestResult test_large_enough_reuse(Ryml2Edn *ryml2edn) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); std::string output; output.resize(2 * edn.len); - size_type reqsize = ys2edn(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - &output[0], (size_type)output.size()); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); CHECK(reqsize == edn.len+1); CHECK(reqsize != 0); output.resize(reqsize - 1u); CHECK(testeq(c4::to_csubstr(output))); return tr; } - TestResult test_2_large_enough() const + TestResult test_large_enough() const { Ys2EdnScoped lib; - return test_2_large_enough_reuse(lib.ryml2edn); + return test_large_enough_reuse(lib.ryml2edn); } // less-happy path: destination string not large enough - TestResult test_3_too_small_reuse(Ryml2Edn *ryml2edn) const + TestResult test_too_small_reuse(Ryml2Edn *ryml2edn) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); std::string output = "?"; - size_type reqsize = ys2edn(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); CHECK(reqsize == edn.len+1); CHECK(reqsize != 0); CHECK(output != "?"); @@ -145,33 +120,35 @@ struct TestCase CHECK(testeq(c4::to_csubstr(output))); return tr; } - TestResult test_3_too_small() const + TestResult test_too_small() const { Ys2EdnScoped lib; - return test_3_too_small_reuse(lib.ryml2edn); + return test_too_small_reuse(lib.ryml2edn); } // safe calling with nullptr - TestResult test_4_nullptr_reuse(Ryml2Edn *ryml2edn) const + TestResult test_nullptr_reuse(Ryml2Edn *ryml2edn) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); - size_type reqsize = ys2edn(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - nullptr, 0); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); CHECK(reqsize == edn.len+1); CHECK(reqsize != 0); return tr; } - TestResult test_4_nullptr() const + TestResult test_nullptr() const { Ys2EdnScoped lib; - return test_4_nullptr_reuse(lib.ryml2edn); + return test_nullptr_reuse(lib.ryml2edn); } }; +//----------------------------------------------------------------------------- + const TestCase test_cases[] = { // case ------------------------------ {"say: 2 + 2", From 122e89f344c1ded73e998c34759cef6098a8ba0a Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 29 Aug 2024 11:33:15 +0100 Subject: [PATCH 05/29] rapidyaml: Reserve memory --- rapidyaml/native/rapidyaml_edn.cpp | 3 ++- rapidyaml/native/rapidyaml_edn_handler.hpp | 25 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/rapidyaml/native/rapidyaml_edn.cpp b/rapidyaml/native/rapidyaml_edn.cpp index 8f5bcfca9..59082f16f 100644 --- a/rapidyaml/native/rapidyaml_edn.cpp +++ b/rapidyaml/native/rapidyaml_edn.cpp @@ -75,8 +75,9 @@ RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); { - TIMED_SECTION("reset"); + TIMED_SECTION("reset + reserve"); ryml2edn->reset(); + ryml2edn->m_handler.reserve(ys_size > edn_size ? 3 * ys_size : edn_size, 256u); } { TIMED_SECTION("parse_in_place"); diff --git a/rapidyaml/native/rapidyaml_edn_handler.hpp b/rapidyaml/native/rapidyaml_edn_handler.hpp index 973c52f57..a50ee026e 100644 --- a/rapidyaml/native/rapidyaml_edn_handler.hpp +++ b/rapidyaml/native/rapidyaml_edn_handler.hpp @@ -82,13 +82,15 @@ struct EventHandlerEdn : public EventHandlerStackresult.reserve(edn_size); + for(size_t i = 0; i < m_val_buffers.size(); ++i) + { + int sz = edn_size / (int(1) << (uint32_t)i); + sz = sz >= 128 ? sz : 128; + m_val_buffers[i].result.reserve((size_t)sz); + } + m_arena.reserve(arena_size); + } + /** @} */ public: From bc4cbee6a53ad77ab9f0a80a03d8ebb80ee05bde Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 5 Jan 2025 20:15:48 +0000 Subject: [PATCH 06/29] rapidyaml: Compile both shared and static --- common/vars.mk | 13 +++++++++++-- rapidyaml/Makefile | 8 +++++--- rapidyaml/native/Makefile | 41 ++++++++++++++++++++------------------- rapidyaml/rapidyaml.mk | 5 ----- 4 files changed, 37 insertions(+), 30 deletions(-) delete mode 100644 rapidyaml/rapidyaml.mk diff --git a/common/vars.mk b/common/vars.mk index afa4ce58d..5f217fec6 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -49,12 +49,15 @@ ifneq (,$(findstring linux,$(ostype))) GCC := gcc -std=gnu99 -fPIC -shared SO := so DY := + DOTLIB := a else ifneq (,$(findstring darwin,$(ostype))) IS_MACOS := true GCC := gcc -dynamiclib SO := dylib DY := DY + DOTLIB := a else + DOTLIB := lib $(error Unsupported OSTYPE: $(ostype)) endif @@ -219,9 +222,15 @@ RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz RAPIDYAML := $(ROOT)/rapidyaml RAPIDYAML_VERSION := 0.7.2 +RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) +RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_TIMED := 0 +RAPIDYAML_JAVA := \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java +RAPIDYAML_SO := $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_LIB := $(ROOT)/rapidyaml/native/librapidyaml.$(DOTLIB) RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar -RAPIDYAML_SO := \ - $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) RAPIDYAML_INSTALLED_DIR := \ $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)/ RAPIDYAML_INSTALLED := \ diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index ce5fb92a1..c0d7649a2 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -1,7 +1,6 @@ include ../common/base.mk include $(COMMON)/clojure.mk include $(COMMON)/java.mk -include ./rapidyaml.mk RAPIDYAML_JAR_DEPS := \ $(wildcard \ @@ -19,11 +18,12 @@ RAPIDYAML_CLASS := \ #------------------------------------------------------------------------------ default:: echo $(RAPIDYAML_SO) + echo $(RAPIDYAML_LIB) x: YS -ce 'foo: var' -build:: $(RAPIDYAML_SO) +build:: $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @: jar: $(RAPIDYAML_JAR) @@ -39,7 +39,7 @@ test-x: build $(JAVA_INSTALLED) mvn -X -e test clean:: - $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) + $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) $(RM) -r reports/ target/ $(MAKE) -C native $@ @@ -53,6 +53,8 @@ sysclean:: #------------------------------------------------------------------------------ $(RAPIDYAML_SO): $(MAKE) -C native $@ +$(RAPIDYAML_LIB): + $(MAKE) -C native $@ $(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED) mvn install diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index d1767be59..a11f5ccd2 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -2,13 +2,12 @@ include ../../common/base.mk include $(COMMON)/clojure.mk include $(COMMON)/java.mk include $(COMMON)/python.mk -include ../rapidyaml.mk RAPIDYAML_H := org_rapidyaml_Rapidyaml.h RAPIDYAML_HPP := rapidyaml_all.hpp RAPIDYAML_AMALGAMATE_OPTS := --stl -RAPIDYAML_SO_DEPS := \ +RAPIDYAML_DEPS := \ Makefile \ CMakeLists.txt \ $(JAVA_HOME) \ @@ -16,28 +15,19 @@ RAPIDYAML_SO_DEPS := \ $(RAPIDYAML_HPP) \ $(wildcard ./*pp) \ -# TODO maybe change to static library! -# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni -SHARED := 1 -TIMED := 0 - CMK_FLAGS := -CMK_FLAGS += -D YS2EDN_TIMED=$(TIMED) -CMK_FLAGS += -D BUILD_SHARED_LIBS=$(SHARED) +CMK_FLAGS += -D YS2EDN_TIMED=$(RAPIDYAML_TIMED) BDIR := rapidyaml-build -CMK_BUILD := cmake --build $(BDIR) --parallel --verbose +CMK_BUILD := #------------------------------------------------------------------------------ default:: -build:: $(RAPIDYAML_SO) - -test:: build - $(CMK_BUILD) --target rapidyaml-test-run - ./$(BDIR)/rapidyaml-test +build:: build-static build-shared +test: test-static test-shared rapidyaml: mkdir -p $@ @@ -58,14 +48,25 @@ realclean:: clean #------------------------------------------------------------------------------ -$(RAPIDYAML_SO): $(RAPIDYAML_SO_DEPS) - cmake -S . -B $(BDIR) $(CMK_FLAGS) - $(CMK_BUILD) --target rapidyaml - mv $(BDIR)/*.so $@ +build-static: $(RAPIDYAML_LIB) +test-static: build-static + cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml-test-run +$(RAPIDYAML_LIB): $(RAPIDYAML_DEPS) + cmake -S . -B $(BDIR)-static $(CMK_FLAGS) -D BUILD_SHARED_LIBS=OFF + cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml + cp -fv $(BDIR)-static/*.a $@ + +build-shared: $(RAPIDYAML_SO) +test-shared: build-shared + cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml-test-run +$(RAPIDYAML_SO): $(RAPIDYAML_DEPS) + cmake -S . -B $(BDIR)-shared $(CMK_FLAGS) -D BUILD_SHARED_LIBS=ON + cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml + cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so $(RAPIDYAML_H): $(RAPIDYAML_JAVA) - javac -h . $^ + javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work $(RAPIDYAML_HPP): rapidyaml $(PYTHON) rapidyaml/tools/amalgamate.py $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/rapidyaml.mk b/rapidyaml/rapidyaml.mk deleted file mode 100644 index 73e7bb913..000000000 --- a/rapidyaml/rapidyaml.mk +++ /dev/null @@ -1,5 +0,0 @@ -RAPIDYAML_TAG ?= v0.7.2 -RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml -RAPIDYAML_JAVA := \ - $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ - $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java From ff5ea4ffac14e2d0fe1ba2888ce113f16eb57978 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 15 Sep 2024 13:36:58 +0100 Subject: [PATCH 07/29] rapidyaml: Add parsing of ys to integer-based events Adds a parser producing integer events, and integers signifying strings by indexing into the parsed YS string. Each event is provided as an integer bitmask. The event bits are defined in both the C++ side (ysparse_evt_handler.hpp) and the Java side (org.rapidyaml.Evt), and they need to stay consistent on both ends. When a string is associated with an event, it is provided as an integer offset and length. For example, the YAML `say: 2 + 2` produces the following sequence of integers: ```c++ BSTR, BDOC, VAL|BMAP|BLCK, KEY|SCLR|PLAI, 0, 3, // "say" VAL|SCLR|PLAI, 5, 5, // "2 + 2" EMAP, EDOC, ESTR, ``` Note that the scalar events, ie "say" and "2 + 2", are followed each by two extra integers encoding the offset and length of the scalar's string. These two extra integers are present whenever the event has any of the bits SCLR, ALIA, ANCH or TAG. For ease of use, there is a bitmask HAS_STR, which enables quick testing by a simple `flags & HAS_STR`. Also, where a string requires filtering, the parser filters it in-place in the input string, and returns the extra integers pertaining to the resulting filtered string. The existing EDN-producing parser was not removed, and the EVT-producing parser was added both on C++ and on the Java JNI bridge. Tests were enlarged to cover the new event parsing, both for C++ and Java. To benefit from this approach, the YS side must implement a mechanism to convert the int event sequence into its internal data-structure, using the symbols in the class org.rapidyaml.Evt. Other changes: - rapidyaml->ysparse: start renaming the C++ library providing the JNI. ysparse is a more accurate description of what the library does. - Directly use only the required rapidyaml source files instead of amalgamating into a single header. - Makefile: - Add target to generate the JNI header - Improve target dependencies --- common/vars.mk | 2 + rapidyaml/Makefile | 17 +- rapidyaml/native/.gitignore | 5 +- rapidyaml/native/CMakeLists.txt | 68 +- rapidyaml/native/Makefile | 29 +- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 58 +- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 24 + rapidyaml/native/rapidyaml_test.cpp | 282 -------- .../{rapidyaml_edn.cpp => ysparse_edn.cpp} | 7 +- .../{rapidyaml_edn.hpp => ysparse_edn.hpp} | 15 +- ...dn_handler.cpp => ysparse_edn_handler.cpp} | 17 +- ...dn_handler.hpp => ysparse_edn_handler.hpp} | 126 ++-- rapidyaml/native/ysparse_evt.cpp | 90 +++ rapidyaml/native/ysparse_evt.hpp | 108 +++ rapidyaml/native/ysparse_evt_handler.cpp | 12 + rapidyaml/native/ysparse_evt_handler.hpp | 680 ++++++++++++++++++ rapidyaml/native/ysparse_test.cpp | 675 +++++++++++++++++ .../src/main/java/org/rapidyaml/Evt.java | 37 + .../main/java/org/rapidyaml/Rapidyaml.java | 19 +- .../java/org/rapidyaml/RapidyamlTest.java | 245 ++++++- 20 files changed, 2081 insertions(+), 435 deletions(-) delete mode 100644 rapidyaml/native/rapidyaml_test.cpp rename rapidyaml/native/{rapidyaml_edn.cpp => ysparse_edn.cpp} (96%) rename rapidyaml/native/{rapidyaml_edn.hpp => ysparse_edn.hpp} (83%) rename rapidyaml/native/{rapidyaml_edn_handler.cpp => ysparse_edn_handler.cpp} (94%) rename rapidyaml/native/{rapidyaml_edn_handler.hpp => ysparse_edn_handler.hpp} (85%) create mode 100644 rapidyaml/native/ysparse_evt.cpp create mode 100644 rapidyaml/native/ysparse_evt.hpp create mode 100644 rapidyaml/native/ysparse_evt_handler.cpp create mode 100644 rapidyaml/native/ysparse_evt_handler.hpp create mode 100644 rapidyaml/native/ysparse_test.cpp create mode 100644 rapidyaml/src/main/java/org/rapidyaml/Evt.java diff --git a/common/vars.mk b/common/vars.mk index 5f217fec6..ef001a228 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -224,6 +224,8 @@ RAPIDYAML := $(ROOT)/rapidyaml RAPIDYAML_VERSION := 0.7.2 RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_BUILD_TYPE := Release +RAPIDYAML_DBG := 0 RAPIDYAML_TIMED := 0 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index c0d7649a2..e29ba8c02 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -10,9 +10,7 @@ RAPIDYAML_JAR_DEPS := \ pom.xml \ ) -RAPIDYAML_CLASS := \ - src/main/java/org/rapidyaml/Rapidyaml.class \ - src/main/java/org/rapidyaml/YamlParseErrorException.class +RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class) #------------------------------------------------------------------------------ @@ -23,7 +21,7 @@ default:: x: YS -ce 'foo: var' -build:: $(RAPIDYAML_SO) $(RAPIDYAML_LIB) +build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @: jar: $(RAPIDYAML_JAR) @@ -51,7 +49,7 @@ sysclean:: #------------------------------------------------------------------------------ -$(RAPIDYAML_SO): +$(RAPIDYAML_SO): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ $(RAPIDYAML_LIB): $(MAKE) -C native $@ @@ -63,9 +61,12 @@ $(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED) jar -cvf $@ $< $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) - # this doesn't work: - #javac $< - # ... but this does: + @# this doesn't work: + @#javac $< + @# ... but this does: javac $(RAPIDYAML_JAVA) +$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) + $(MAKE) -C native $@ + crl: $(RAPIDYAML_CLASS) diff --git a/rapidyaml/native/.gitignore b/rapidyaml/native/.gitignore index 226243a92..8e53ad039 100644 --- a/rapidyaml/native/.gitignore +++ b/rapidyaml/native/.gitignore @@ -1,4 +1,5 @@ /rapidyaml/ -/rapidyaml-build/ -/rapidyaml_all.hpp +/_build/ /librapidyaml.* +/.cache +/compile_commands.json diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 877eeb4fd..61fd11e3c 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -1,45 +1,69 @@ cmake_minimum_required(VERSION 3.12) -project(rapidyaml - DESCRIPTION "rapidyaml -> yamlscript" +project(ysparse + DESCRIPTION "ysparse: rapidyaml -> yamlscript" HOMEPAGE_URL "https://github.com/biojppm/rapidyaml -> https://github.com/yaml/yamlscript" LANGUAGES CXX) find_package(JNI REQUIRED) -option(YS2EDN_TIMED "add timings to sections" OFF) +option(YSPARSE_TIMED "add timings to sections" OFF) +option(YSPARSE_DBG "enable debug logs" OFF) if(UNIX) set(CMAKE_SHARED_LIBRARY_SUFFIX .so) endif() -add_library(rapidyaml +set(libname rapidyaml) # TODO rename to ysparse + +add_library(${libname} + # + # JNI bridge org_rapidyaml_Rapidyaml.h org_rapidyaml_Rapidyaml.cpp - rapidyaml_all.hpp - rapidyaml_edn_handler.hpp - rapidyaml_edn_handler.cpp - rapidyaml_edn.hpp - rapidyaml_edn.cpp + # + # ysparse files + ysparse_edn_handler.hpp + ysparse_edn_handler.cpp + ysparse_edn.hpp + ysparse_edn.cpp + ysparse_evt_handler.hpp + ysparse_evt_handler.cpp + ysparse_evt.hpp + ysparse_evt.cpp + # + # files required from rapidyaml + rapidyaml/src/c4/yml/common.cpp + rapidyaml/src/c4/yml/node_type.cpp + rapidyaml/src/c4/yml/parse.cpp + rapidyaml/src/c4/yml/tree.cpp + rapidyaml/src/c4/yml/tag.cpp + rapidyaml/src/c4/yml/reference_resolver.cpp + # + # files required from rapidyaml/ext/c4core + rapidyaml/ext/c4core/src/c4/base64.cpp + rapidyaml/ext/c4core/src/c4/error.cpp + rapidyaml/ext/c4core/src/c4/language.cpp + rapidyaml/ext/c4core/src/c4/utf.cpp ) -target_include_directories(rapidyaml PUBLIC +target_include_directories(${libname} PUBLIC ${CMAKE_CURRENT_LIST_DIR} - ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/test + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/ext/c4core/src ) -target_compile_definitions(rapidyaml PUBLIC +target_compile_definitions(${libname} PUBLIC RYML_WITH_TAB_TOKENS RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS - RYML_SINGLE_HEADER - $<$:YS2EDN_TIMED> + $<$:YSPARSE_TIMED> + $<$:RYML_DBG> ) -set_target_properties(rapidyaml PROPERTIES CXX_STANDARD 17) +set_target_properties(${libname} PROPERTIES CXX_STANDARD 17) -# target_link_libraries(rapidyaml PUBLIC JNI::JNI) -target_include_directories(rapidyaml PUBLIC ${JNI_INCLUDE_DIRS}) +target_include_directories(${libname} PUBLIC ${JNI_INCLUDE_DIRS}) -add_executable(rapidyaml-test rapidyaml_test.cpp) -target_link_libraries(rapidyaml-test rapidyaml) -add_custom_target(rapidyaml-test-run - DEPENDS rapidyaml-test - COMMAND $ +add_executable(${libname}-test ysparse_test.cpp) +target_link_libraries(${libname}-test ${libname}) +add_custom_target(${libname}-test-run + DEPENDS ${libname}-test + COMMAND $ COMMENT "running C++ tests" ) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index a11f5ccd2..91dd05ea8 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -3,24 +3,27 @@ include $(COMMON)/clojure.mk include $(COMMON)/java.mk include $(COMMON)/python.mk -RAPIDYAML_H := org_rapidyaml_Rapidyaml.h -RAPIDYAML_HPP := rapidyaml_all.hpp -RAPIDYAML_AMALGAMATE_OPTS := --stl +# TODO change to static library! +# https://www.graalvm.org/latest/reference-manual/native-image/guides/build-static-executables/ +# https://www.blog.akhil.cc/static-jni +# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni + +THIS_DIR := $(shell pwd) +BDIR := $(THIS_DIR)/_build/$(BUILD_TYPE)-shared$(SHARED)-timed$(TIMED)-dbg$(DBG) RAPIDYAML_DEPS := \ Makefile \ CMakeLists.txt \ $(JAVA_HOME) \ - $(RAPIDYAML_H) \ - $(RAPIDYAML_HPP) \ + $(RAPIDYAML_JNI_H) \ $(wildcard ./*pp) \ -CMK_FLAGS := -CMK_FLAGS += -D YS2EDN_TIMED=$(RAPIDYAML_TIMED) - -BDIR := rapidyaml-build - -CMK_BUILD := +CMK_ENV := +CMK_FLAGS := \ + -D CMAKE_BUILD_TYPE=$(RAPIDYAML_BUILD_TYPE) \ + -D YSPARSE_TIMED=$(RAPIDYAML_TIMED) \ + -D YSPARSE_DBG=$(RAPIDYAML_DBG) \ + -D CMAKE_EXPORT_COMPILE_COMMANDS=ON #------------------------------------------------------------------------------ @@ -39,7 +42,6 @@ rapidyaml: clean:: $(RM) librapidyaml.* - $(RM) $(RAPIDYAML_HPP) $(RM) -r $(BDIR) $(RM) -r rapidyaml-install @@ -67,6 +69,3 @@ $(RAPIDYAML_SO): $(RAPIDYAML_DEPS) $(RAPIDYAML_H): $(RAPIDYAML_JAVA) javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work - -$(RAPIDYAML_HPP): rapidyaml - $(PYTHON) rapidyaml/tools/amalgamate.py $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 4e57b87d8..63fa9efd1 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -1,5 +1,6 @@ #include -#include "./rapidyaml_edn.hpp" +#include "ysparse_edn.hpp" +#include "ysparse_evt.hpp" #include #ifndef _Included_org_rapidyaml_Rapidyaml @@ -50,6 +51,18 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) return (jlong)obj; } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) +{ + Ryml2Evt *obj = ys2evt_init(); + return (jlong)obj; +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_destroy @@ -61,6 +74,17 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) ys2edn_destroy((Ryml2Edn*)obj); } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_destroy + * Signature: (Ljava/lang/Object;)V + */ +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) +{ + ys2evt_destroy((Ryml2Evt*)obj); +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_parse @@ -93,6 +117,38 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, return rc; } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_parse + * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I + */ +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jbyteArray src, jint src_len, + jintArray dst, jint dst_len) +{ + jboolean src_is_copy, dst_is_copy; + jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); + int* dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + const char *filename = env->GetStringUTFChars(jfilename, 0); + int rc = 0; + try + { + rc = ys2evt_parse((Ryml2Evt*)obj, filename, + (char*)src_, src_len, + dst_, dst_len); + } + catch (Ryml2EvtParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + env->ReleaseByteArrayElements(src, src_, 0); + env->ReleaseIntArrayElements(dst, dst_, 0); + env->ReleaseStringUTFChars(jfilename, filename); + return rc; +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_retry_get diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index c73497636..daba52eff 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -39,6 +39,30 @@ JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get (JNIEnv *, jobject, jlong, jbyteArray, jint); +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init + (JNIEnv *, jobject); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy + (JNIEnv *, jobject, jlong); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_parse + * Signature: (JLjava/lang/String;[BI[II)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); + #ifdef __cplusplus } #endif diff --git a/rapidyaml/native/rapidyaml_test.cpp b/rapidyaml/native/rapidyaml_test.cpp deleted file mode 100644 index 50ca878de..000000000 --- a/rapidyaml/native/rapidyaml_test.cpp +++ /dev/null @@ -1,282 +0,0 @@ -#include - -using c4::csubstr; -using c4::substr; - - -struct Ys2EdnScoped -{ - Ryml2Edn *ryml2edn; - Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} - ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } -}; - - -struct TestResult -{ - uint32_t num_assertions; - uint32_t num_tests; - uint32_t num_failed_assertions; - uint32_t num_failed_tests; - operator bool() const { return num_failed_tests == 0; } - void add(TestResult const& that) - { - num_tests += 1 + that.num_tests; - num_assertions += that.num_assertions; - num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; - num_failed_assertions += that.num_failed_assertions; - } -}; - - -struct TestCase -{ - csubstr ys; - csubstr edn; - bool testeq(csubstr actual) const - { - const bool status = (actual == edn); - if(!status) - printf("------\n" - "FAIL:\n" - "input:~~~%.*s~~~\n" - "expected:~~~%.*s~~~\n" - "actual:~~~%.*s~~~\n", - (int)ys.len, ys.str, - (int)edn.len, edn.str, - (int)actual.len, actual.str); - return status; - } - - #define _runtest(name, ...) \ - do { \ - printf("[ RUN ] %s ... \n", #name); \ - TestResult tr_ = name(__VA_ARGS__); \ - tr.add(tr_); \ - printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ - } while(0) - #define CHECK(cond) \ - do { \ - bool pass = !!(cond); \ - ++tr.num_assertions; \ - if(!pass) { \ - printf("%s:%d: fail! %s", __FILE__, __LINE__, #cond); \ - ++tr.num_failed_assertions; \ - } \ - } while(0) - - TestResult test(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - _runtest(test_large_enough, ); - _runtest(test_too_small, ); - _runtest(test_nullptr, ); - _runtest(test_large_enough_reuse, ryml2edn); - _runtest(test_too_small_reuse, ryml2edn); - _runtest(test_nullptr_reuse, ryml2edn); - return tr; - } - - // happy path: large-enough destination string - TestResult test_large_enough_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output; - output.resize(2 * edn.len); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - &output[0], (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } - TestResult test_large_enough() const - { - Ys2EdnScoped lib; - return test_large_enough_reuse(lib.ryml2edn); - } - - // less-happy path: destination string not large enough - TestResult test_too_small_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output = "?"; - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - CHECK(output != "?"); - output.resize(reqsize); - size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); - CHECK(getsize == reqsize); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } - TestResult test_too_small() const - { - Ys2EdnScoped lib; - return test_too_small_reuse(lib.ryml2edn); - } - - // safe calling with nullptr - TestResult test_nullptr_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - nullptr, 0); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - return tr; - } - TestResult test_nullptr() const - { - Ys2EdnScoped lib; - return test_nullptr_reuse(lib.ryml2edn); - } -}; - - -//----------------------------------------------------------------------------- - -const TestCase test_cases[] = { - // case ------------------------------ - {"say: 2 + 2", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "say"} -{:+ "=VAL", := "2 + 2"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"a: 1", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "1"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"𝄞: ✅", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "𝄞"} -{:+ "=VAL", := "✅"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"[a, b, c]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "=VAL", := "c"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"[a: b]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "-MAP"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {R"(--- !yamlscript/v0 -foo: ! -- {x: y} -- [x, y] -- foo -- 'foo' -- "foo" -- | - foo -- > - foo -- [1, 2, true, false, null] -- &anchor-1 !tag-1 foobar ---- -another: doc -)", - R"(( -{:+ "+MAP", :! "yamlscript/v0"} -{:+ "=VAL", := "foo"} -{:+ "+SEQ", :! ""} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-MAP"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-SEQ"} -{:+ "=VAL", := "foo"} -{:+ "=VAL", :' "foo"} -{:+ "=VAL", :$ "foo"} -{:+ "=VAL", :| "foo\n"} -{:+ "=VAL", :> "foo\n"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "1"} -{:+ "=VAL", := "2"} -{:+ "=VAL", := "true"} -{:+ "=VAL", := "false"} -{:+ "=VAL", := "null"} -{:+ "-SEQ"} -{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} -{:+ "-SEQ"} -{:+ "-MAP"} -{:+ "-DOC"} -{:+ "+DOC"} -{:+ "+MAP"} -{:+ "=VAL", := "another"} -{:+ "=VAL", := "doc"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"} -}; - - -int main() -{ - Ys2EdnScoped ys2edn; - TestResult total = {}; - size_t failed_cases = {}; - size_t num_cases = C4_COUNTOF(test_cases); - for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) - { - printf("-----------------------------------------\n" - "case %zu/%zu ...\n" - "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); - const TestResult tr = test_cases[i].test(ys2edn.ryml2edn); - total.add(tr); - failed_cases += (!tr); - printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); - } - printf("%u/%u assertions failed\n", total.num_failed_assertions, total.num_assertions); - printf("%u/%u tests failed\n", total.num_failed_tests, total.num_tests); - printf("%zu/%zu cases failed\n", failed_cases, num_cases); - return total ? 0 : -1; -} diff --git a/rapidyaml/native/rapidyaml_edn.cpp b/rapidyaml/native/ysparse_edn.cpp similarity index 96% rename from rapidyaml/native/rapidyaml_edn.cpp rename to rapidyaml/native/ysparse_edn.cpp index 59082f16f..b074c4848 100644 --- a/rapidyaml/native/rapidyaml_edn.cpp +++ b/rapidyaml/native/ysparse_edn.cpp @@ -1,6 +1,4 @@ -#define RYML_SINGLE_HDR_DEFINE_NOW -#include -#include "rapidyaml_edn.hpp" +#include "ysparse_edn.hpp" #include namespace ryml { @@ -10,7 +8,7 @@ using namespace c4::yml; using namespace ryml; -#ifndef YS2EDN_TIMED +#ifndef YSPARSE_TIMED #define TIMED_SECTION(name) #else #include @@ -64,6 +62,7 @@ RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn) { TIMED_SECTION("ys2edn_destroy"); ryml2edn->~Ryml2Edn(); + _RYML_CB_FREE(get_callbacks(), ryml2edn, Ryml2Edn, 1); } RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, diff --git a/rapidyaml/native/rapidyaml_edn.hpp b/rapidyaml/native/ysparse_edn.hpp similarity index 83% rename from rapidyaml/native/rapidyaml_edn.hpp rename to rapidyaml/native/ysparse_edn.hpp index 28ab0d370..5a5878967 100644 --- a/rapidyaml/native/rapidyaml_edn.hpp +++ b/rapidyaml/native/ysparse_edn.hpp @@ -1,10 +1,9 @@ #pragma once -#ifndef RAPIDYAML_EVENTS_H -#define RAPIDYAML_EVENTS_H +#ifndef YSPARSE_EDN_HPP_ +#define YSPARSE_EDN_HPP_ #include -#include -#include "rapidyaml_edn_handler.hpp" +#include "ysparse_edn_handler.hpp" namespace ryml { using namespace c4; @@ -19,9 +18,9 @@ using size_type = int; struct RYML_EXPORT Ryml2Edn { - c4::yml::EventHandlerEdn::EventSink m_sink; - c4::yml::EventHandlerEdn m_handler; - c4::yml::ParseEngine m_parser; + ys::EventHandlerEdn::EventSink m_sink; + ys::EventHandlerEdn m_handler; + c4::yml::ParseEngine m_parser; Ryml2Edn() : m_sink() , m_handler(&m_sink) @@ -68,4 +67,4 @@ RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, } #endif -#endif /* RAPIDYAML_EVENTS_H */ +#endif /* YSPARSE_EDN_HPP_ */ diff --git a/rapidyaml/native/rapidyaml_edn_handler.cpp b/rapidyaml/native/ysparse_edn_handler.cpp similarity index 94% rename from rapidyaml/native/rapidyaml_edn_handler.cpp rename to rapidyaml/native/ysparse_edn_handler.cpp index 635f44601..ab295234e 100644 --- a/rapidyaml/native/rapidyaml_edn_handler.cpp +++ b/rapidyaml/native/ysparse_edn_handler.cpp @@ -1,18 +1,17 @@ -#ifndef RYML_SINGLE_HEADER +#include "ysparse_edn_handler.hpp" #include -#include #include -#endif -#include "./rapidyaml_edn_handler.hpp" +// instantiate the template namespace c4 { namespace yml { - -// instantiate the template -template class ParseEngine; +template class ParseEngine; +} // namespace yml +} // namespace c4 +namespace ys { void EventHandlerEdn::EventSink::append_escaped(csubstr val) { #define _c4flush_use_instead(repl, skip) \ @@ -78,6 +77,4 @@ void EventHandlerEdn::EventSink::append_escaped(csubstr val) this->append(val.sub(prev)); #undef _c4flush_use_instead } - -} // namespace yml -} // namespace c4 +} // namespace ys diff --git a/rapidyaml/native/rapidyaml_edn_handler.hpp b/rapidyaml/native/ysparse_edn_handler.hpp similarity index 85% rename from rapidyaml/native/rapidyaml_edn_handler.hpp rename to rapidyaml/native/ysparse_edn_handler.hpp index a50ee026e..5497fc609 100644 --- a/rapidyaml/native/rapidyaml_edn_handler.hpp +++ b/rapidyaml/native/ysparse_edn_handler.hpp @@ -1,19 +1,12 @@ -#ifndef _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ -#define _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ - -#ifdef RYML_SINGLE_HEADER -#include -#else -#ifndef _C4_YML_EVENT_HANDLER_STACK_HPP_ -#include "c4/yml/event_handler_stack.hpp" -#endif -#ifndef _C4_YML_STD_STRING_HPP_ -#include "c4/yml/std/string.hpp" -#endif -#ifndef _C4_YML_DETAIL_PRINT_HPP_ -#include "c4/yml/detail/print.hpp" -#endif -#endif +#ifndef _YSPARSE_EDN_HANDLER_HPP_ +#define _YSPARSE_EDN_HANDLER_HPP_ + +#include +#include +#include +#include +#include +#include #include @@ -22,21 +15,26 @@ C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") -namespace c4 { -namespace yml { - - -/** @addtogroup doc_event_handlers - * @{ */ +namespace ys { +using c4::csubstr; +using c4::substr; +using c4::to_substr; +using c4::to_csubstr; +using c4::yml::id_type; +using c4::yml::NodeType_e; +using c4::yml::type_bits; +#ifdef RYML_DBG +using c4::_dbg_printf; +#endif -struct EventHandlerEdnState : public ParserState +struct EventHandlerEdnState : public c4::yml::ParserState { - NodeData ev_data; + c4::yml::NodeData ev_data; }; -struct EventHandlerEdn : public EventHandlerStack +struct EventHandlerEdn : public c4::yml::EventHandlerStack { /** @name types @@ -82,20 +80,20 @@ struct EventHandlerEdn : public EventHandlerStackflags |= RUNK|RTOP; + m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; m_val_buffers.resize((size_t)m_stack.size()); m_arena.clear(); m_first_doc = true; @@ -123,7 +121,7 @@ struct EventHandlerEdn : public EventHandlerStack_stack_start_parse(filename, relocate_arena, relocate_arena_data); } @@ -173,7 +171,7 @@ struct EventHandlerEdn : public EventHandlerStack'); - _enable_(KEY|KEY_FOLDED); + _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); } C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) { _send_val_scalar_(scalar, '>'); - _enable_(VAL|VAL_FOLDED); + _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); } @@ -423,26 +421,26 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.anchor = anchor; } void set_val_anchor(csubstr anchor) { - _enable_(VALANCH); + _enable_(c4::yml::VALANCH); m_curr->ev_data.m_val.anchor = anchor; } void set_key_ref(csubstr ref) { _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); - _enable_(KEY|KEYREF); + _enable_(c4::yml::KEY|c4::yml::KEYREF); _send_("{:+ \"=ALI\" :* \""); _send_(ref.sub(1)); _send_("\"}\n"); } void set_val_ref(csubstr ref) { - _enable_(VAL|VALREF); + _enable_(c4::yml::VAL|c4::yml::VALREF); _send_("{:+ \"=ALI\" :* \""); _send_(ref.sub(1)); _send_("\"}\n"); @@ -457,12 +455,12 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.tag = _transform_directive(tag, m_key_tag_buf); } void set_val_tag(csubstr tag) { - _enable_(VALTAG); + _enable_(c4::yml::VALTAG); m_curr->ev_data.m_val.tag = _transform_directive(tag, m_val_tag_buf); } @@ -513,8 +511,6 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.anchor); _send_('\"'); } - if(_has_any_(KEYTAG)) + if(_has_any_(c4::yml::KEYTAG)) { _send_(", :! \""); _send_tag_(m_curr->ev_data.m_key.tag); _send_('\"'); } m_curr->ev_data.m_key = {}; - _disable_(KEYANCH|KEYREF|KEYTAG); + _disable_(c4::yml::KEYANCH|c4::yml::KEYREF|c4::yml::KEYTAG); } void _send_val_props_() { - if(_has_any_(VALANCH|VALREF)) + if(_has_any_(c4::yml::VALANCH|c4::yml::VALREF)) { _send_(", :& \""); _send_(m_curr->ev_data.m_val.anchor); _send_('\"'); } - if(m_curr->ev_data.m_type.type & VALTAG) + if(m_curr->ev_data.m_type.type & c4::yml::VALTAG) { _send_(", :! \""); _send_tag_(m_curr->ev_data.m_val.tag); _send_('\"'); } m_curr->ev_data.m_val = {}; - _disable_(VALANCH|VALREF|VALTAG); + _disable_(c4::yml::VALANCH|c4::yml::VALREF|c4::yml::VALTAG); } void _send_tag_(csubstr tag) { @@ -663,12 +659,12 @@ struct EventHandlerEdn : public EventHandlerStackpos); } } - csubstr result = normalize_tag_long(tag, output); + csubstr result = c4::yml::normalize_tag_long(tag, output); _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); _RYML_CB_CHECK(m_stack.m_callbacks, result.str); return result; @@ -677,14 +673,10 @@ struct EventHandlerEdn : public EventHandlerStack + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +using namespace ryml; + +#ifndef YS2PARSE_TIMED +#define TIMED_SECTION(name) +#else +#include +#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) +struct timed_section +{ + using myclock = std::chrono::steady_clock; + using fmsecs = std::chrono::duration; + csubstr name; + myclock::time_point start; + fmsecs since() const { return myclock::now() - start; } + timed_section(csubstr n) : name(n), start(myclock::now()) {} + ~timed_section() + { + fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); + } +}; +#endif + + +#if defined(__cplusplus) +extern "C" { +#endif +// see +// https://stackoverflow.com/questions/230689/best-way-to-throw-exceptions-in-jni-code +// https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method + +namespace { +C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) +{ + Ryml2EvtParseError exc; + exc.location = location; + exc.msg.assign(msg, msg_len); + throw exc; +} +} // anon namespace + +RYML_EXPORT Ryml2Evt *ys2evt_init() +{ + TIMED_SECTION("ys2evt_init"); + Callbacks cb = {}; + cb.m_error = &ys2evt_parse_error; + set_callbacks(cb); + Ryml2Evt *ryml2evt = _RYML_CB_ALLOC(get_callbacks(), Ryml2Evt, 1); + _RYML_CB_CHECK(get_callbacks(), ryml2evt != nullptr); + new ((void*)ryml2evt) Ryml2Evt(); + return ryml2evt; +} + +RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt) +{ + TIMED_SECTION("ys2evt_destroy"); + ryml2evt->~Ryml2Evt(); + _RYML_CB_FREE(get_callbacks(), ryml2evt, Ryml2Evt, 1); +} + +RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *events, size_type evt_size) +{ + TIMED_SECTION("ys2evt_parse"); + csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; + substr ys_(ys, (size_t)ys_size); + { + TIMED_SECTION("reset + reserve"); + ryml2evt->reset(ys_, events, evt_size); + ryml2evt->m_handler.reserve(256u); + } + { + TIMED_SECTION("parse_in_place"); + ryml2evt->m_parser.parse_in_place_ev(filename_, ys_); + } + return (size_type)ryml2evt->m_handler.m_evt_curr; +} + +#if defined(__cplusplus) +} +#endif diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp new file mode 100644 index 000000000..7712f489f --- /dev/null +++ b/rapidyaml/native/ysparse_evt.hpp @@ -0,0 +1,108 @@ +#pragma once +#ifndef YSPARSE_EVT_HPP_ +#define YSPARSE_EVT_HPP_ + +#include +#include "ysparse_evt_handler.hpp" + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +#if defined(__cplusplus) +extern "C" { +#endif + +using size_type = int; + +struct RYML_EXPORT Ryml2Evt +{ + ys::EventHandlerEvt m_handler; + c4::yml::ParseEngine m_parser; + Ryml2Evt() + : m_handler() + , m_parser(&m_handler) + { + RYML_CHECK(m_parser.options().scalar_filtering()); + } + void reset(c4::csubstr src, evt::DataType *evt, int32_t evt_size) + { + m_handler.reset(src, evt, evt_size); + } +}; + +struct RYML_EXPORT Ryml2EvtParseError : public std::exception +{ + c4::yml::Location location; + std::string msg; + const char* what() const noexcept override { return msg.c_str(); } +}; + + +//----------------------------------------------------------------------------- + +/** Initialize the resources */ +RYML_EXPORT Ryml2Evt *ys2evt_init(); + +/** Destroy the resources */ +RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt); + +/** Parse YAML in the string `ys` of size `ys_size`, and write the + * result into the array of (integer) events `evt` of size + * `evt_size`. Each event is encoded as a mask of evt::EventFlags + * (note that it uses the integer evt::DataType as the underlying + * type), and when an event has an associated string, it is followed + * in the array by two extra values, which encode the offset and the + * length of the string in the `ys` string. The `ys` string is mutated + * during parsing. + * + * @return the size needed for `evt`. The caller must check if the + * returned size is larger than `evt_size`. If so, this means that + * `evt` could not accomodate all events produced from `ys`, and is + * incomplete. The caller must then (1) resize `evt` to at least the + * return value, (2) re-copy the original YS into `ys` and (3) call + * again this function, passing in the resized `evt` and the fresh + * copy in `ys`. + * + * @note nothing is written beyond `evt_size`. This means that when + * `evt_size` is 0, then `evt` can be null. This function can be + * safely called for any valid pair of `evt` and `evt_size`, and will + * always return the same required size. + * + * For example, the YAML `say: 2 + 2` produces the following sequence of + * 12 integers: + * + * ```c++ + * BSTR, + * BDOC, + * VAL|BMAP|BLCK, + * KEY|SCLR|PLAI, 0, 3, // "say" + * VAL|SCLR|PLAI, 5, 5, // "2 + 2" + * EMAP, + * EDOC, + * ESTR, + * ``` + * + * Note that the scalar events, ie "say" and "2 + 2", are followed + * each by two extra integers encoding the offset and length of the + * scalar's string. These two extra integers are present whenever the + * event has any of the bits `SCLR`, `ALIA`, `ANCH` or `TAG`. For ease + * of use, there is a bitmask `HAS_STR`, which enables quick testing + * by a simple `flags & HAS_STR`. Refer to evt::EventFlags for the + * full list of flags and their meaning. + * + * Also, where a string requires filtering, the parser filters it + * in-place in the input string, and the extra integers will pertain + * to the resulting filtered string. + */ +RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *evt, size_type evt_size); + +#if defined(__cplusplus) +} +#endif + +#endif /* YSPARSE_EVT_HPP_ */ diff --git a/rapidyaml/native/ysparse_evt_handler.cpp b/rapidyaml/native/ysparse_evt_handler.cpp new file mode 100644 index 000000000..24e5fd662 --- /dev/null +++ b/rapidyaml/native/ysparse_evt_handler.cpp @@ -0,0 +1,12 @@ +#include "ysparse_evt_handler.hpp" +#include +#include + +namespace c4 { +namespace yml { + +// instantiate the template +template class ParseEngine; + +} // namespace yml +} // namespace c4 diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp new file mode 100644 index 000000000..f00d7fe58 --- /dev/null +++ b/rapidyaml/native/ysparse_evt_handler.hpp @@ -0,0 +1,680 @@ +#ifndef _YSPARSE_EVT_HANDLER_HPP_ +#define _YSPARSE_EVT_HANDLER_HPP_ + +#include +#include +#include +#include +#include +#include + +C4_SUPPRESS_WARNING_GCC_CLANG_PUSH +C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") +C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") + +namespace evt { +using DataType = int32_t; +typedef enum : DataType { + // --------------------- + // structure flags + KEY_ = 1 << 0, // as key + VAL_ = 1 << 1, // as value + SCLR = 1 << 2, // =VAL + BSEQ = 1 << 3, // +SEQ + ESEQ = 1 << 4, // -SEQ + BMAP = 1 << 5, // +MAP + EMAP = 1 << 6, // -MAP + ALIA = 1 << 7, // ref + ANCH = 1 << 8, // anchor + TAG_ = 1 << 9, // tag + // --------------------- + // style flags + PLAI = 1 << 10, // : (plain scalar) + SQUO = 1 << 11, // ' (single-quoted scalar) + DQUO = 1 << 12, // " (double-quoted scalar) + LITL = 1 << 13, // | (block literal scalar) + FOLD = 1 << 14, // > (block folded scalar) + FLOW = 1 << 15, // flow container: [] for seqs or {} for maps + BLCK = 1 << 16, // block container + // --------------------- + // document flags + BDOC = 1 << 17, // +DOC + EDOC = 1 << 18, // -DOC + EXPL = 1 << 21, // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) + BSTR = 1 << 19, // +STR + ESTR = 1 << 20, // -STR + // --------------------- + // utility flags + LAST = EXPL, + MASK = (LAST << 1) - 1, + HAS_STR = SCLR|ALIA|ANCH|TAG_ // the event requires a string. the next two integers will provide respectively the string's offset and length +} EventFlags; +} // namespace evt + + +namespace ys { + +using c4::csubstr; +using c4::substr; +using c4::to_substr; +using c4::to_csubstr; +#ifdef RYML_DBG +using c4::_dbg_printf; +#endif + +struct EventHandlerEvtState : public c4::yml::ParserState +{ + c4::yml::type_bits evt_type; + int32_t evt_id; +}; + + +struct EventHandlerEvt : public c4::yml::EventHandlerStack +{ + + /** @name types + * @{ */ + + // our internal state must inherit from parser state + using state = EventHandlerEvtState; + + /** @} */ + +public: + + /** @cond dev */ + csubstr m_str; + evt::DataType * m_evt; + int32_t m_evt_curr; + int32_t m_evt_prev; + int32_t m_evt_size; + char m_key_tag_buf[256]; + char m_val_tag_buf[256]; + std::string m_arena; + + // undefined at the end + #define _enable_(bits) _enable__() + #define _disable_(bits) _disable__() + #define _has_any_(bits) _has_any__() + /** @endcond */ + +public: + + /** @name construction and resetting + * @{ */ + + EventHandlerEvt(c4::yml::Callbacks const& cb) + : EventHandlerStack(cb) + { + reset({}, nullptr, 0); + } + EventHandlerEvt() + : EventHandlerEvt(c4::yml::get_callbacks()) + { + } + + void reset(csubstr str, evt::DataType *dst, int32_t dst_size) + { + _stack_reset_root(); + m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; + m_curr->evt_type = {}; + m_curr->evt_id = 0; + m_arena.clear(); + m_str = str; + m_evt = dst; + m_evt_size = dst_size; + m_evt_curr = 0; + m_evt_prev = 0; + } + + void reserve(int arena_size) + { + m_arena.reserve(arena_size); + } + + /** @} */ + +public: + + /** @name parse events + * @{ */ + + void start_parse(const char* filename, c4::yml::detail::pfn_relocate_arena relocate_arena, void *relocate_arena_data) + { + this->_stack_start_parse(filename, relocate_arena, relocate_arena_data); + } + + void finish_parse() + { + this->_stack_finish_parse(); + } + + void cancel_parse() + { + while(m_stack.size() > 1) + _pop(); + } + + /** @} */ + +public: + + /** @name YAML stream events */ + /** @{ */ + + void begin_stream() + { + _send_flag_only_(evt::BSTR); + } + + void end_stream() + { + _send_flag_only_(evt::ESTR); + } + + /** @} */ + +public: + + /** @name YAML document events */ + /** @{ */ + + /** implicit doc start (without ---) */ + void begin_doc() + { + _c4dbgpf("{}/{}: begin_doc", m_evt_curr, m_evt_size); + _send_flag_only_(evt::BDOC); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + } + } + /** implicit doc end (without ...) */ + void end_doc() + { + _c4dbgpf("{}/{}: end_doc", m_evt_curr, m_evt_size); + _send_flag_only_(evt::EDOC); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** explicit doc start, with --- */ + void begin_doc_expl() + { + _c4dbgpf("{}/{}: begin_doc_expl", m_evt_curr, m_evt_size); + _send_flag_only_(evt::BDOC|evt::EXPL); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + } + } + /** explicit doc end, with ... */ + void end_doc_expl() + { + _c4dbgpf("{}/{}: end_doc_expl", m_evt_curr, m_evt_size); + _send_flag_only_(evt::EDOC|evt::EXPL); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** @} */ + +public: + + /** @name YAML map functions */ + /** @{ */ + + void begin_map_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_map_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_map_val_flow() + { + _c4dbgpf("{}/{}: bmap flow", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BMAP|evt::FLOW); + _mark_parent_with_children_(); + _enable_(c4::yml::MAP|c4::yml::FLOW_SL); + _push(); + } + void begin_map_val_block() + { + _c4dbgpf("{}/{}: bmap block", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BMAP|evt::BLCK); + _mark_parent_with_children_(); + _enable_(c4::yml::MAP|c4::yml::BLOCK); + _push(); + } + + void end_map() + { + _pop(); + _send_flag_only_(evt::EMAP); + } + + /** @} */ + +public: + + /** @name YAML seq events */ + /** @{ */ + + void begin_seq_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_seq_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_seq_val_flow() + { + _c4dbgpf("{}/{}: bseq flow", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BSEQ|evt::FLOW); + _mark_parent_with_children_(); + _enable_(c4::yml::SEQ|c4::yml::FLOW_SL); + _push(); + } + void begin_seq_val_block() + { + _c4dbgpf("{}/{}: bseq block", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BSEQ|evt::BLCK); + _mark_parent_with_children_(); + _enable_(c4::yml::SEQ|c4::yml::BLOCK); + _push(); + } + + void end_seq() + { + _pop(); + _send_flag_only_(evt::ESEQ); + } + + /** @} */ + +public: + + /** @name YAML structure events */ + /** @{ */ + + void add_sibling() + { + _RYML_CB_ASSERT(m_stack.m_callbacks, m_parent); + m_curr->evt_type = {}; + } + + /** set the previous val as the first key of a new map, with flow style. + * + * See the documentation for @ref doc_event_handlers, which has + * important notes about this event. + */ + void actually_val_is_first_key_of_new_map_flow() + { + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt_curr > 2); + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt_prev > 0); + _c4dbgpf("{}/{}: prev={} actually_val_is_first_key_of_new_map_flow", m_evt_curr, m_evt_size, m_evt_prev); + // BEFORE + // ... flag start len (free) + // | | + // prev curr + // AFTER + // ... flag flag start len (free) + // | | + // prev curr + if(m_evt_prev < m_evt_size) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (m_evt[m_evt_prev] & evt::HAS_STR) || m_evt_curr >= m_evt_size); + if(m_evt_curr < m_evt_size) + { + // watchout: it must be in this order! + m_evt[m_evt_curr ] = m_evt[m_evt_prev + 2]; + m_evt[m_evt_curr - 1] = m_evt[m_evt_prev + 1]; + m_evt[m_evt_curr - 2] = m_evt[m_evt_prev] | evt::KEY_; + m_evt[m_evt_curr - 2] &= ~evt::VAL_; + } + m_evt[m_evt_prev] = evt::BMAP|evt::FLOW|evt::VAL_; + } + m_curr->evt_id = m_evt_curr - 2; + ++m_evt_prev; + ++m_evt_curr; + _enable_(c4::yml::MAP|c4::yml::FLOW); + _push(); + } + + void actually_val_is_first_key_of_new_map_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + /** @} */ + +public: + + /** @name YAML scalar events */ + /** @{ */ + + + C4_ALWAYS_INLINE void set_key_scalar_plain(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_plain: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::PLAI); + _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN); + } + C4_ALWAYS_INLINE void set_val_scalar_plain(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_plain: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::PLAI); + _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN); + } + + + C4_ALWAYS_INLINE void set_key_scalar_dquoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_dquo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::DQUO); + _enable_(c4::yml::KEY|c4::yml::KEY_DQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_dquoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_dquo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::DQUO); + _enable_(c4::yml::VAL|c4::yml::VAL_DQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_squoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_squo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::SQUO); + _enable_(c4::yml::KEY|c4::yml::KEY_SQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_squoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_squo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::SQUO); + _enable_(c4::yml::VAL|c4::yml::VAL_SQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_literal(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_literal: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::LITL); + _enable_(c4::yml::KEY|c4::yml::KEY_LITERAL); + } + C4_ALWAYS_INLINE void set_val_scalar_literal(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_literal: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::LITL); + _enable_(c4::yml::VAL|c4::yml::VAL_LITERAL); + } + + + C4_ALWAYS_INLINE void set_key_scalar_folded(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_folded: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::FOLD); + _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); + } + C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_folded: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::FOLD); + _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); + } + + + C4_ALWAYS_INLINE void mark_key_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + C4_ALWAYS_INLINE void mark_val_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + + /** @} */ + +public: + +#define _add_scalar_(i, scalar) \ + _c4dbgpf("{}/{}: scalar!", i, m_evt_size); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, scalar.is_sub(m_str)); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt[i] & evt::HAS_STR); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, i + 2 < m_evt_size); \ + m_evt[i + 1] = (evt::DataType)(scalar.str - m_str.str); \ + m_evt[i + 2] = (evt::DataType)scalar.len + + /** @name YAML anchor/reference events */ + /** @{ */ + + void set_key_anchor(csubstr anchor) + { + _c4dbgpf("{}/{}: set_key_anchor", m_evt_curr, m_evt_size); + _enable_(c4::yml::KEYANCH); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::KEY_|evt::ANCH; + _add_scalar_(m_evt_curr, anchor); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + void set_val_anchor(csubstr anchor) + { + _c4dbgpf("{}/{}: set_val_anchor", m_evt_curr, m_evt_size); + _enable_(c4::yml::VALANCH); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::VAL_|evt::ANCH; + _add_scalar_(m_evt_curr, anchor); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + void set_key_ref(csubstr ref) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); + _enable_(c4::yml::KEY|c4::yml::KEYREF); + _send_key_scalar_(ref, evt::KEY_|evt::ALIA); + } + void set_val_ref(csubstr ref) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); + _enable_(c4::yml::VAL|c4::yml::VALREF); + _send_val_scalar_(ref, evt::VAL_|evt::ALIA); + } + + /** @} */ + +public: + + /** @name YAML tag events */ + /** @{ */ + + void set_key_tag(csubstr tag) + { + _enable_(c4::yml::KEYTAG); + csubstr ttag = _transform_directive(tag, m_key_tag_buf); + _RYML_CB_ASSERT(m_stack.m_callbacks, !ttag.empty()); + if(ttag.begins_with('!')) + ttag = ttag.sub(1); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::KEY_|evt::TAG_; + _add_scalar_(m_evt_curr, ttag); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + void set_val_tag(csubstr tag) + { + _enable_(c4::yml::VALTAG); + csubstr ttag = _transform_directive(tag, m_val_tag_buf); + _RYML_CB_ASSERT(m_stack.m_callbacks, !ttag.empty()); + if(ttag.begins_with('!')) + ttag = ttag.sub(1); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::VAL_|evt::TAG_; + _add_scalar_(m_evt_curr, ttag); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + /** @} */ + +public: + + /** @name YAML directive events */ + /** @{ */ + + void add_directive(csubstr directive) + { + _RYML_CB_ERR(m_stack.m_callbacks, "tag directives not supported"); + } + + /** @} */ + +public: + + /** @name YAML arena events */ + /** @{ */ + + substr alloc_arena(size_t len) + { + const size_t sz = m_arena.size(); + csubstr prev = to_csubstr(m_arena); + m_arena.resize(sz + len); + substr out = to_substr(m_arena).sub(sz); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + _stack_relocate_to_new_arena(prev, curr); + return out; + } + + substr alloc_arena(size_t len, substr *relocated) + { + csubstr prev = to_csubstr(m_arena); + if(!prev.is_super(*relocated)) + return alloc_arena(len); + substr out = alloc_arena(len); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); + return out; + } + + /** @} */ + +public: + + /** push a new parent, add a child to the new parent, and set the + * child as the current node */ + void _push() + { + _stack_push(); + m_curr->evt_type = {}; + } + + /** end the current scope */ + void _pop() + { + _stack_pop(); + } + + template C4_ALWAYS_INLINE void _enable__() noexcept + { + m_curr->evt_type |= bits; + } + template C4_ALWAYS_INLINE void _disable__() noexcept + { + m_curr->evt_type &= ~bits; + } + template C4_ALWAYS_INLINE bool _has_any__() const noexcept + { + return (m_curr->evt_type & bits) != c4::yml::type_bits(0); + } + + void _mark_parent_with_children_() + { + if(m_parent) + m_parent->has_children = true; + } + + C4_ALWAYS_INLINE void _send_flag_only_(evt::DataType flags) + { + _c4dbgpf("{}/{}: flag only", m_evt_curr, m_evt_size); + if(m_evt_curr < m_evt_size) + m_evt[m_evt_curr] = flags; + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + ++m_evt_curr; + } + + C4_ALWAYS_INLINE void _send_key_scalar_(csubstr scalar, evt::DataType flags) + { + _c4dbgpf("{}/{}: key scalar", m_evt_curr, m_evt_size); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::SCLR|evt::KEY_|flags; + _add_scalar_(m_evt_curr, scalar); + } + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + C4_ALWAYS_INLINE void _send_val_scalar_(csubstr scalar, evt::DataType flags) + { + _c4dbgpf("{}/{}: val scalar", m_evt_curr, m_evt_size); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::SCLR|evt::VAL_|flags; + _add_scalar_(m_evt_curr, scalar); + } + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + csubstr _transform_directive(csubstr tag, substr output) + { + if(tag.begins_with('!')) + { + if(c4::yml::is_custom_tag(tag)) + { + _RYML_CB_ERR_(m_stack.m_callbacks, "tag not found", m_curr->pos); + } + } + csubstr result = c4::yml::normalize_tag_long(tag, output); + _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); + _RYML_CB_CHECK(m_stack.m_callbacks, result.str); + return result; + } +#undef _enable_ +#undef _disable_ +#undef _has_any_ + +}; + +} // namespace ys + +C4_SUPPRESS_WARNING_GCC_POP + +#endif /* _C4_YML_EVENT_HANDLER_EVT_HPP_ */ diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp new file mode 100644 index 000000000..40a7e5a7a --- /dev/null +++ b/rapidyaml/native/ysparse_test.cpp @@ -0,0 +1,675 @@ +#include +#include +#include + +using c4::csubstr; +using c4::substr; + +namespace c4 +{ +template<> +c4::EnumSymbols const esyms() +{ + static constexpr typename c4::EnumSymbols::Sym syms[] = { + {evt::KEY_, "KEY_"}, + {evt::VAL_, "VAL_"}, + {evt::SCLR, "SCLR"}, + {evt::BSEQ, "BSEQ"}, + {evt::ESEQ, "ESEQ"}, + {evt::BMAP, "BMAP"}, + {evt::EMAP, "EMAP"}, + {evt::ALIA, "ALIA"}, + {evt::ANCH, "ANCH"}, + {evt::TAG_, "TAG_"}, + {evt::PLAI, "PLAI"}, + {evt::SQUO, "SQUO"}, + {evt::DQUO, "DQUO"}, + {evt::LITL, "LITL"}, + {evt::FOLD, "FOLD"}, + {evt::FLOW, "FLOW"}, + {evt::BLCK, "BLCK"}, + {evt::BDOC, "BDOC"}, + {evt::EDOC, "EDOC"}, + {evt::BSTR, "BSTR"}, + {evt::ESTR, "ESTR"}, + {evt::EXPL, "EXPL"}, + }; + return c4::EnumSymbols(syms); +} +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +struct Ys2EdnScoped +{ + Ryml2Edn *ryml2edn; + Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} + ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } +}; +struct Ys2EvtScoped +{ + Ryml2Evt *ryml2evt; + Ys2EvtScoped() : ryml2evt(ys2evt_init()) {} + ~Ys2EvtScoped() { if(ryml2evt) ys2evt_destroy(ryml2evt); } +}; + + +struct TestResult +{ + uint32_t num_assertions; + uint32_t num_tests; + uint32_t num_failed_assertions; + uint32_t num_failed_tests; + operator bool() const { return num_failed_tests == 0 && num_failed_assertions == 0; } + void add(TestResult const& that) + { + num_tests += 1 + that.num_tests; + num_assertions += that.num_assertions; + num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; + num_failed_assertions += that.num_failed_assertions; + } +}; + +// provide a structured input for the events, grouping the relevant +// data in a single structure +struct EvtWithScalar +{ + evt::DataType flags, str_start, str_len; + csubstr scalar; + bool needs_filter; + EvtWithScalar(evt::DataType t, evt::DataType start=0, evt::DataType len=0, csubstr sclr={}, bool needs_filter_=false) + { + flags = t; + str_start = start; + str_len = len; + scalar = sclr; + needs_filter = needs_filter_; + } + size_t required_size() const { return (flags & evt::HAS_STR) ? 3u : 1u; } +}; + +size_t expected_size(std::vector const& evt) +{ + size_t exp = 0; + for(EvtWithScalar const& e : evt) + exp += e.required_size(); + return exp; +} + +struct TestCase +{ + csubstr ys; + csubstr edn; + std::vector evt; + +public: + + #define _runtest(name, ...) \ + do { \ + printf("[ RUN ] %s ... \n", #name); \ + TestResult tr_ = name(__VA_ARGS__); \ + tr.add(tr_); \ + printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ + } while(0) + #define CHECK(cond) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s\n", __FILE__, __LINE__, #cond); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + #define CHECK_MSG(cond, fmt, ...) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s:" fmt "\n", __FILE__, __LINE__, #cond, ## __VA_ARGS__); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + + TestResult test(Ryml2Edn *ryml2edn, Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + _runtest(test_edn_large_enough, ); + _runtest(test_edn_too_small, ); + _runtest(test_edn_nullptr, ); + _runtest(test_edn_large_enough_reuse, ryml2edn); + _runtest(test_edn_too_small_reuse, ryml2edn); + _runtest(test_edn_nullptr_reuse, ryml2edn); + _runtest(test_evt_large_enough, ); + _runtest(test_evt_too_small, ); + _runtest(test_evt_nullptr, ); + _runtest(test_evt_large_enough_reuse, ryml2evt); + _runtest(test_evt_too_small_reuse, ryml2evt); + _runtest(test_evt_nullptr_reuse, ryml2evt); + return tr; + } + + // happy path: large-enough destination string + TestResult test_edn_large_enough_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output; + output.resize(2 * edn.len); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_evt_large_enough_reuse(Ryml2Evt *ryml2evt) const + { + if(evt.empty()) return {}; + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::vector output; + output.resize(2 * expected_size(evt)); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); + CHECK_MSG((size_t)reqsize == expected_size(evt), "%d vs %zu", reqsize, expected_size(evt)); + CHECK(reqsize != 0); + output.resize(reqsize); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_large_enough() const + { + Ys2EdnScoped lib; + return test_edn_large_enough_reuse(lib.ryml2edn); + } + TestResult test_evt_large_enough() const + { + Ys2EvtScoped lib; + return test_evt_large_enough_reuse(lib.ryml2evt); + } + + // less-happy path: destination string not large enough + TestResult test_edn_too_small_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output = "?"; + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + CHECK(output != "?"); + output.resize(reqsize); + size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); + CHECK(getsize == reqsize); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_evt_too_small_reuse(Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::vector output; + output.resize(expected_size(evt)); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize == expected_size(evt)); + CHECK(reqsize != 0); + output.resize(reqsize); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize2 == reqsize); + output.resize(reqsize2); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_too_small() const + { + Ys2EdnScoped lib; + return test_edn_too_small_reuse(lib.ryml2edn); + } + TestResult test_evt_too_small() const + { + Ys2EvtScoped lib; + return test_evt_too_small_reuse(lib.ryml2evt); + } + + // safe calling with nullptr + TestResult test_edn_nullptr_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + return tr; + } + TestResult test_evt_nullptr_reuse(Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + CHECK(reqsize == expected_size(evt)); + CHECK(reqsize != 0); + std::vector output; + output.resize(reqsize); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize2 == reqsize); + CHECK(reqsize2 == output.size()); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_nullptr() const + { + Ys2EdnScoped lib; + return test_edn_nullptr_reuse(lib.ryml2edn); + } + TestResult test_evt_nullptr() const + { + Ys2EvtScoped lib; + return test_evt_nullptr_reuse(lib.ryml2evt); + } + +public: + + bool testeq(csubstr actual) const + { + const bool status = (actual == edn); + if(!status) + printf("------\n" + "FAIL:\n" + "input:[%zu]~~~%.*s~~~\n" + "expected:[%zu]~~~%.*s~~~\n" + "actual:[%zu]~~~%.*s~~~\n", + ys.len, (int)ys.len, ys.str, + edn.len, (int)edn.len, edn.str, + actual.len, (int)actual.len, actual.str); + return status; + } + + bool testeq(std::vector const& actual, csubstr parsed_source) const + { + int status = true; + size_t num_events_expected = evt.size(); + size_t num_ints_expected = expected_size(evt); + if(actual.size() != num_ints_expected) + { + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n" + "expected size:~~~%zu~~~\n" + "actual size:~~~%zu~~~\n", + (int)ys.len, ys.str, + num_ints_expected, + actual.size()); + status = false; + } + for(size_t i = 0, ie = 0; ie < num_events_expected; ++ie) + { + if(i >= actual.size()) + { + printf("fail: bad actual size. i=%zu vs %zu=actual.size()=\n", i, actual.size()); + status = false; + break; + } + #define _testcmp(fmt, cmp, ...) \ + printf("status=%d cmp=%d evt=%zu i=%zu: " fmt "\n", status, (cmp), ie, i, ## __VA_ARGS__); \ + status &= (cmp) + char actualbuf[100]; + char expectedbuf[100]; + size_t reqsize_actual = c4::bm2str(actual[i] & evt::MASK, actualbuf, sizeof(actualbuf)); + size_t reqsize_expected = c4::bm2str(evt[ie].flags & evt::MASK, expectedbuf, sizeof(expectedbuf)); + C4_CHECK(reqsize_actual < sizeof(actualbuf)); + C4_CHECK(reqsize_expected < sizeof(expectedbuf)); + _testcmp("exp=%d(%s) vs act=%d(%s)", evt[ie].flags == actual[i], evt[ie].flags, expectedbuf, actual[i], actualbuf); + status &= (evt[ie].flags == actual[i]); + if((evt[ie].flags & evt::HAS_STR) && (actual[i] & evt::HAS_STR)) + { + _testcmp(" exp=%d vs act=%d", evt[ie].str_start == actual[i + 1], evt[ie].str_start, actual[i + 1]); + _testcmp(" exp=%d vs act=%d", evt[ie].str_len == actual[i + 2], evt[ie].str_len, actual[i + 2]); + bool safeactual = (i + 2 < actual.size()) && (actual[i + 1] < (int)parsed_source.len && actual[i + 1] + actual[i + 2] <= (int)parsed_source.len); + bool safeexpected = (evt[ie].str_start < (int)parsed_source.len && evt[ie].str_start + evt[ie].str_len <= (int)parsed_source.len); + _testcmp(" safeactual=%d", safeactual, safeactual); + _testcmp(" safeactual=%d safeexpected=%d", safeactual == safeexpected, safeactual, safeexpected); + if(safeactual && safeexpected) + { + csubstr evtstr = parsed_source.sub((size_t)evt[ie].str_start, (size_t)evt[ie].str_len); + csubstr actualstr = parsed_source.sub((size_t)actual[i + 1], (size_t)actual[i + 2]); + _testcmp(" ref=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", + evt[ie].scalar == actualstr, + evt[ie].scalar.len, (int)evt[ie].scalar.len, evt[ie].scalar.str, + actualstr.len, (int)actualstr.len, actualstr.str); + if( ! evt[ie].needs_filter) + { + _testcmp(" exp=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", + evtstr == actualstr, + evtstr.len, (int)evtstr.len, evtstr.str, + actualstr.len, (int)actualstr.len, actualstr.str); + } + } + } + i += (actual[i] & evt::HAS_STR) ? 3 : 1; + } + if(!status) + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n", + (int)ys.len, ys.str); + return status; + } +}; + + +//----------------------------------------------------------------------------- + +namespace { +// make the declarations shorter +#define tc(ys, edn, ...) {ys, edn, std::vector(__VA_ARGS__)} +#define e(...) EvtWithScalar{__VA_ARGS__} +using namespace evt; +inline constexpr bool needs_filter = true; +const TestCase test_cases[] = { + // case ------------------------------------------------- + tc("a: 1", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "1"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 1, "a"), + e(VAL_|SCLR|PLAI, 3, 1, "1"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("say: 2 + 2", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "say"} +{:+ "=VAL", := "2 + 2"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 3, "say"), + e(VAL_|SCLR|PLAI, 5, 5, "2 + 2"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("𝄞: ✅", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "𝄞"} +{:+ "=VAL", := "✅"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 4, "𝄞"), + e(VAL_|SCLR|PLAI, 6, 3, "✅"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("[a, b, c]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "=VAL", := "c"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI, 4, 1, "b"), + e(VAL_|SCLR|PLAI, 7, 1, "c"), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------ + tc("[a: b]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "-MAP"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|FLOW), + e(VAL_|BMAP|FLOW), + e(KEY_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI, 4, 1, "b"), + e(EMAP), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------ + tc(R"(--- !yamlscript/v0 +foo: ! +- {x: y} +- [x, y] +- foo +- 'foo' +- "foo" +- | + foo +- > + foo +- [1, 2, true, false, null] +- &anchor-1 !tag-1 foobar +--- +another: doc +)", + R"(( +{:+ "+MAP", :! "yamlscript/v0"} +{:+ "=VAL", := "foo"} +{:+ "+SEQ", :! ""} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-MAP"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-SEQ"} +{:+ "=VAL", := "foo"} +{:+ "=VAL", :' "foo"} +{:+ "=VAL", :$ "foo"} +{:+ "=VAL", :| "foo\n"} +{:+ "=VAL", :> "foo\n"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "1"} +{:+ "=VAL", := "2"} +{:+ "=VAL", := "true"} +{:+ "=VAL", := "false"} +{:+ "=VAL", := "null"} +{:+ "-SEQ"} +{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} +{:+ "-SEQ"} +{:+ "-MAP"} +{:+ "-DOC"} +{:+ "+DOC"} +{:+ "+MAP"} +{:+ "=VAL", := "another"} +{:+ "=VAL", := "doc"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC|EXPL), + e(VAL_|TAG_, 5, 13, "yamlscript/v0"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 19, 3, "foo"), + e(VAL_|TAG_, 25, 0, ""), + e(VAL_|BSEQ|BLCK), + e(VAL_|BMAP|FLOW), + e(KEY_|SCLR|PLAI, 29, 1, "x"), + e(VAL_|SCLR|PLAI, 32, 1, "y"), + e(EMAP), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 38, 1, "x"), + e(VAL_|SCLR|PLAI, 41, 1, "y"), + e(ESEQ), + e(VAL_|SCLR|PLAI, 46, 3, "foo"), + e(VAL_|SCLR|SQUO, 53, 3, "foo"), + e(VAL_|SCLR|DQUO, 61, 3, "foo"), + e(VAL_|SCLR|LITL, 70, 4, "foo\n", needs_filter), + e(VAL_|SCLR|FOLD, 80, 4, "foo\n", needs_filter), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 89, 1, "1"), + e(VAL_|SCLR|PLAI, 92, 1, "2"), + e(VAL_|SCLR|PLAI, 95, 4, "true"), + e(VAL_|SCLR|PLAI, 101, 5, "false"), + e(VAL_|SCLR|PLAI, 108, 4, "null"), + e(ESEQ), + e(VAL_|TAG_, 127, 5, "tag-1"), + e(VAL_|ANCH, 117, 8, "anchor-1"), + e(VAL_|SCLR|PLAI, 133, 6, "foobar"), + e(ESEQ), + e(EMAP), + e(EDOC), + e(BDOC|EXPL), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 144, 7, "another"), + e(VAL_|SCLR|PLAI, 153, 3, "doc"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"(plain: well + a + b + c +squo: 'single''quote' +dquo: "x\t\ny" +lit: | + X + Y + Z +fold: > + U + V + W +)", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "plain"} +{:+ "=VAL", := "well a b c"} +{:+ "=VAL", := "squo"} +{:+ "=VAL", :' "single'quote"} +{:+ "=VAL", := "dquo"} +{:+ "=VAL", :$ "x\t\ny"} +{:+ "=VAL", := "lit"} +{:+ "=VAL", :| "X\nY\nZ\n"} +{:+ "=VAL", := "fold"} +{:+ "=VAL", :> "U V W\n"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 5, "plain"), + e(VAL_|SCLR|PLAI, 7, 10, "well a b c"), + e(KEY_|SCLR|PLAI, 24, 4, "squo"), + e(VAL_|SCLR|SQUO, 31, 12, "single'quote", needs_filter), + e(KEY_|SCLR|PLAI, 46, 4, "dquo"), + e(VAL_|SCLR|DQUO, 53, 4, "x\t\ny", needs_filter), + e(KEY_|SCLR|PLAI, 61, 3, "lit"), + e(VAL_|SCLR|LITL, 68, 6, "X\nY\nZ\n", needs_filter), + e(KEY_|SCLR|PLAI, 89, 4, "fold"), + e(VAL_|SCLR|FOLD, 97, 6, "U V W\n", needs_filter), + e(EMAP), + e(EDOC), + e(ESTR), + }), +}; +} // namespace + +int main() +{ + Ys2EdnScoped ys2edn; + Ys2EvtScoped ys2evt; + TestResult total = {}; + size_t failed_cases = {}; + size_t num_cases = C4_COUNTOF(test_cases); + for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) + { + printf("-----------------------------------------\n" + "case %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); + const TestResult tr = test_cases[i].test(ys2edn.ryml2edn, ys2evt.ryml2evt); + total.add(tr); + failed_cases += (!tr); + printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + } + printf("assertions: %u/%u pass %u/%u fail\n", total.num_assertions - total.num_failed_assertions, total.num_assertions, total.num_failed_assertions, total.num_assertions); + printf("tests: %u/%u pass %u/%u fail\n", total.num_tests - total.num_failed_tests, total.num_tests, total.num_failed_tests, total.num_tests); + printf("cases: %zu/%zu pass %zu/%zu fail\n", num_cases-failed_cases, num_cases, failed_cases, num_cases); + if(total) + printf("TESTS SUCCEED!\n"); + return total ? 0 : -1; +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/Evt.java b/rapidyaml/src/main/java/org/rapidyaml/Evt.java new file mode 100644 index 000000000..4f06211f3 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/Evt.java @@ -0,0 +1,37 @@ +package org.rapidyaml; + +public class Evt { + // --------------------- + // structure flags + public static final int KEY_ = 1 << 0; // as key + public static final int VAL_ = 1 << 1; // as value + public static final int SCLR = 1 << 2; // =VAL + public static final int BSEQ = 1 << 3; // +SEQ + public static final int ESEQ = 1 << 4; // -SEQ + public static final int BMAP = 1 << 5; // +MAP + public static final int EMAP = 1 << 6; // -MAP + public static final int ALIA = 1 << 7; // ref + public static final int ANCH = 1 << 8; // anchor + public static final int TAG_ = 1 << 9; // tag + // --------------------- + // style flags + public static final int PLAI = 1 << 10; // : (plain scalar) + public static final int SQUO = 1 << 11; // ' (single-quoted scalar) + public static final int DQUO = 1 << 12; // " (double-quoted scalar) + public static final int LITL = 1 << 13; // | (block literal scalar) + public static final int FOLD = 1 << 14; // > (block folded scalar) + public static final int FLOW = 1 << 15; // flow container: [] for seqs or {} for maps + public static final int BLCK = 1 << 16; // block container + // --------------------- + // document flags + public static final int BDOC = 1 << 17; // +DOC + public static final int EDOC = 1 << 18; // -DOC + public static final int EXPL = 1 << 21; // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) + public static final int BSTR = 1 << 19; // +STR + public static final int ESTR = 1 << 20; // -STR + // --------------------- + // utility flags + public static final int LAST = ESTR; + public static final int MASK = ((LAST << 1) - 1); + public static final int HAS_STR = SCLR|ALIA|ANCH|TAG_; +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index ccf7d4370..c36eb768f 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -6,7 +6,6 @@ /** * Interface with the shared librapidyaml library - * */ public class Rapidyaml { public static String RAPIDYAML_VERSION = "0.7.2"; @@ -19,13 +18,20 @@ private native int ys2edn_parse(long ryml2edn, String filename, private native int ys2edn_retry_get( long ryml2edn, byte[] edn, int edn_size ); - private final long ryml2edn; + private native long ys2evt_init(); + private native void ys2evt_destroy(long ryml2evt); + private native int ys2evt_parse(long ryml2evt, String filename, + byte[] ys, int ys_length, + int[] evt, int evt_length); + private final long ryml2evt; + public Rapidyaml() { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); this.ryml2edn = this.ys2edn_init(); + this.ryml2evt = this.ys2evt_init(); } // Likely bad idea to implement finalize: @@ -35,13 +41,14 @@ public Rapidyaml() { protected void finalize() throws Throwable { try { this.ys2edn_destroy(this.ryml2edn); + this.ys2evt_destroy(this.ryml2evt); } finally { super.finalize(); } } - public String parseYS(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { String filename = "yamlscript"; // fixme byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); int edn_size = 10 * src.length; @@ -58,4 +65,10 @@ public String parseYS(String srcstr) throws RuntimeException, org.rapidyaml.Yaml String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); return ret; } + + public int parseYsToEvt(byte[] src, int[] evts) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + String filename = "yamlscript"; // fixme + int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); + return required_size; + } } diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index 48e9eaf83..db62cc2bc 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -1,6 +1,8 @@ package org.rapidyaml; import org.rapidyaml.*; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; @@ -32,10 +34,110 @@ private void testEdn_(String ys, String expected) { Rapidyaml rapidyaml = new Rapidyaml(); try { - String actual = rapidyaml.parseYS(ys); + String actual = rapidyaml.parseYsToEdn(ys); try { - assertEquals(expected.length(), actual.length()); assertEquals(expected, actual); + assertEquals(expected.length(), actual.length()); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + catch (YamlParseErrorException e) { + fail("parse error:\n" + e.getMessage()); + } + } + + // the result is an array of integers, but we use this to simplify + // running the tests + private class ExpectedEvent + { + int flags; + int str_start; + int str_len; + String str; + ExpectedEvent(int flags) + { + this.flags = flags; + this.str_start = 0; + this.str_len = 0; + this.str = ""; + } + ExpectedEvent(int flags, int str_start, int str_len, String str) + { + this.flags = flags; + this.str_start = str_start; + this.str_len = str_len; + this.str = str; + } + int required_size() { return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; } + }; + private static int required_size_(ExpectedEvent[] evts) + { + int sz = 0; + for(int i = 0; i < evts.length; ++i) { + sz += evts[i].required_size(); + } + return sz; + } + + private void testEvt_(String ys, ExpectedEvent[] expected) + { + Rapidyaml rapidyaml = new Rapidyaml(); + try { + int[] actual = new int[2 * required_size_(expected)]; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + int numEvts = rapidyaml.parseYsToEvt(src, actual); + assertTrue(numEvts < actual.length); + try { + int ia = 0; + int ie = 0; + int status = 1; + while(true) { + if((ia < numEvts) != (ie < expected.length)) { + status = 0; + break; + } + if(ia >= numEvts) + break; + if(ie >= expected.length) + break; + int cmp = 1; + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; + if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { + cmp &= (ia + 2 < numEvts) ? 1 : 0; + if(cmp != 0) { + cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; + cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + if(cmp != 0) { + cmp &= (actual[ia + 1] >= 0) ? 1 : 0; + cmp &= (actual[ia + 2] >= 0) ? 1 : 0; + cmp &= (actual[ia + 1] + actual[ia + 2] <= src.length) ? 1 : 0; + if(cmp != 0) { + String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); + cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + } + else { + System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); + } + } + } + } + System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); + status &= cmp; + ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; + ++ie; + } + if(required_size_(expected) != numEvts) + status = 0; + assertEquals(1, status); } catch (Exception e) { System.err.println("expected:"); @@ -52,9 +154,8 @@ private void testEdn_(String ys, String expected) public void testPlainMap() { - testEdn_( - "a: 1" - , + String ys = "a: 1"; + testEdn_(ys, "(\n" + "{:+ \"+MAP\"}\n" + "{:+ \"=VAL\", := \"a\"}\n" + @@ -63,13 +164,23 @@ public void testPlainMap() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 1, "a"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 3, 1, "1"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testUtf8() { - testEdn_( - "𝄞: ✅" - , + String ys = "𝄞: ✅"; + testEdn_(ys, "(\n" + "{:+ \"+MAP\"}\n" + "{:+ \"=VAL\", := \"𝄞\"}\n" + @@ -78,11 +189,22 @@ public void testUtf8() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "𝄞"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "✅"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testLargeCase() { - testEdn_( + String ys = "--- !yamlscript/v0\n" + "foo: !\n" + "- {x: y}\n" + "- [x, y]\n" + @@ -96,10 +218,10 @@ public void testLargeCase() "- [1, 2, true, false, null]\n" + "- &anchor-1 !tag-1 foobar\n" + "---\n" + - "another: doc\n" - , + "another: doc\n"; + testEdn_(ys, "(\n" + - "{:+ \"+MAP\"}\n" + + "{:+ \"+MAP\", :! \"yamlscript/v0\"}\n" + "{:+ \"=VAL\", := \"foo\"}\n" + "{:+ \"+SEQ\", :! \"\"}\n" + "{:+ \"+MAP\", :flow true}\n" + @@ -134,6 +256,103 @@ public void testLargeCase() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC|Evt.EXPL), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 5, 13, "yamlscript/v0"), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 25, 0, ""), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.FLOW), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 29, 1, "x"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 32, 1, "y"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 38, 1, "x"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 41, 1, "y"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 46, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 53, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 61, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 70, 4, "foo\n"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 80, 4, "foo\n"), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 89, 1, "1"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 92, 1, "2"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 95, 4, "true"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 101, 5, "false"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 108, 4, "null"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 127, 5, "tag-1"), + new ExpectedEvent(Evt.VAL_|Evt.ANCH, 117, 8, "anchor-1"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 133, 6, "foobar"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.BDOC|Evt.EXPL), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 144, 7, "another"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 153, 3, "doc"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testFilterCase() + { + String ys = "" + + "plain: well\n" + + " a\n" + + " b\n" + + " c\n" + + "squo: 'single''quote'\n" + + "dquo: \"x\\t\\ny\"\n" + + "lit: |\n" + + " X\n" + + " Y\n" + + " Z\n" + + "fold: >\n" + + " U\n" + + " V\n" + + " W\n"; + testEdn_(ys, + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"plain\"}\n" + + "{:+ \"=VAL\", := \"well a b c\"}\n" + + "{:+ \"=VAL\", := \"squo\"}\n" + + "{:+ \"=VAL\", :' \"single'quote\"}\n" + + "{:+ \"=VAL\", := \"dquo\"}\n" + + "{:+ \"=VAL\", :$ \"x\\t\\ny\"}\n" + + "{:+ \"=VAL\", := \"lit\"}\n" + + "{:+ \"=VAL\", :| \"X\\nY\\nZ\\n\"}\n" + + "{:+ \"=VAL\", := \"fold\"}\n" + + "{:+ \"=VAL\", :> \"U V W\\n\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n"); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 5, "plain"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 7, 10, "well a b c"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 24, 4, "squo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 31, 12, "single'quote"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 46, 4, "dquo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 53, 4, "x\t\ny"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 61, 3, "lit"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 68, 6, "X\nY\nZ\n"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 89, 4, "fold"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 97, 6, "U V W\n"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testFailure() @@ -142,7 +361,7 @@ public void testFailure() String ys = ": : : :"; boolean gotit = false; try { - rapidyaml.parseYS(ys); + rapidyaml.parseYsToEdn(ys); } catch(YamlParseErrorException e) { gotit = true; From 6e721f6e500d2904a9c0fc9dfe1d19f9d8f83349 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 15 Feb 2025 15:23:42 +0000 Subject: [PATCH 08/29] rapidyaml: Update to 0.8.0 --- common/vars.mk | 2 +- core/deps.edn | 2 +- core/project.clj | 2 +- rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/vars.mk b/common/vars.mk index ef001a228..2bf93f6f1 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -221,7 +221,7 @@ RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz #------------------------------------------------------------------------------ RAPIDYAML := $(ROOT)/rapidyaml -RAPIDYAML_VERSION := 0.7.2 +RAPIDYAML_VERSION := 0.8.0 RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml RAPIDYAML_BUILD_TYPE := Release diff --git a/core/deps.edn b/core/deps.edn index 5637484a0..2aa96efb0 100644 --- a/core/deps.edn +++ b/core/deps.edn @@ -5,7 +5,7 @@ org.clojure/data.json {:mvn/version "2.4.0"}, clj-commons/clj-yaml {:mvn/version "1.0.27"}, org.flatland/ordered {:mvn/version "1.15.11"}, - org.rapidyaml/rapidyaml {:mvn/version "0.7.2"}, + org.rapidyaml/rapidyaml {:mvn/version "0.8.0"}, org.snakeyaml/snakeyaml-engine {:mvn/version "2.7"}, babashka/babashka.pods {:mvn/version "0.2.0"}, babashka/fs {:mvn/version "0.5.20"}, diff --git a/core/project.clj b/core/project.clj index dbefdf0fd..414c061b0 100644 --- a/core/project.clj +++ b/core/project.clj @@ -22,7 +22,7 @@ [org.clojure/data.json "2.4.0"] [clj-commons/clj-yaml "1.0.27"] [org.flatland/ordered "1.15.11"] - [org.rapidyaml/rapidyaml "0.7.2"] + [org.rapidyaml/rapidyaml "0.8.0"] [org.snakeyaml/snakeyaml-engine "2.7"] [babashka/babashka.pods "0.2.0"] [babashka/fs "0.5.20"] diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index c36eb768f..26e10fb0c 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -8,7 +8,7 @@ * Interface with the shared librapidyaml library */ public class Rapidyaml { - public static String RAPIDYAML_VERSION = "0.7.2"; + public static String RAPIDYAML_VERSION = "0.8.0"; private native long ys2edn_init(); private native void ys2edn_destroy(long ryml2edn); From 1fbf6a29c5b93f05f09673aa14465f74c6f759ce Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 17 Feb 2025 18:38:35 +0000 Subject: [PATCH 09/29] rapidyaml: YS integration updates --- common/vars.mk | 11 +- rapidyaml/Makefile | 6 +- rapidyaml/native/CMakeLists.txt | 1 + rapidyaml/native/Makefile | 18 +- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 123 +++++++++---- rapidyaml/native/ysparse_common.hpp | 44 +++++ rapidyaml/native/ysparse_edn.cpp | 33 +--- rapidyaml/native/ysparse_edn.hpp | 16 +- rapidyaml/native/ysparse_edn_handler.hpp | 14 ++ rapidyaml/native/ysparse_evt.cpp | 33 +--- rapidyaml/native/ysparse_evt.hpp | 16 +- rapidyaml/native/ysparse_evt_handler.hpp | 48 ++++- rapidyaml/native/ysparse_test.cpp | 173 +++++++++++++++++- .../main/java/org/rapidyaml/Rapidyaml.java | 20 +- .../java/org/rapidyaml/cmp/CmpEdnEvt.java | 52 ++++++ .../main/java/org/rapidyaml/cmp/manifest.mf | 1 + .../src/main/java/org/rapidyaml/cmp/run.sh | 17 ++ .../java/org/rapidyaml/RapidyamlTest.java | 73 +++++++- 18 files changed, 543 insertions(+), 156 deletions(-) create mode 100644 rapidyaml/native/ysparse_common.hpp create mode 100644 rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java create mode 100644 rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf create mode 100755 rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh diff --git a/common/vars.mk b/common/vars.mk index 2bf93f6f1..f028ec0ab 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -222,14 +222,17 @@ RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz RAPIDYAML := $(ROOT)/rapidyaml RAPIDYAML_VERSION := 0.8.0 -RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) +#RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) +RAPIDYAML_TAG ?= 8c37616378aefd376690a19459c31a56ce596b5e RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml -RAPIDYAML_BUILD_TYPE := Release -RAPIDYAML_DBG := 0 -RAPIDYAML_TIMED := 0 +RAPIDYAML_BUILD_TYPE ?= Release +RAPIDYAML_DBG ?= 0 +RAPIDYAML_TIMED ?= 0 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java +RAPIDYAML_JNI_H := $(ROOT)/rapidyaml/native/org_rapidyaml_Rapidyaml.h RAPIDYAML_SO := $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) RAPIDYAML_LIB := $(ROOT)/rapidyaml/native/librapidyaml.$(DOTLIB) RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index e29ba8c02..c01c44d09 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -47,11 +47,13 @@ realclean:: clean sysclean:: $(RM) -r $(MAVEN_REPOSITORY)/org/rapidyaml +jni: $(RAPIDYAML_JNI_H) + #------------------------------------------------------------------------------ $(RAPIDYAML_SO): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ -$(RAPIDYAML_LIB): +$(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ $(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED) @@ -68,5 +70,3 @@ $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) $(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) $(MAKE) -C native $@ - -crl: $(RAPIDYAML_CLASS) diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 61fd11e3c..4f8524e97 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -22,6 +22,7 @@ add_library(${libname} org_rapidyaml_Rapidyaml.cpp # # ysparse files + ysparse_common.hpp ysparse_edn_handler.hpp ysparse_edn_handler.cpp ysparse_edn.hpp diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index 91dd05ea8..f86b2e630 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -9,7 +9,7 @@ include $(COMMON)/python.mk # https://stackoverflow.com/questions/24493337/linking-static-library-with-jni THIS_DIR := $(shell pwd) -BDIR := $(THIS_DIR)/_build/$(BUILD_TYPE)-shared$(SHARED)-timed$(TIMED)-dbg$(DBG) +BDIR := $(THIS_DIR)/_build/$(RAPIDYAML_BUILD_TYPE)-shared$(RAPIDYAML_SHARED)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) RAPIDYAML_DEPS := \ Makefile \ @@ -18,7 +18,8 @@ RAPIDYAML_DEPS := \ $(RAPIDYAML_JNI_H) \ $(wildcard ./*pp) \ -CMK_ENV := +CMK_ENV ?= +CMK_FLAGS_EXTRA ?= CMK_FLAGS := \ -D CMAKE_BUILD_TYPE=$(RAPIDYAML_BUILD_TYPE) \ -D YSPARSE_TIMED=$(RAPIDYAML_TIMED) \ @@ -50,22 +51,25 @@ realclean:: clean #------------------------------------------------------------------------------ + +$(RAPIDYAML_LIB): rapidyaml build-static: $(RAPIDYAML_LIB) test-static: build-static - cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml-test-run + $(CMK_ENV) cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml-test-run $(RAPIDYAML_LIB): $(RAPIDYAML_DEPS) - cmake -S . -B $(BDIR)-static $(CMK_FLAGS) -D BUILD_SHARED_LIBS=OFF + $(CMK_ENV) cmake -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml cp -fv $(BDIR)-static/*.a $@ +$(RAPIDYAML_SO): rapidyaml build-shared: $(RAPIDYAML_SO) test-shared: build-shared cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml-test-run $(RAPIDYAML_SO): $(RAPIDYAML_DEPS) - cmake -S . -B $(BDIR)-shared $(CMK_FLAGS) -D BUILD_SHARED_LIBS=ON - cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml + $(CMK_ENV) cmake -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON + $(CMK_ENV) cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so -$(RAPIDYAML_H): $(RAPIDYAML_JAVA) +$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 63fa9efd1..48d3d8b87 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -85,6 +85,22 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) ys2evt_destroy((Ryml2Evt*)obj); } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2edn_retry_get + * Signature: (Ljava/lang/Object;[BI)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, + jlong obj, + jbyteArray dst, jint dst_len) +{ + jboolean dst_is_copy; + jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); + int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); + env->ReleaseByteArrayElements(dst, dst_, 0); + return rc; +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_parse @@ -96,10 +112,27 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, jbyteArray src, jint src_len, jbyteArray dst, jint dst_len) { - jboolean src_is_copy, dst_is_copy; - jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); - jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); - const char *filename = env->GetStringUTFChars(jfilename, 0); + TIMED_SECTION("jni_ys2edn_parse", (size_type)src_len); + jbyte* src_ = nullptr; + jbyte* dst_ = nullptr; + jboolean src_is_copy = false; + jboolean dst_is_copy = false; + const char *filename = nullptr; + { + TIMED_SECTION("jni_ys2edn_parse/get_jni_data", (size_type)src_len); + { + TIMED_SECTION("jni_ys2edn_parse/GetByteArray(src)"); + src_ = env->GetByteArrayElements(src, &src_is_copy); + } + { + TIMED_SECTION("jni_ys2edn_parse/GetByteArray(dst)"); + dst_ = env->GetByteArrayElements(dst, &dst_is_copy); + } + { + TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); + filename = env->GetStringUTFChars(jfilename, 0); + } + } int rc = 0; try { @@ -107,13 +140,16 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, (char*)src_, src_len, (char*)dst_, dst_len); } - catch (Ryml2EdnParseError const& exc) + catch (YsParseError const& exc) { throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); } - env->ReleaseByteArrayElements(src, src_, 0); - env->ReleaseByteArrayElements(dst, dst_, 0); - env->ReleaseStringUTFChars(jfilename, filename); + { + TIMED_SECTION("jni_ys2edn_parse/release"); + env->ReleaseByteArrayElements(src, src_, 0); + env->ReleaseByteArrayElements(dst, dst_, 0); + env->ReleaseStringUTFChars(jfilename, filename); + } return rc; } @@ -128,10 +164,36 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, jbyteArray src, jint src_len, jintArray dst, jint dst_len) { - jboolean src_is_copy, dst_is_copy; - jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); - int* dst_ = env->GetIntArrayElements(dst, &dst_is_copy); - const char *filename = env->GetStringUTFChars(jfilename, 0); + TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + jbyte* src_ = nullptr; + int* dst_ = nullptr; + const char *filename = nullptr; + jboolean dst_is_copy = false; + jboolean src_is_copy = false; + { + TIMED_SECTION("jni_ys2evt_parse/get_jni_data", (size_type)src_len); + { + TIMED_SECTION("jni_ys2evt_parse/GetByteArray(src)"); + src_ = env->GetByteArrayElements(src, &src_is_copy); + } + { + // TODO this is __S__L__O__W__ + // + // the problem is with GetIntArrayElements(). we should + // use GetDirectBufferAddress(), but that requires a ByteBuffer->jobject + // instead of a int[]->jintArray + // + // see: + // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time + // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string + TIMED_SECTION("jni_ys2evt_parse/GetIntArray(dst)"); + dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + } + { + TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); + filename = env->GetStringUTFChars(jfilename, 0); + } + } int rc = 0; try { @@ -139,29 +201,26 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, (char*)src_, src_len, dst_, dst_len); } - catch (Ryml2EvtParseError const& exc) + catch (YsParseError const& exc) { throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); } - env->ReleaseByteArrayElements(src, src_, 0); - env->ReleaseIntArrayElements(dst, dst_, 0); - env->ReleaseStringUTFChars(jfilename, filename); - return rc; -} - -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_retry_get - * Signature: (Ljava/lang/Object;[BI)I - */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, - jlong obj, - jbyteArray dst, jint dst_len) -{ - jboolean src_is_copy, dst_is_copy; - jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); - int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); - env->ReleaseByteArrayElements(dst, dst_, 0); + { + TIMED_SECTION("jni_ys2evt_parse/release"); + { + TIMED_SECTION("jni_ys2evt_parse/ReleaseByteArray(src)"); + env->ReleaseByteArrayElements(src, src_, 0); + } + { + // TODO __S__L__O__W__ + TIMED_SECTION("jni_ys2evt_parse/ReleaseIntArray(dst)"); + env->ReleaseIntArrayElements(dst, dst_, 0); + } + { + TIMED_SECTION("jni_ys2evt_parse/ReleaseStringUTFChars()"); + env->ReleaseStringUTFChars(jfilename, filename); + } + } return rc; } diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp new file mode 100644 index 000000000..9026f951e --- /dev/null +++ b/rapidyaml/native/ysparse_common.hpp @@ -0,0 +1,44 @@ +#pragma once +#ifndef YSPARSE_COMMON_HPP_ +#define YSPARSE_COMMON_HPP_ + +#include + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +using size_type = int; + +struct YsParseError : public std::exception +{ + ryml::Location location; + std::string msg; + const char* what() const noexcept override { return msg.c_str(); } +}; + +#ifndef YSPARSE_TIMED +#define TIMED_SECTION(...) +#else +#include +#define TIMED_SECTION(...) timed_section C4_XCAT(ts, __LINE__)(__VA_ARGS__) +struct timed_section +{ + using myclock = std::chrono::steady_clock; + ryml::csubstr name; + size_type len; + myclock::time_point start; + timed_section(ryml::csubstr n, size_type len_=0) : name(n), len(len_), start(myclock::now()) {} + ~timed_section() + { + const std::chrono::duration t = myclock::now() - start; + fprintf(stderr, "%.6fms: %.*s", t.count(), (int)name.len, name.str); + if(len) + fprintf(stderr, " %.3fMB/s", (double)len / t.count() * 1.e-3); + fprintf(stderr, "\n"); + } +}; +#endif // YSPARSE_TIMED + +#endif // YSPARSE_COMMON_HPP_ diff --git a/rapidyaml/native/ysparse_edn.cpp b/rapidyaml/native/ysparse_edn.cpp index b074c4848..586c3a79e 100644 --- a/rapidyaml/native/ysparse_edn.cpp +++ b/rapidyaml/native/ysparse_edn.cpp @@ -1,34 +1,7 @@ #include "ysparse_edn.hpp" -#include - -namespace ryml { -using namespace c4; -using namespace c4::yml; -} // namespace ryml using namespace ryml; -#ifndef YSPARSE_TIMED -#define TIMED_SECTION(name) -#else -#include -#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) -struct timed_section -{ - using myclock = std::chrono::steady_clock; - using fmsecs = std::chrono::duration; - csubstr name; - myclock::time_point start; - fmsecs since() const { return myclock::now() - start; } - timed_section(csubstr n) : name(n), start(myclock::now()) {} - ~timed_section() - { - fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); - } -}; -#endif - - #if defined(__cplusplus) extern "C" { #endif @@ -39,7 +12,7 @@ extern "C" { namespace { C4_NORETURN void ys2edn_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) { - Ryml2EdnParseError exc; + YsParseError exc; exc.location = location; exc.msg.assign(msg, msg_len); throw exc; @@ -70,7 +43,7 @@ RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, char *ys, size_type ys_size, char *edn, size_type edn_size) { - TIMED_SECTION("ys2edn_parse"); + TIMED_SECTION("ys2edn_parse", ys_size); csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); { @@ -79,7 +52,7 @@ RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, ryml2edn->m_handler.reserve(ys_size > edn_size ? 3 * ys_size : edn_size, 256u); } { - TIMED_SECTION("parse_in_place"); + TIMED_SECTION("parse_in_place", ys_size); ryml2edn->m_parser.parse_in_place_ev(filename_, ys_); } return ys2edn_retry_get(ryml2edn, edn, edn_size); diff --git a/rapidyaml/native/ysparse_edn.hpp b/rapidyaml/native/ysparse_edn.hpp index 5a5878967..3dfe73606 100644 --- a/rapidyaml/native/ysparse_edn.hpp +++ b/rapidyaml/native/ysparse_edn.hpp @@ -2,20 +2,13 @@ #ifndef YSPARSE_EDN_HPP_ #define YSPARSE_EDN_HPP_ -#include #include "ysparse_edn_handler.hpp" - -namespace ryml { -using namespace c4; -using namespace c4::yml; -} // namespace ryml +#include "ysparse_common.hpp" #if defined(__cplusplus) extern "C" { #endif -using size_type = int; - struct RYML_EXPORT Ryml2Edn { ys::EventHandlerEdn::EventSink m_sink; @@ -34,13 +27,6 @@ struct RYML_EXPORT Ryml2Edn } }; -struct RYML_EXPORT Ryml2EdnParseError : public std::exception -{ - c4::yml::Location location; - std::string msg; - const char* what() const noexcept override { return msg.c_str(); } -}; - //----------------------------------------------------------------------------- diff --git a/rapidyaml/native/ysparse_edn_handler.hpp b/rapidyaml/native/ysparse_edn_handler.hpp index 5497fc609..89c36813f 100644 --- a/rapidyaml/native/ysparse_edn_handler.hpp +++ b/rapidyaml/native/ysparse_edn_handler.hpp @@ -343,6 +343,18 @@ struct EventHandlerEdn : public c4::yml::EventHandlerStack')) + result = result.offs(1, 1); _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); _RYML_CB_CHECK(m_stack.m_callbacks, result.str); return result; diff --git a/rapidyaml/native/ysparse_evt.cpp b/rapidyaml/native/ysparse_evt.cpp index f866aec75..9ef064b3f 100644 --- a/rapidyaml/native/ysparse_evt.cpp +++ b/rapidyaml/native/ysparse_evt.cpp @@ -1,34 +1,7 @@ #include "ysparse_evt.hpp" -#include - -namespace ryml { -using namespace c4; -using namespace c4::yml; -} // namespace ryml using namespace ryml; -#ifndef YS2PARSE_TIMED -#define TIMED_SECTION(name) -#else -#include -#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) -struct timed_section -{ - using myclock = std::chrono::steady_clock; - using fmsecs = std::chrono::duration; - csubstr name; - myclock::time_point start; - fmsecs since() const { return myclock::now() - start; } - timed_section(csubstr n) : name(n), start(myclock::now()) {} - ~timed_section() - { - fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); - } -}; -#endif - - #if defined(__cplusplus) extern "C" { #endif @@ -39,7 +12,7 @@ extern "C" { namespace { C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) { - Ryml2EvtParseError exc; + YsParseError exc; exc.location = location; exc.msg.assign(msg, msg_len); throw exc; @@ -70,7 +43,7 @@ RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, char *ys, size_type ys_size, evt::DataType *events, size_type evt_size) { - TIMED_SECTION("ys2evt_parse"); + TIMED_SECTION("ys2evt_parse", ys_size); csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); { @@ -79,7 +52,7 @@ RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, ryml2evt->m_handler.reserve(256u); } { - TIMED_SECTION("parse_in_place"); + TIMED_SECTION("parse_in_place", ys_size); ryml2evt->m_parser.parse_in_place_ev(filename_, ys_); } return (size_type)ryml2evt->m_handler.m_evt_curr; diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp index 7712f489f..cc231ae1a 100644 --- a/rapidyaml/native/ysparse_evt.hpp +++ b/rapidyaml/native/ysparse_evt.hpp @@ -2,20 +2,13 @@ #ifndef YSPARSE_EVT_HPP_ #define YSPARSE_EVT_HPP_ -#include #include "ysparse_evt_handler.hpp" - -namespace ryml { -using namespace c4; -using namespace c4::yml; -} // namespace ryml +#include "ysparse_common.hpp" #if defined(__cplusplus) extern "C" { #endif -using size_type = int; - struct RYML_EXPORT Ryml2Evt { ys::EventHandlerEvt m_handler; @@ -32,13 +25,6 @@ struct RYML_EXPORT Ryml2Evt } }; -struct RYML_EXPORT Ryml2EvtParseError : public std::exception -{ - c4::yml::Location location; - std::string msg; - const char* what() const noexcept override { return msg.c_str(); } -}; - //----------------------------------------------------------------------------- diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp index f00d7fe58..5892492d1 100644 --- a/rapidyaml/native/ysparse_evt_handler.hpp +++ b/rapidyaml/native/ysparse_evt_handler.hpp @@ -367,6 +367,27 @@ struct EventHandlerEvt : public c4::yml::EventHandlerStackevt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + csubstr _transform_directive(csubstr tag, substr output) { - if(tag.begins_with('!')) + if(tag.begins_with("!!")) + { + return tag; + } + else if(tag.begins_with('!')) { if(c4::yml::is_custom_tag(tag)) { diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 40a7e5a7a..5688d1ebf 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -57,6 +57,7 @@ struct Ys2EvtScoped }; +static bool showcmp = false; struct TestResult { uint32_t num_assertions; @@ -318,17 +319,18 @@ struct TestCase int status = true; size_t num_events_expected = evt.size(); size_t num_ints_expected = expected_size(evt); + bool same_size = true; if(actual.size() != num_ints_expected) { printf("------\n" - "FAIL:\n" + "FAIL: different size\n" "input:~~~%.*s~~~\n" "expected size:~~~%zu~~~\n" "actual size:~~~%zu~~~\n", (int)ys.len, ys.str, num_ints_expected, actual.size()); - status = false; + same_size = false; } for(size_t i = 0, ie = 0; ie < num_events_expected; ++ie) { @@ -339,7 +341,7 @@ struct TestCase break; } #define _testcmp(fmt, cmp, ...) \ - printf("status=%d cmp=%d evt=%zu i=%zu: " fmt "\n", status, (cmp), ie, i, ## __VA_ARGS__); \ + if(showcmp) { printf("status=%d cmp=%d evt=%zu i=%zu: " fmt "\n", status, (cmp), ie, i, ## __VA_ARGS__); } \ status &= (cmp) char actualbuf[100]; char expectedbuf[100]; @@ -381,7 +383,7 @@ struct TestCase "FAIL:\n" "input:~~~%.*s~~~\n", (int)ys.len, ys.str); - return status; + return status && same_size; } }; @@ -646,6 +648,169 @@ fold: > e(EDOC), e(ESTR), }), + // case ------------------------------------------------- + tc("- !!seq []", + R"(( +{:+ "+SEQ"} +{:+ "+SEQ", :! "tag:yaml.org,2002:seq", :flow true} +{:+ "-SEQ"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|BLCK), + e(VAL_|TAG_, 2, 5, "!!seq"), + e(VAL_|BSEQ|FLOW), + e(ESEQ), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"_(defn run(prompt session=nil): + when session: + write session _ :append true: |+ + Q: $(orig-prompt:trim) + A ($api-model): + $(answer:trim) +)_", + R"_(( +{:+ "+MAP"} +{:+ "=VAL", := "defn run(prompt session=nil)"} +{:+ "+MAP"} +{:+ "=VAL", := "when session"} +{:+ "+MAP"} +{:+ "=VAL", := "write session _ :append true"} +{:+ "=VAL", :| "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)_", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 28, "defn run(prompt session=nil)"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 32, 12, "when session"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 50, 28, "write session _ :append true"), + e(VAL_|SCLR|LITL, 83, 54, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n", needs_filter), + e(EMAP), + e(EMAP), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"_(#!/usr/bin/env ys-0 + +defn run(prompt session=nil): + session-text =: + when session && session:fs-e: + + answer =: + cond: + api-model =~ /^dall-e/: + openai-image(prompt).data.0.url + api-model.in?(anthropic-models): + anthropic(prompt):anthropic-message:format + api-model.in?(groq-models): + groq(prompt).choices.0.message.content:format + api-model.in?(openai-models): + openai-chat(prompt).choices.0.message.content:format + else: die() + + say: answer + + when session: + write session _ :append true: |+ + Q: $(orig-prompt:trim) + A ($api-model): + $(answer:trim) + +)_", + R"_(( +{:+ "+MAP"} +{:+ "=VAL", := "defn run(prompt session=nil)"} +{:+ "+MAP"} +{:+ "=VAL", := "session-text ="} +{:+ "+MAP"} +{:+ "=VAL", := "when session && session:fs-e"} +{:+ "=VAL", :: ""} +{:+ "-MAP"} +{:+ "=VAL", := "answer ="} +{:+ "+MAP"} +{:+ "=VAL", := "cond"} +{:+ "+MAP"} +{:+ "=VAL", := "api-model =~ /^dall-e/"} +{:+ "=VAL", := "openai-image(prompt).data.0.url"} +{:+ "=VAL", := "api-model.in?(anthropic-models)"} +{:+ "=VAL", := "anthropic(prompt):anthropic-message:format"} +{:+ "=VAL", := "api-model.in?(groq-models)"} +{:+ "=VAL", := "groq(prompt).choices.0.message.content:format"} +{:+ "=VAL", := "api-model.in?(openai-models)"} +{:+ "=VAL", := "openai-chat(prompt).choices.0.message.content:format"} +{:+ "=VAL", := "else"} +{:+ "=VAL", := "die()"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "=VAL", := "say"} +{:+ "=VAL", := "answer"} +{:+ "=VAL", := "when session"} +{:+ "+MAP"} +{:+ "=VAL", := "write session _ :append true"} +{:+ "=VAL", :| "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)_", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 21, 28, "defn run(prompt session=nil)"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 53, 14, "session-text ="), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 73, 28, "when session && session:fs-e"), + e(VAL_|SCLR|PLAI, 0, 0, ""), // note empty scalar pointing at the front + e(EMAP), + e(KEY_|SCLR|PLAI, 106, 8, "answer ="), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 120, 4, "cond"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 132, 22, "api-model =~ /^dall-e/"), + e(VAL_|SCLR|PLAI, 164, 31, "openai-image(prompt).data.0.url"), + e(KEY_|SCLR|PLAI, 202, 31, "api-model.in?(anthropic-models)"), + e(VAL_|SCLR|PLAI, 243, 42, "anthropic(prompt):anthropic-message:format"), + e(KEY_|SCLR|PLAI, 292, 26, "api-model.in?(groq-models)"), + e(VAL_|SCLR|PLAI, 328, 45, "groq(prompt).choices.0.message.content:format"), + e(KEY_|SCLR|PLAI, 380, 28, "api-model.in?(openai-models)"), + e(VAL_|SCLR|PLAI, 418, 52, "openai-chat(prompt).choices.0.message.content:format"), + e(KEY_|SCLR|PLAI, 477, 4, "else"), + e(VAL_|SCLR|PLAI, 483, 5, "die()"), + e(EMAP), + e(EMAP), + e(KEY_|SCLR|PLAI, 492, 3, "say"), + e(VAL_|SCLR|PLAI, 497, 6, "answer"), + e(KEY_|SCLR|PLAI, 507, 12, "when session"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 525, 28, "write session _ :append true"), + e(VAL_|SCLR|LITL, 558, 55, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n", needs_filter), + e(EMAP), + e(EMAP), + e(EMAP), + e(EDOC), + e(ESTR), + }), }; } // namespace diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 26e10fb0c..805cd9e08 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -27,7 +27,8 @@ private native int ys2evt_parse(long ryml2evt, String filename, int[] evt, int evt_length); private final long ryml2evt; - public Rapidyaml() { + public Rapidyaml() + { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); this.ryml2edn = this.ys2edn_init(); @@ -38,7 +39,8 @@ public Rapidyaml() { // // https://stackoverflow.com/questions/158174/why-would-you-ever-implement-finalize // - protected void finalize() throws Throwable { + protected void finalize() throws Throwable + { try { this.ys2edn_destroy(this.ryml2edn); this.ys2evt_destroy(this.ryml2evt); @@ -48,8 +50,11 @@ protected void finalize() throws Throwable { } } - public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + public String parseYsToEdn(String srcstr) + throws RuntimeException, org.rapidyaml.YamlParseErrorException + { String filename = "yamlscript"; // fixme +long t = System.nanoTime(); byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); int edn_size = 10 * src.length; byte[] edn = new byte[edn_size]; @@ -62,13 +67,20 @@ public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml throw new RuntimeException("inconsistent size"); } } +t = System.nanoTime() - t; +System.out.printf(" edn@java=%.6fms\n", (double)t / 1.e6); String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); return ret; } - public int parseYsToEvt(byte[] src, int[] evts) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + public int parseYsToEvt(byte[] src, int[] evts) + throws RuntimeException, org.rapidyaml.YamlParseErrorException + { String filename = "yamlscript"; // fixme +long t = System.nanoTime(); int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); +t = System.nanoTime() - t; +System.out.printf(" evt@java=%.6fms\n", (double)t / 1.e6); return required_size; } } diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java new file mode 100644 index 000000000..cf6654d81 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java @@ -0,0 +1,52 @@ +package cmp; + +import org.rapidyaml.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +// https://stackoverflow.com/questions/804466/how-do-i-create-executable-java-program +public class CmpEdnEvt +{ + public static void main(String[] args) throws Throwable + { + Rapidyaml rapidyaml = new Rapidyaml(); + int evtSize = 10000000; + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); + compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); + } + + public static void compareEdnEvt(int evtSize, Rapidyaml rapidyaml, String path) throws Throwable + { + String ys = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); + int[] evt = new int[evtSize]; + // + long t0 = System.nanoTime(); + byte[] ysBytes = ys.getBytes(StandardCharsets.UTF_8); + long tys2Bytes = System.nanoTime() - t0; + System.out.printf("-----\n"); + System.out.printf("%s\n", path); + System.out.printf(" ys.length=%d\n", ys.length()); + System.out.printf(" ys2bytes=%fms\n", (double)tys2Bytes / 1.e6); + // + System.out.printf(" edn...\n"); + t0 = System.nanoTime(); + String edn = rapidyaml.parseYsToEdn(ys); + long tEdn = System.nanoTime() - t0; + System.out.printf(" edn=%.6fms, length=%d -> %dB @%.3fMB/s\n", (double)tEdn / 1.e6, edn.length(), edn.length(), (double)ysBytes.length / tEdn * 1.e3); + // + System.out.printf(" evt...\n"); + t0 = System.nanoTime(); + int numEvts = rapidyaml.parseYsToEvt(ysBytes, evt); + long tEvt = System.nanoTime() - t0; + System.out.printf(" evt=%.6fms, length=%d -> %dB @%.3fMB/s\n", (double)tEvt / 1.e6, numEvts, 4*numEvts, (double)ysBytes.length / tEvt * 1.e3); + // + } +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf new file mode 100644 index 000000000..97704e895 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf @@ -0,0 +1 @@ +Main-class: cmp.CmpEdnEvt diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh new file mode 100755 index 000000000..d3a5e8f10 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -xe + +thisdir=$(dirname $0) +nativedir=$(cd $thisdir/../../../../../../native ; pwd) +rymldir=$(cd $nativedir/.. ; pwd) + +make -C $nativedir build RAPIDYAML_TIMED=1 +make -C $rymldir test RAPIDYAML_TIMED=1 + +cd $thisdir +jd=${jd:-/usr/lib/jvm/java-23-openjdk/bin} +$jd/javac -d . ../*.java +$jd/javac -d . -cp . CmpEdnEvt.java +$jd/jar -cmf manifest.mf CmpEdnEvt.jar cmp org +$jd/java -jar -Djava.library.path=$nativedir CmpEdnEvt.jar diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index db62cc2bc..bed13660e 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -1,11 +1,14 @@ package org.rapidyaml; import org.rapidyaml.*; -import java.nio.charset.StandardCharsets; -import java.nio.ByteBuffer; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; /** * Unit test for simple App. @@ -87,6 +90,7 @@ private static int required_size_(ExpectedEvent[] evts) private void testEvt_(String ys, ExpectedEvent[] expected) { + boolean dbglog = false; Rapidyaml rapidyaml = new Rapidyaml(); try { int[] actual = new int[2 * required_size_(expected)]; @@ -107,14 +111,16 @@ private void testEvt_(String ys, ExpectedEvent[] expected) if(ie >= expected.length) break; int cmp = 1; - System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + if(dbglog) + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { cmp &= (ia + 2 < numEvts) ? 1 : 0; if(cmp != 0) { cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; - System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + if(dbglog) + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); if(cmp != 0) { cmp &= (actual[ia + 1] >= 0) ? 1 : 0; cmp &= (actual[ia + 2] >= 0) ? 1 : 0; @@ -122,15 +128,18 @@ private void testEvt_(String ys, ExpectedEvent[] expected) if(cmp != 0) { String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; - System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + if(dbglog) + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); } else { - System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); + if(dbglog) + System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); } } } } - System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); + if(dbglog) + System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); status &= cmp; ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; ++ie; @@ -202,6 +211,56 @@ public void testUtf8() testEvt_(ys, expected); } + public void testTaggedInt() + { + String ys = "- !!int 42"; + testEdn_(ys, + "(\n" + + "{:+ \"+SEQ\"}\n" + + "{:+ \"=VAL\", :! \"tag:yaml.org,2002:int\", := \"42\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 2, 5, "!!int"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 8, 2, "42"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testTaggedSeq() + { + String ys = "- !!seq []"; + testEdn_(ys, + "(\n" + + "{:+ \"+SEQ\"}\n" + + "{:+ \"+SEQ\", :! \"tag:yaml.org,2002:seq\", :flow true}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"-SEQ\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n" + ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 2, 5, "!!seq"), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); + } + public void testLargeCase() { String ys = "--- !yamlscript/v0\n" + From bb24eb1ecb0a0cc84bbd1e3a59c95fda87a76be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Fri, 21 Feb 2025 14:00:13 -0500 Subject: [PATCH 10/29] rapidyaml: Integrate rapidyaml evt int parser.clj --- core/src/yamlscript/parser.clj | 115 +++++++++++++++--- core/test/yamlscript/parser_test.clj | 7 +- rapidyaml/native/ysparse_evt_handler.hpp | 69 ++++++----- rapidyaml/pom.xml | 2 +- .../src/main/java/org/rapidyaml/Evt.java | 69 ++++++----- .../main/java/org/rapidyaml/Rapidyaml.java | 16 ++- 6 files changed, 188 insertions(+), 90 deletions(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index ae9d390a5..1c799a78e 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -4,16 +4,15 @@ ;; The yamlscript.parser is responsible for parsing YAML into a sequence of ;; event objects. -;; TODO -;; - switch from snakeyaml to libfyaml (ffi) - (ns yamlscript.parser (:require [clojure.string :as str] [yamlscript.common]) (:import (java.util Optional) - (org.rapidyaml Rapidyaml) + (java.nio ByteBuffer) + (java.nio.charset StandardCharsets) + (org.rapidyaml Evt Rapidyaml) (org.snakeyaml.engine.v2.api LoadSettings) (org.snakeyaml.engine.v2.api.lowlevel Parse) (org.snakeyaml.engine.v2.exceptions Mark) @@ -31,20 +30,17 @@ SequenceEndEvent)) (:refer-clojure)) -(declare - parse-snakeyaml - parse-rapidyaml) +(declare parse-fn) (def shebang-ys #"^#!.*/env ys-0(?:\.d+\.\d+)?\n") (def shebang-bash #"^#!.*[/ ]bash\n+source +<\(") (defn parse "Parse a YAML string into a sequence of event objects." [yaml-string] - (let [has-code-mode-shebang (or (re-find shebang-ys yaml-string) + (let [has-code-mode-shebang (or + (re-find shebang-ys yaml-string) (re-find shebang-bash yaml-string)) - events (if (System/getenv "YS_PARSER_RAPIDYAML") - (parse-rapidyaml yaml-string) - (parse-snakeyaml yaml-string)) + events (parse-fn yaml-string) [first-event & rest-events] events first-event-tag (:! first-event) first-event (if (and has-code-mode-shebang @@ -68,19 +64,98 @@ (remove nil?) rest))) -(defn parse-rapidyaml [yaml-string] +(defn parse-rapidyaml-edn [yaml-string] (let [rapid-parser (new Rapidyaml)] (->> yaml-string - ( #(.parseYS ^Rapidyaml rapid-parser %1)) - #_(#(do (println %1) %1)) - (#(str/replace %1 #"^\(\n\{:\+\ \"\+DOC\"\}" "(")) - #_(#(do (println %1) %1)) + (#(.parseYsToEdn ^Rapidyaml rapid-parser %1)) read-string))) -(defn parse-test-case [yaml-string] - (->> yaml-string - parse - (remove (fn [ev] (= "DOC" (subs (:+ ev) 1)))))) +(defn event-type [mask] + (condp = (bit-and mask 2r11111111111) + Evt/BSTR nil + Evt/ESTR nil + Evt/BDOC "+DOC" + Evt/EDOC "-DOC" + Evt/BMAP "+MAP" + Evt/EMAP "-MAP" + Evt/BSEQ "+SEQ" + Evt/ESEQ "-SEQ" + Evt/SCLR "=VAL" + Evt/ALIA "=ALI" + nil)) + +(defmacro flag? [flag mask] + `(pos? (bit-and ~mask (. Evt ~flag)))) + +(defn get-skey [mask] + (condp = (bit-and mask 2r111110000000000000000) + Evt/PLAI := + Evt/SQUO :' + Evt/DQUO :$ + Evt/LITL :| + Evt/FOLD :> + nil)) + +(defn parse-rapidyaml-evt [^String yaml-string] + (rest + (let [parser ^Rapidyaml (new Rapidyaml) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array 5) + needed (.parseYsToEvt parser buffer masks) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array needed) + _ (.parseYsToEvt parser buffer masks) + get-str (fn [i] + (let [off (aget masks (inc i)) + len (aget masks (+ i 2))] + (reduce + (fn [slice i] (str slice (char (aget buffer i)))) + "" (range off (+ off len)))))] + + (loop [i 0, tag nil, anchor nil, events []] + (if (< i needed) + (let [mask (aget masks i) + type (event-type mask) + ; _ (WWW (Integer/toString mask 2) type) + sval (when (flag? HAS_STR mask) (get-str i)) + tag (if (flag? TAG_ mask) sval tag) + anchor (if (flag? ANCH mask) sval anchor) + event (when type + (let [event {:+ type} + event (if (flag? FLOW mask) + (assoc event :flow true) event) + event (if anchor (assoc event :& anchor) event) + event (if tag + (let [tag (str/replace tag + #"^!!" + "tag:yaml.org,2002:")] + (assoc event :! tag)) event) + event (if sval (assoc event + (get-skey mask) sval) event) + event (if (= type "=ALI") + {:+ "=ALI" :* sval} + event)] + event)) + events (if event (conj events event) events) + i (+ i (if sval 3 1))] + (if event + (recur i nil nil events) + (recur i tag anchor events))) + events))))) + +(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")] + (condp = parser-name + "" parse-snakeyaml + "snake" parse-snakeyaml + "ry1" parse-rapidyaml-edn + "ry2" parse-rapidyaml-evt + "ry-edn" parse-rapidyaml-edn + "ry-evt" parse-rapidyaml-evt + "ryml-edn" parse-rapidyaml-edn + "ryml-evt" parse-rapidyaml-evt + (die "Unknown YS_PARSER value: " parser-name)) + ; parse-rapidyaml-evt + parse-snakeyaml)) ;; ;; Functions to turn Java event objects into Clojure objects diff --git a/core/test/yamlscript/parser_test.clj b/core/test/yamlscript/parser_test.clj index 29420b7d8..ef04eec4b 100644 --- a/core/test/yamlscript/parser_test.clj +++ b/core/test/yamlscript/parser_test.clj @@ -8,6 +8,11 @@ [yamlscript.parser :as parser] [yamltest.core :as test])) +(defn parse-test-case [yaml-string] + (->> yaml-string + parser/parse + (remove (fn [ev] (= "DOC" (subs (:+ ev) 1)))))) + (test/load-yaml-test-files ["test/compiler-stack.yaml" "test/resolver.yaml" @@ -16,7 +21,7 @@ :test (fn [test] (->> test :yamlscript - parser/parse-test-case + parse-test-case (map pr-str) (map #(subs %1 4 (dec (count %1)))))) :want (fn [test] diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp index 5892492d1..a8f098b99 100644 --- a/rapidyaml/native/ysparse_evt_handler.hpp +++ b/rapidyaml/native/ysparse_evt_handler.hpp @@ -15,39 +15,46 @@ C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") namespace evt { using DataType = int32_t; typedef enum : DataType { - // --------------------- - // structure flags - KEY_ = 1 << 0, // as key - VAL_ = 1 << 1, // as value - SCLR = 1 << 2, // =VAL - BSEQ = 1 << 3, // +SEQ - ESEQ = 1 << 4, // -SEQ - BMAP = 1 << 5, // +MAP - EMAP = 1 << 6, // -MAP - ALIA = 1 << 7, // ref - ANCH = 1 << 8, // anchor - TAG_ = 1 << 9, // tag - // --------------------- - // style flags - PLAI = 1 << 10, // : (plain scalar) - SQUO = 1 << 11, // ' (single-quoted scalar) - DQUO = 1 << 12, // " (double-quoted scalar) - LITL = 1 << 13, // | (block literal scalar) - FOLD = 1 << 14, // > (block folded scalar) - FLOW = 1 << 15, // flow container: [] for seqs or {} for maps - BLCK = 1 << 16, // block container - // --------------------- - // document flags - BDOC = 1 << 17, // +DOC - EDOC = 1 << 18, // -DOC - EXPL = 1 << 21, // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) - BSTR = 1 << 19, // +STR - ESTR = 1 << 20, // -STR - // --------------------- - // utility flags + // Event types + BSTR = 1 << 0, // +STR + ESTR = 1 << 1, // -STR + BDOC = 1 << 2, // +DOC + EDOC = 1 << 3, // -DOC + BMAP = 1 << 4, // +MAP + EMAP = 1 << 5, // -MAP + BSEQ = 1 << 6, // +SEQ + ESEQ = 1 << 7, // -SEQ + SCLR = 1 << 8, // =VAL + ALIA = 1 << 9, // =ALI + + // Style flags + PLAI = 1 << 16, // : (plain scalar) + SQUO = 1 << 17, // ' (single-quoted scalar) + DQUO = 1 << 18, // " (double-quoted scalar) + LITL = 1 << 19, // | (block literal scalar) + FOLD = 1 << 20, // > (block folded scalar) + FLOW = 1 << 21, // flow container: + // [] for seqs or {} for maps + BLCK = 1 << 22, // block container + + // Modifiers + ANCH = 1 << 24, // anchor + TAG_ = 1 << 25, // tag + + // Structure flags + KEY_ = 1 << 26, // as key + VAL_ = 1 << 27, // as value + EXPL = 1 << 28, // --- (with BDOC) or + // ... (with EDOC) + // (may be fused with FLOW + // if needed) + + // Utility flags LAST = EXPL, MASK = (LAST << 1) - 1, - HAS_STR = SCLR|ALIA|ANCH|TAG_ // the event requires a string. the next two integers will provide respectively the string's offset and length + // the event requires a string. the next two integers will provide + // respectively the string's offset and length + HAS_STR = SCLR|ALIA|ANCH|TAG_ } EventFlags; } // namespace evt diff --git a/rapidyaml/pom.xml b/rapidyaml/pom.xml index 3efb8c7c4..54902b7fd 100644 --- a/rapidyaml/pom.xml +++ b/rapidyaml/pom.xml @@ -8,7 +8,7 @@ rapidyaml - 0.7.2 + 0.8.0 rapidyaml diff --git a/rapidyaml/src/main/java/org/rapidyaml/Evt.java b/rapidyaml/src/main/java/org/rapidyaml/Evt.java index 4f06211f3..5305610be 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Evt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Evt.java @@ -1,37 +1,44 @@ package org.rapidyaml; public class Evt { - // --------------------- - // structure flags - public static final int KEY_ = 1 << 0; // as key - public static final int VAL_ = 1 << 1; // as value - public static final int SCLR = 1 << 2; // =VAL - public static final int BSEQ = 1 << 3; // +SEQ - public static final int ESEQ = 1 << 4; // -SEQ - public static final int BMAP = 1 << 5; // +MAP - public static final int EMAP = 1 << 6; // -MAP - public static final int ALIA = 1 << 7; // ref - public static final int ANCH = 1 << 8; // anchor - public static final int TAG_ = 1 << 9; // tag - // --------------------- - // style flags - public static final int PLAI = 1 << 10; // : (plain scalar) - public static final int SQUO = 1 << 11; // ' (single-quoted scalar) - public static final int DQUO = 1 << 12; // " (double-quoted scalar) - public static final int LITL = 1 << 13; // | (block literal scalar) - public static final int FOLD = 1 << 14; // > (block folded scalar) - public static final int FLOW = 1 << 15; // flow container: [] for seqs or {} for maps - public static final int BLCK = 1 << 16; // block container - // --------------------- - // document flags - public static final int BDOC = 1 << 17; // +DOC - public static final int EDOC = 1 << 18; // -DOC - public static final int EXPL = 1 << 21; // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) - public static final int BSTR = 1 << 19; // +STR - public static final int ESTR = 1 << 20; // -STR - // --------------------- - // utility flags - public static final int LAST = ESTR; + // Event types + public static final int BSTR = 1 << 0; // +STR + public static final int ESTR = 1 << 1; // -STR + public static final int BDOC = 1 << 2; // +DOC + public static final int EDOC = 1 << 3; // -DOC + public static final int BMAP = 1 << 4; // +MAP + public static final int EMAP = 1 << 5; // -MAP + public static final int BSEQ = 1 << 6; // +SEQ + public static final int ESEQ = 1 << 7; // -SEQ + public static final int SCLR = 1 << 8; // =VAL + public static final int ALIA = 1 << 9; // =ALI + + // Style flags + public static final int PLAI = 1 << 16; // : (plain scalar) + public static final int SQUO = 1 << 17; // ' (single-quoted scalar) + public static final int DQUO = 1 << 18; // " (double-quoted scalar) + public static final int LITL = 1 << 19; // | (block literal scalar) + public static final int FOLD = 1 << 20; // > (block folded scalar) + + public static final int FLOW = 1 << 21; // flow container: + // [] for seqs or {} for maps + public static final int BLCK = 1 << 22; // block container + + // Modifiers + public static final int ANCH = 1 << 24; // anchor + public static final int TAG_ = 1 << 25; // tag + + // Structure flags + public static final int KEY_ = 1 << 26; // as key + public static final int VAL_ = 1 << 27; // as value + public static final int EXPL = 1 << 28; // --- (with BDOC) or + // ... (with EDOC) + // (may be fused with FLOW + // if needed) + + // Utility flags + public static final int LAST = EXPL; public static final int MASK = ((LAST << 1) - 1); public static final int HAS_STR = SCLR|ALIA|ANCH|TAG_; + } diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 805cd9e08..58d6cb729 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -54,7 +54,7 @@ public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { String filename = "yamlscript"; // fixme -long t = System.nanoTime(); + long t = System.nanoTime(); byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); int edn_size = 10 * src.length; byte[] edn = new byte[edn_size]; @@ -67,8 +67,10 @@ public String parseYsToEdn(String srcstr) throw new RuntimeException("inconsistent size"); } } -t = System.nanoTime() - t; -System.out.printf(" edn@java=%.6fms\n", (double)t / 1.e6); + if (System.getenv("YS_RYML_TIMER") != null) { + t = System.nanoTime() - t; + System.out.printf(" edn@java=%.6fms\n", (double)t / 1.e6); + } String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); return ret; } @@ -77,10 +79,12 @@ public int parseYsToEvt(byte[] src, int[] evts) throws RuntimeException, org.rapidyaml.YamlParseErrorException { String filename = "yamlscript"; // fixme -long t = System.nanoTime(); + long t = System.nanoTime(); int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); -t = System.nanoTime() - t; -System.out.printf(" evt@java=%.6fms\n", (double)t / 1.e6); + if (System.getenv("YS_RYML_TIMER") != null) { + t = System.nanoTime() - t; + System.out.printf(" evt@java=%.6fms\n", (double)t / 1.e6); + } return required_size; } } From 98d4eee9ee8892c2ed66614088086d03401e8f65 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 22 Feb 2025 19:53:07 +0000 Subject: [PATCH 11/29] rapidyaml: Test parsing yamllm.ys --- common/java.mk | 7 + common/vars.mk | 8 + rapidyaml/Makefile | 12 +- rapidyaml/native/.gitignore | 1 + rapidyaml/native/Makefile | 44 ++-- .../java/org/rapidyaml/cmp/CmpEdnEvt.java | 1 + .../src/main/java/org/rapidyaml/cmp/run.sh | 3 + .../java/org/rapidyaml/RapidyamlTest.java | 219 ++++++++++-------- 8 files changed, 168 insertions(+), 127 deletions(-) diff --git a/common/java.mk b/common/java.mk index bc9997d8c..f95a424de 100644 --- a/common/java.mk +++ b/common/java.mk @@ -5,6 +5,10 @@ export JAVA_HOME := $(GRAALVM_HOME) export PATH := $(JAVA_HOME)/bin:$(PATH) +MVN ?= mvn +JAR ?= $(GRAALVM_HOME)/bin/jar +JAVAC ?= $(GRAALVM_HOME)/bin/javac + YAMLSCRIPT_JAVA_INSTALLED := \ $(MAVEN_REPOSITORY)/org/yamlscript/yamlscript/maven-metadata-local.xml @@ -31,3 +35,6 @@ endif $(YAMLSCRIPT_JAVA_INSTALLED): $(YAMLSCRIPT_JAVA_SRC) $(MAKE) -C $(ROOT)/java install + +$(JAVAC): $(GRAALVM_INSTALLED) +$(JAR): $(GRAALVM_INSTALLED) diff --git a/common/vars.mk b/common/vars.mk index f028ec0ab..0bae90453 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -242,6 +242,14 @@ RAPIDYAML_INSTALLED := \ $(RAPIDYAML_INSTALLED_DIR)/rapidyaml-$(RAPIDYAML_VERSION).jar +#------------------------------------------------------------------------------ +# Programs +#------------------------------------------------------------------------------ + +GIT ?= git +CMAKE ?= cmake + + #------------------------------------------------------------------------------ default:: diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index c01c44d09..af0d7ea2b 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -30,11 +30,11 @@ install:: $(RAPIDYAML_INSTALLED) test:: build $(JAVA_INSTALLED) $(MAKE) -C native $@ - mvn $@ + $(MVN) $@ test-x: build $(JAVA_INSTALLED) $(MAKE) -C native $@ - mvn -X -e test + $(MVN) -X -e test clean:: $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @@ -57,16 +57,16 @@ $(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ $(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED) - mvn install + $(MVN) install $(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED) - jar -cvf $@ $< + $(JAR) -cvf $@ $< $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) @# this doesn't work: - @#javac $< + @#$(JAVAC) $< @# ... but this does: - javac $(RAPIDYAML_JAVA) + $(JAVAC) $(RAPIDYAML_JAVA) $(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) $(MAKE) -C native $@ diff --git a/rapidyaml/native/.gitignore b/rapidyaml/native/.gitignore index 8e53ad039..2a816d627 100644 --- a/rapidyaml/native/.gitignore +++ b/rapidyaml/native/.gitignore @@ -1,5 +1,6 @@ /rapidyaml/ /_build/ +/build/ /librapidyaml.* /.cache /compile_commands.json diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index f86b2e630..ec0037293 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -9,7 +9,7 @@ include $(COMMON)/python.mk # https://stackoverflow.com/questions/24493337/linking-static-library-with-jni THIS_DIR := $(shell pwd) -BDIR := $(THIS_DIR)/_build/$(RAPIDYAML_BUILD_TYPE)-shared$(RAPIDYAML_SHARED)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) +BDIR := $(THIS_DIR)/_build/$(RAPIDYAML_BUILD_TYPE)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) RAPIDYAML_DEPS := \ Makefile \ @@ -30,46 +30,52 @@ CMK_FLAGS := \ #------------------------------------------------------------------------------ default:: +cfg:: cfg-static cfg-shared build:: build-static build-shared test: test-static test-shared rapidyaml: mkdir -p $@ - git -C $@ init -q . - git -C $@ remote add origin $(RAPIDYAML_REPO) - git -C $@ fetch origin $(RAPIDYAML_TAG) - git -C $@ reset --hard FETCH_HEAD - git -C $@ submodule update --init --recursive + $(GIT) -C $@ init -q . + $(GIT) -C $@ remote add origin $(RAPIDYAML_REPO) + $(GIT) -C $@ fetch origin $(RAPIDYAML_TAG) + $(GIT) -C $@ reset --hard FETCH_HEAD + $(GIT) -C $@ submodule update --init --recursive clean:: $(RM) librapidyaml.* - $(RM) -r $(BDIR) + $(RM) -r _build $(RM) -r rapidyaml-install realclean:: clean $(RM) -r rapidyaml +jni: $(RAPIDYAML_JNI_H) + +jnicheck: jni + $(GIT) diff --exit-code $(RAPIDYAML_JNI_H) + #------------------------------------------------------------------------------ -$(RAPIDYAML_LIB): rapidyaml build-static: $(RAPIDYAML_LIB) +cfg-static: rapidyaml + $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF test-static: build-static - $(CMK_ENV) cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml-test-run -$(RAPIDYAML_LIB): $(RAPIDYAML_DEPS) - $(CMK_ENV) cmake -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF - cmake --build $(BDIR)-static --parallel --verbose --target rapidyaml + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run +$(RAPIDYAML_LIB): cfg-static $(RAPIDYAML_DEPS) + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --target rapidyaml --parallel --verbose cp -fv $(BDIR)-static/*.a $@ -$(RAPIDYAML_SO): rapidyaml build-shared: $(RAPIDYAML_SO) +cfg-shared: rapidyaml + $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON test-shared: build-shared - cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml-test-run -$(RAPIDYAML_SO): $(RAPIDYAML_DEPS) - $(CMK_ENV) cmake -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON - $(CMK_ENV) cmake --build $(BDIR)-shared --parallel --verbose --target rapidyaml + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --target rapidyaml-test-run --verbose +$(RAPIDYAML_SO): cfg-shared $(RAPIDYAML_DEPS) + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --target rapidyaml --parallel --verbose cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so -$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) - javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work +$(RAPIDYAML_JNI_H): $(JAVAC) $(RAPIDYAML_JAVA) + $(JAVAC) -h . $(RAPIDYAML_JAVA) # $^ doesn't work diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java index cf6654d81..192ffc9a0 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java @@ -13,6 +13,7 @@ public static void main(String[] args) throws Throwable { Rapidyaml rapidyaml = new Rapidyaml(); int evtSize = 10000000; + compareEdnEvt(evtSize, rapidyaml, "./yamllm.ys"); compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh index d3a5e8f10..da951ab24 100755 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh @@ -9,7 +9,10 @@ rymldir=$(cd $nativedir/.. ; pwd) make -C $nativedir build RAPIDYAML_TIMED=1 make -C $rymldir test RAPIDYAML_TIMED=1 + cd $thisdir +wget https://raw.githubusercontent.com/yaml/yamllm/refs/heads/main/bin/yamllm.ys +ls -lFhp jd=${jd:-/usr/lib/jvm/java-23-openjdk/bin} $jd/javac -d . ../*.java $jd/javac -d . -cp . CmpEdnEvt.java diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index bed13660e..b78b876be 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -77,7 +77,10 @@ private class ExpectedEvent this.str_len = str_len; this.str = str; } - int required_size() { return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; } + int required_size() + { + return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; + } }; private static int required_size_(ExpectedEvent[] evts) { @@ -90,7 +93,7 @@ private static int required_size_(ExpectedEvent[] evts) private void testEvt_(String ys, ExpectedEvent[] expected) { - boolean dbglog = false; + boolean dbglog = false; Rapidyaml rapidyaml = new Rapidyaml(); try { int[] actual = new int[2 * required_size_(expected)]; @@ -111,8 +114,8 @@ private void testEvt_(String ys, ExpectedEvent[] expected) if(ie >= expected.length) break; int cmp = 1; - if(dbglog) - System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + if(dbglog) + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { cmp &= (ia + 2 < numEvts) ? 1 : 0; @@ -120,7 +123,7 @@ private void testEvt_(String ys, ExpectedEvent[] expected) cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; if(dbglog) - System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); if(cmp != 0) { cmp &= (actual[ia + 1] >= 0) ? 1 : 0; cmp &= (actual[ia + 2] >= 0) ? 1 : 0; @@ -129,16 +132,16 @@ private void testEvt_(String ys, ExpectedEvent[] expected) String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; if(dbglog) - System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); } else { - if(dbglog) - System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); + if(dbglog) + System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); } } } } - if(dbglog) + if(dbglog) System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); status &= cmp; ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; @@ -161,6 +164,16 @@ private void testEvt_(String ys, ExpectedEvent[] expected) } } + ExpectedEvent mkev(int flags) + { + return new ExpectedEvent(flags); + } + + ExpectedEvent mkev(int flags, int offs, int len, String ref) + { + return new ExpectedEvent(flags, offs, len, ref); + } + public void testPlainMap() { String ys = "a: 1"; @@ -174,14 +187,14 @@ public void testPlainMap() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 1, "a"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 3, 1, "1"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 1, "a"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 3, 1, "1"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -199,14 +212,14 @@ public void testUtf8() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "𝄞"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "✅"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "𝄞"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "✅"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -223,14 +236,14 @@ public void testTaggedInt() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 2, 5, "!!int"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 8, 2, "42"), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.TAG_, 2, 5, "!!int"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 8, 2, "42"), + mkev(Evt.ESEQ), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -248,15 +261,15 @@ public void testTaggedSeq() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 2, 5, "!!seq"), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.TAG_, 2, 5, "!!seq"), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.ESEQ), + mkev(Evt.ESEQ), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -271,9 +284,11 @@ public void testLargeCase() "- 'foo'\n" + "- \"foo\"\n" + "- |\n" + - " foo\n" + + " foo\n" + + " literal\n" + "- >\n" + - " foo\n" + + " foo\n" + + " folded\n" + "- [1, 2, true, false, null]\n" + "- &anchor-1 !tag-1 foobar\n" + "---\n" + @@ -294,8 +309,8 @@ public void testLargeCase() "{:+ \"=VAL\", := \"foo\"}\n" + "{:+ \"=VAL\", :' \"foo\"}\n" + "{:+ \"=VAL\", :$ \"foo\"}\n" + - "{:+ \"=VAL\", :| \"foo\\n\"}\n" + - "{:+ \"=VAL\", :> \"foo\\n\"}\n" + + "{:+ \"=VAL\", :| \"foo\\nliteral\\n\"}\n" + + "{:+ \"=VAL\", :> \"foo folded\\n\"}\n" + "{:+ \"+SEQ\", :flow true}\n" + "{:+ \"=VAL\", := \"1\"}\n" + "{:+ \"=VAL\", := \"2\"}\n" + @@ -316,46 +331,46 @@ public void testLargeCase() ")\n" ); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC|Evt.EXPL), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 5, 13, "yamlscript/v0"), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 25, 0, ""), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.FLOW), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 29, 1, "x"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 32, 1, "y"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 38, 1, "x"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 41, 1, "y"), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 46, 3, "foo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 53, 3, "foo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 61, 3, "foo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 70, 4, "foo\n"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 80, 4, "foo\n"), - new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 89, 1, "1"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 92, 1, "2"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 95, 4, "true"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 101, 5, "false"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 108, 4, "null"), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.VAL_|Evt.TAG_, 127, 5, "tag-1"), - new ExpectedEvent(Evt.VAL_|Evt.ANCH, 117, 8, "anchor-1"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 133, 6, "foobar"), - new ExpectedEvent(Evt.ESEQ), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.BDOC|Evt.EXPL), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 144, 7, "another"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 153, 3, "doc"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC|Evt.EXPL), + mkev(Evt.VAL_|Evt.TAG_, 5, 13, "yamlscript/v0"), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), + mkev(Evt.VAL_|Evt.TAG_, 25, 0, ""), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.BMAP|Evt.FLOW), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 29, 1, "x"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 32, 1, "y"), + mkev(Evt.EMAP), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 38, 1, "x"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 41, 1, "y"), + mkev(Evt.ESEQ), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 46, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.SQUO, 53, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.DQUO, 61, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.LITL, 70, 12, "foo\nliteral\n"), + mkev(Evt.VAL_|Evt.SCLR|Evt.FOLD, 98, 11, "foo folded\n"), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 124, 1, "1"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 127, 1, "2"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 130, 4, "true"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 136, 5, "false"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 143, 4, "null"), + mkev(Evt.ESEQ), + mkev(Evt.VAL_|Evt.TAG_, 162, 5, "tag-1"), + mkev(Evt.VAL_|Evt.ANCH, 152, 8, "anchor-1"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 168, 6, "foobar"), + mkev(Evt.ESEQ), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.BDOC|Evt.EXPL), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 179, 7, "another"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 188, 3, "doc"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } @@ -394,22 +409,22 @@ public void testFilterCase() "{:+ \"-DOC\"}\n" + ")\n"); ExpectedEvent[] expected = { - new ExpectedEvent(Evt.BSTR), - new ExpectedEvent(Evt.BDOC), - new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 5, "plain"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 7, 10, "well a b c"), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 24, 4, "squo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 31, 12, "single'quote"), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 46, 4, "dquo"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 53, 4, "x\t\ny"), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 61, 3, "lit"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 68, 6, "X\nY\nZ\n"), - new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 89, 4, "fold"), - new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 97, 6, "U V W\n"), - new ExpectedEvent(Evt.EMAP), - new ExpectedEvent(Evt.EDOC), - new ExpectedEvent(Evt.ESTR), + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 5, "plain"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 7, 10, "well a b c"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 24, 4, "squo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.SQUO, 31, 12, "single'quote"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 46, 4, "dquo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.DQUO, 53, 4, "x\t\ny"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 61, 3, "lit"), + mkev(Evt.VAL_|Evt.SCLR|Evt.LITL, 68, 6, "X\nY\nZ\n"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 89, 4, "fold"), + mkev(Evt.VAL_|Evt.SCLR|Evt.FOLD, 97, 6, "U V W\n"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), }; testEvt_(ys, expected); } From 2a91cc5efa7d7bd20012085b969ce23974ce2b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sun, 23 Feb 2025 09:01:02 -0500 Subject: [PATCH 12/29] core: Add a YS_PARSER_TIME variable to time parsing --- core/src/yamlscript/parser.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 1c799a78e..26b5248c0 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -40,7 +40,9 @@ (let [has-code-mode-shebang (or (re-find shebang-ys yaml-string) (re-find shebang-bash yaml-string)) - events (parse-fn yaml-string) + events (if (System/getenv "YS_PARSER_TIME") + (time (parse-fn yaml-string)) + (parse-fn yaml-string)) [first-event & rest-events] events first-event-tag (:! first-event) first-event (if (and has-code-mode-shebang From 26736d9adb9275015db97a3d97f961a6e7346024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sun, 23 Feb 2025 09:01:23 -0500 Subject: [PATCH 13/29] core: Refactor parser to get rid of rapidyaml-edn --- core/src/yamlscript/parser.clj | 184 ++++++++++++++++----------------- 1 file changed, 90 insertions(+), 94 deletions(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 26b5248c0..8cfdd70a2 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -10,7 +10,6 @@ [yamlscript.common]) (:import (java.util Optional) - (java.nio ByteBuffer) (java.nio.charset StandardCharsets) (org.rapidyaml Evt Rapidyaml) (org.snakeyaml.engine.v2.api LoadSettings) @@ -57,6 +56,10 @@ (declare snake-event) +;; +;; SnakeYAML Parser +;; + ;; TODO - Set bigger buffer size in scanner class (defn parse-snakeyaml [yaml-string] (let [parser (new Parse (.build (LoadSettings/builder)))] @@ -66,99 +69,6 @@ (remove nil?) rest))) -(defn parse-rapidyaml-edn [yaml-string] - (let [rapid-parser (new Rapidyaml)] - (->> yaml-string - (#(.parseYsToEdn ^Rapidyaml rapid-parser %1)) - read-string))) - -(defn event-type [mask] - (condp = (bit-and mask 2r11111111111) - Evt/BSTR nil - Evt/ESTR nil - Evt/BDOC "+DOC" - Evt/EDOC "-DOC" - Evt/BMAP "+MAP" - Evt/EMAP "-MAP" - Evt/BSEQ "+SEQ" - Evt/ESEQ "-SEQ" - Evt/SCLR "=VAL" - Evt/ALIA "=ALI" - nil)) - -(defmacro flag? [flag mask] - `(pos? (bit-and ~mask (. Evt ~flag)))) - -(defn get-skey [mask] - (condp = (bit-and mask 2r111110000000000000000) - Evt/PLAI := - Evt/SQUO :' - Evt/DQUO :$ - Evt/LITL :| - Evt/FOLD :> - nil)) - -(defn parse-rapidyaml-evt [^String yaml-string] - (rest - (let [parser ^Rapidyaml (new Rapidyaml) - buffer (.getBytes yaml-string StandardCharsets/UTF_8) - masks (int-array 5) - needed (.parseYsToEvt parser buffer masks) - buffer (.getBytes yaml-string StandardCharsets/UTF_8) - masks (int-array needed) - _ (.parseYsToEvt parser buffer masks) - get-str (fn [i] - (let [off (aget masks (inc i)) - len (aget masks (+ i 2))] - (reduce - (fn [slice i] (str slice (char (aget buffer i)))) - "" (range off (+ off len)))))] - - (loop [i 0, tag nil, anchor nil, events []] - (if (< i needed) - (let [mask (aget masks i) - type (event-type mask) - ; _ (WWW (Integer/toString mask 2) type) - sval (when (flag? HAS_STR mask) (get-str i)) - tag (if (flag? TAG_ mask) sval tag) - anchor (if (flag? ANCH mask) sval anchor) - event (when type - (let [event {:+ type} - event (if (flag? FLOW mask) - (assoc event :flow true) event) - event (if anchor (assoc event :& anchor) event) - event (if tag - (let [tag (str/replace tag - #"^!!" - "tag:yaml.org,2002:")] - (assoc event :! tag)) event) - event (if sval (assoc event - (get-skey mask) sval) event) - event (if (= type "=ALI") - {:+ "=ALI" :* sval} - event)] - event)) - events (if event (conj events event) events) - i (+ i (if sval 3 1))] - (if event - (recur i nil nil events) - (recur i tag anchor events))) - events))))) - -(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")] - (condp = parser-name - "" parse-snakeyaml - "snake" parse-snakeyaml - "ry1" parse-rapidyaml-edn - "ry2" parse-rapidyaml-evt - "ry-edn" parse-rapidyaml-edn - "ry-evt" parse-rapidyaml-evt - "ryml-edn" parse-rapidyaml-edn - "ryml-evt" parse-rapidyaml-evt - (die "Unknown YS_PARSER value: " parser-name)) - ; parse-rapidyaml-evt - parse-snakeyaml)) - ;; ;; Functions to turn Java event objects into Clojure objects ;; @@ -241,5 +151,91 @@ (defmethod snake-event AliasEvent [event] (alias-val event)) (defmethod snake-event :default [_] nil) +;; +;; RapidYAML Parser +;; + +(defn event-type [mask] + (condp = (bit-and mask 2r11111111111) + Evt/BSTR nil + Evt/ESTR nil + Evt/BDOC "+DOC" + Evt/EDOC "-DOC" + Evt/BMAP "+MAP" + Evt/EMAP "-MAP" + Evt/BSEQ "+SEQ" + Evt/ESEQ "-SEQ" + Evt/SCLR "=VAL" + Evt/ALIA "=ALI" + nil)) + +(defmacro flag? [flag mask] + `(pos? (bit-and ~mask (. Evt ~flag)))) + +(defn get-skey [mask] + (condp = (bit-and mask 2r111110000000000000000) + Evt/PLAI := + Evt/SQUO :' + Evt/DQUO :$ + Evt/LITL :| + Evt/FOLD :> + nil)) + +(defn parse-rapidyaml [^String yaml-string] + (rest + (let [parser ^Rapidyaml (new Rapidyaml) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array 5) + needed (.parseYsToEvt parser buffer masks) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array needed) + _ (.parseYsToEvt parser buffer masks) + get-str (fn [i] + (let [off (aget masks (inc i)) + len (aget masks (+ i 2))] + (reduce + (fn [slice i] (str slice (char (aget buffer i)))) + "" (range off (+ off len)))))] + + (loop [i 0, tag nil, anchor nil, events []] + (if (< i needed) + (let [mask (aget masks i) + type (event-type mask) + ; _ (WWW (Integer/toString mask 2) type) + sval (when (flag? HAS_STR mask) (get-str i)) + tag (if (flag? TAG_ mask) sval tag) + anchor (if (flag? ANCH mask) sval anchor) + event (when type + (let [event {:+ type} + event (if (flag? FLOW mask) + (assoc event :flow true) event) + event (if anchor (assoc event :& anchor) event) + event (if tag + (let [tag (str/replace tag + #"^!!" + "tag:yaml.org,2002:")] + (assoc event :! tag)) event) + event (if sval (assoc event + (get-skey mask) sval) event) + event (if (= type "=ALI") + {:+ "=ALI" :* sval} + event)] + event)) + events (if event (conj events event) events) + i (+ i (if sval 3 1))] + (if event + (recur i nil nil events) + (recur i tag anchor events))) + events))))) + +(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")] + (condp = parser-name + "" parse-snakeyaml + "snake" parse-snakeyaml + "rapid" parse-rapidyaml + (die "Unknown YS_PARSER value: " parser-name)) + ; parse-rapidyaml-evt + parse-snakeyaml)) + (comment ) From c848abd4697e86878d5ef30028164b1724333b1e Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 17 Feb 2025 18:38:35 +0000 Subject: [PATCH 14/29] rapidyaml: Add directbuffer JNI methods, save >14ms/call ----- ./yamllm.ys ys.length=9973 call:edn... java:ys2edn... 0,001334ms: jni_ys2edn_parse/GetByteArray(src) 2,034402ms: jni_ys2edn_parse/GetByteArray(dst) 0,001137ms: jni_ys2evt_parse/GetStringUTFChars() 2,054268ms: jni_ys2edn_parse/get_jni_data 4,855MB/s 0,012533ms: reset + reserve 0,171602ms: parse_in_place 58,117MB/s 0,002153ms: ys2edn_retry_get 0,196662ms: ys2edn_parse 50,711MB/s 0,904333ms: jni_ys2edn_parse/release 3,171271ms: jni_ys2edn_parse 3,145MB/s java:ys2edn: 3.207848ms 3.109MB/s 9973B call:edn: 5.332002ms 1.870MB/s 9973B call:ednBuf... java:ys2ednBuf... 0,000844ms: jni_ys2edn_parse/get_jni_data 11816,350MB/s 0,001602ms: reset + reserve 0,113475ms: parse_in_place 87,887MB/s 0,001107ms: ys2edn_retry_get 0,124749ms: ys2edn_parse 79,945MB/s 0,127846ms: jni_ys2edn_parse/call_parse 78,008MB/s 0,139938ms: jni_ys2edn_parse 71,267MB/s java:ys2ednBuf: 0.168987ms 59.016MB/s 9973B call:ednBuf: 1.989086ms 5.014MB/s 9973B call:evt... java:ys2evtBuf... 0,017777ms: jni_ys2evt_parse/GetByteArray(src) 8,084726ms: jni_ys2evt_parse/GetIntArray(dst) 0,003211ms: jni_ys2evt_parse/GetStringUTFChars() 8,132883ms: jni_ys2evt_parse/get_jni 1,226MB/s 0,001486ms: reset + reserve 0,124894ms: parse_in_place 79,852MB/s 0,131290ms: ys2evt_parse 75,962MB/s 0,139889ms: jni_ys2evt_parse/call_parse 71,292MB/s 0,001429ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,264938ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000834ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,282080ms: jni_ys2evt_parse/release 13,560444ms: jni_ys2evt_parse 0,735MB/s java:ys2evt: 13.608809ms 0.733MB/s 9973B call:evt: 19.330048ms 0.516MB/s 9973B call:evtBuf... java:ys2evtBuf... 0,001106ms: jni_ys2evt_parse/get_jni 9017,179MB/s 0,000639ms: reset + reserve 0,067489ms: parse_in_place 147,772MB/s 0,073704ms: ys2evt_parse 135,312MB/s 0,077551ms: jni_ys2evt_parse/call_parse 128,599MB/s 0,092300ms: jni_ys2evt_parse 108,050MB/s java:ys2evtBuf: 0.117877ms 84.605MB/s 9973B call:evtBuf: 5.819274ms 1.714MB/s 9973B ----- /home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml ys.length=2129 call:edn... java:ys2edn... 0,000978ms: jni_ys2edn_parse/GetByteArray(src) 2,058415ms: jni_ys2edn_parse/GetByteArray(dst) 0,001872ms: jni_ys2evt_parse/GetStringUTFChars() 2,082492ms: jni_ys2edn_parse/get_jni_data 1,022MB/s 0,002746ms: reset + reserve 0,034371ms: parse_in_place 61,942MB/s 0,000172ms: ys2edn_retry_get 0,045708ms: ys2edn_parse 46,578MB/s 0,773825ms: jni_ys2edn_parse/release 2,913689ms: jni_ys2edn_parse 0,731MB/s java:ys2edn: 2.921583ms 0.729MB/s 2129B call:edn: 4.583358ms 0.465MB/s 2129B call:ednBuf... java:ys2ednBuf... 0,001411ms: jni_ys2edn_parse/get_jni_data 1508,859MB/s 0,001593ms: reset + reserve 0,030772ms: parse_in_place 69,186MB/s 0,000219ms: ys2edn_retry_get 0,042616ms: ys2edn_parse 49,958MB/s 0,045226ms: jni_ys2edn_parse/call_parse 47,075MB/s 0,055543ms: jni_ys2edn_parse 38,331MB/s java:ys2ednBuf: 0.059724ms 35.647MB/s 2129B call:ednBuf: 0.811932ms 2.622MB/s 2129B call:evt... java:ys2evtBuf... 0,001258ms: jni_ys2evt_parse/GetByteArray(src) 8,076122ms: jni_ys2evt_parse/GetIntArray(dst) 0,002259ms: jni_ys2evt_parse/GetStringUTFChars() 8,100922ms: jni_ys2evt_parse/get_jni 0,263MB/s 0,000875ms: reset + reserve 0,022655ms: parse_in_place 93,975MB/s 0,031664ms: ys2evt_parse 67,237MB/s 0,034036ms: jni_ys2evt_parse/call_parse 62,551MB/s 0,001057ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,419873ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000952ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,435716ms: jni_ys2evt_parse/release 13,580720ms: jni_ys2evt_parse 0,157MB/s java:ys2evt: 13.590641ms 0.157MB/s 2129B call:evt: 19.339376ms 0.110MB/s 2129B call:evtBuf... java:ys2evtBuf... 0,002461ms: jni_ys2evt_parse/get_jni 865,095MB/s 0,000434ms: reset + reserve 0,025479ms: parse_in_place 83,559MB/s 0,033737ms: ys2evt_parse 63,106MB/s 0,038801ms: jni_ys2evt_parse/call_parse 54,870MB/s 0,056482ms: jni_ys2evt_parse 37,693MB/s java:ys2evtBuf: 0.062105ms 34.281MB/s 2129B call:evtBuf: 5.527016ms 0.385MB/s 2129B ----- /home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json ys.length=46825 call:edn... java:ys2edn... 0,021512ms: jni_ys2edn_parse/GetByteArray(src) 2,120464ms: jni_ys2edn_parse/GetByteArray(dst) 0,001398ms: jni_ys2evt_parse/GetStringUTFChars() 2,160656ms: jni_ys2edn_parse/get_jni_data 21,672MB/s 0,002448ms: reset + reserve 0,219965ms: parse_in_place 212,875MB/s 0,003424ms: ys2edn_retry_get 0,236103ms: ys2edn_parse 198,324MB/s 0,802344ms: jni_ys2edn_parse/release 3,211247ms: jni_ys2edn_parse 14,582MB/s java:ys2edn: 3.219141ms 14.546MB/s 46825B call:edn: 5.002863ms 9.360MB/s 46825B call:ednBuf... java:ys2ednBuf... 0,001421ms: jni_ys2edn_parse/get_jni_data 32952,146MB/s 0,001445ms: reset + reserve 0,183100ms: parse_in_place 255,735MB/s 0,003680ms: ys2edn_retry_get 0,197663ms: ys2edn_parse 236,893MB/s 0,201021ms: jni_ys2edn_parse/call_parse 232,936MB/s 0,213063ms: jni_ys2edn_parse 219,771MB/s java:ys2ednBuf: 0.219173ms 213.644MB/s 46825B call:ednBuf: 0.979759ms 47.792MB/s 46825B call:evt... java:ys2evtBuf... 0,007270ms: jni_ys2evt_parse/GetByteArray(src) 8,447130ms: jni_ys2evt_parse/GetIntArray(dst) 0,001928ms: jni_ys2evt_parse/GetStringUTFChars() 8,480045ms: jni_ys2evt_parse/get_jni 5,522MB/s 0,000848ms: reset + reserve 0,095276ms: parse_in_place 491,467MB/s 0,101429ms: ys2evt_parse 461,653MB/s 0,104018ms: jni_ys2evt_parse/call_parse 450,162MB/s 0,003393ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,601116ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000839ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,620317ms: jni_ys2evt_parse/release 14,210879ms: jni_ys2evt_parse 3,295MB/s java:ys2evt: 14.220492ms 3.293MB/s 46825B call:evt: 19.957447ms 2.346MB/s 46825B call:evtBuf... java:ys2evtBuf... 0,002473ms: jni_ys2evt_parse/get_jni 18934,492MB/s 0,000712ms: reset + reserve 0,071757ms: parse_in_place 652,550MB/s 0,077564ms: ys2evt_parse 603,695MB/s 0,081212ms: jni_ys2evt_parse/call_parse 576,577MB/s 0,097283ms: jni_ys2evt_parse 481,328MB/s java:ys2evtBuf: 0.101433ms 461.635MB/s 46825B call:evtBuf: 5.983458ms 7.826MB/s 46825B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml ys.length=393000 call:edn... java:ys2edn... 0,142092ms: jni_ys2edn_parse/GetByteArray(src) 2,126455ms: jni_ys2edn_parse/GetByteArray(dst) 0,001700ms: jni_ys2evt_parse/GetStringUTFChars() 2,285052ms: jni_ys2edn_parse/get_jni_data 171,987MB/s 0,002109ms: reset + reserve 8,302745ms: parse_in_place 47,334MB/s 0,166759ms: ys2edn_retry_get 8,485969ms: ys2edn_parse 46,312MB/s 0,999833ms: jni_ys2edn_parse/release 11,780786ms: jni_ys2edn_parse 33,359MB/s java:ys2edn: 11.790444ms 33.332MB/s 393000B call:edn: 14.609382ms 26.901MB/s 393000B call:ednBuf... java:ys2ednBuf... 0,002017ms: jni_ys2edn_parse/get_jni_data 194843,824MB/s 0,001862ms: reset + reserve 7,312239ms: parse_in_place 53,746MB/s 0,158746ms: ys2edn_retry_get 7,486341ms: ys2edn_parse 52,496MB/s 7,489023ms: jni_ys2edn_parse/call_parse 52,477MB/s 7,501143ms: jni_ys2edn_parse 52,392MB/s java:ys2ednBuf: 7.508824ms 52.338MB/s 393000B call:ednBuf: 8.480082ms 46.344MB/s 393000B call:evt... java:ys2evtBuf... 0,045725ms: jni_ys2evt_parse/GetByteArray(src) 8,427749ms: jni_ys2evt_parse/GetIntArray(dst) 0,002128ms: jni_ys2evt_parse/GetStringUTFChars() 8,498642ms: jni_ys2evt_parse/get_jni 46,243MB/s 0,000749ms: reset + reserve 2,269042ms: parse_in_place 173,201MB/s 2,277721ms: ys2evt_parse 172,541MB/s 2,280334ms: jni_ys2evt_parse/call_parse 172,343MB/s 0,030380ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,384011ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,001033ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,426795ms: jni_ys2evt_parse/release 16,211987ms: jni_ys2evt_parse 24,241MB/s java:ys2evt: 16.221876ms 24.227MB/s 393000B call:evt: 22.275141ms 17.643MB/s 393000B call:evtBuf... java:ys2evtBuf... 0,002334ms: jni_ys2evt_parse/get_jni 168380,464MB/s 0,000480ms: reset + reserve 2,258723ms: parse_in_place 173,992MB/s 2,268857ms: ys2evt_parse 173,215MB/s 2,271363ms: jni_ys2evt_parse/call_parse 173,024MB/s 2,286242ms: jni_ys2evt_parse 171,898MB/s java:ys2evtBuf: 2.296298ms 171.145MB/s 393000B call:evtBuf: 8.421487ms 46.666MB/s 393000B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml ys.length=783000 call:edn... java:ys2edn... 0,092991ms: jni_ys2edn_parse/GetByteArray(src) 0,835160ms: jni_ys2edn_parse/GetByteArray(dst) 0,001818ms: jni_ys2evt_parse/GetStringUTFChars() 0,945805ms: jni_ys2edn_parse/get_jni_data 827,866MB/s 0,002132ms: reset + reserve 14,063376ms: parse_in_place 55,677MB/s 0,322975ms: ys2edn_retry_get 14,405284ms: ys2edn_parse 54,355MB/s 1,018919ms: jni_ys2edn_parse/release 16,381964ms: jni_ys2edn_parse 47,796MB/s java:ys2edn: 16.394997ms 47.758MB/s 783000B call:edn: 23.038956ms 33.986MB/s 783000B call:ednBuf... java:ys2ednBuf... 0,001667ms: jni_ys2edn_parse/get_jni_data 469706,048MB/s 0,001485ms: reset + reserve 13,745988ms: parse_in_place 56,962MB/s 0,314753ms: ys2edn_retry_get 14,076574ms: ys2edn_parse 55,624MB/s 14,081208ms: jni_ys2edn_parse/call_parse 55,606MB/s 14,093569ms: jni_ys2edn_parse 55,557MB/s java:ys2ednBuf: 14.105751ms 55.509MB/s 783000B call:ednBuf: 15.099686ms 51.855MB/s 783000B call:evt... java:ys2evtBuf... 0,077090ms: jni_ys2evt_parse/GetByteArray(src) 8,194263ms: jni_ys2evt_parse/GetIntArray(dst) 0,002475ms: jni_ys2evt_parse/GetStringUTFChars() 8,297913ms: jni_ys2evt_parse/get_jni 94,361MB/s 0,001157ms: reset + reserve 4,913968ms: parse_in_place 159,342MB/s 4,923253ms: ys2evt_parse 159,041MB/s 4,926467ms: jni_ys2evt_parse/call_parse 158,937MB/s 0,073749ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,589391ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000823ms: jni_ys2evt_parse/ReleaseStringUTFChars() 5,674467ms: jni_ys2evt_parse/release 18,906420ms: jni_ys2evt_parse 41,415MB/s java:ys2evt: 18.918636ms 41.388MB/s 783000B call:evt: 21.359373ms 36.658MB/s 783000B call:evtBuf... java:ys2evtBuf... 0,002228ms: jni_ys2evt_parse/get_jni 351436,256MB/s 0,000757ms: reset + reserve 4,970937ms: parse_in_place 157,516MB/s 4,981731ms: ys2evt_parse 157,174MB/s 4,984258ms: jni_ys2evt_parse/call_parse 157,095MB/s 5,000104ms: jni_ys2evt_parse 156,597MB/s java:ys2evtBuf: 5.007758ms 156.357MB/s 783000B call:evtBuf: 10.750488ms 72.834MB/s 783000B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml ys.length=3894000 call:edn... java:ys2edn... 0,298067ms: jni_ys2edn_parse/GetByteArray(src) 1,110594ms: jni_ys2edn_parse/GetByteArray(dst) 0,001205ms: jni_ys2evt_parse/GetStringUTFChars() 1,427416ms: jni_ys2edn_parse/get_jni_data 2728,006MB/s 0,001941ms: reset + reserve 84,243172ms: parse_in_place 46,223MB/s 0,000034ms: ys2edn_retry_get 84,266029ms: ys2edn_parse 46,211MB/s 1,505299ms: jni_ys2edn_parse/release 87,211784ms: jni_ys2edn_parse 44,650MB/s java:ys2edn: 87.222893ms 44.644MB/s 3894000B java:ys2edn... 0,459691ms: jni_ys2edn_parse/GetByteArray(src) 5,877667ms: jni_ys2edn_parse/GetByteArray(dst) 0,001699ms: jni_ys2evt_parse/GetStringUTFChars() 6,357506ms: jni_ys2edn_parse/get_jni_data 612,504MB/s 0,135379ms: reset + reserve 75,206131ms: parse_in_place 51,778MB/s 2,077905ms: ys2edn_retry_get 77,437271ms: ys2edn_parse 50,286MB/s 2,688614ms: jni_ys2edn_parse/release 86,492966ms: jni_ys2edn_parse 45,021MB/s java:ys2edn: 86.502785ms 45.016MB/s 3894000B call:edn: 184.363892ms 21.121MB/s 3894000B call:ednBuf... java:ys2ednBuf... 0,001956ms: jni_ys2edn_parse/get_jni_data 1990797,568MB/s 0,002000ms: reset + reserve 71,676682ms: parse_in_place 54,327MB/s 0,000020ms: ys2edn_retry_get 71,692154ms: ys2edn_parse 54,316MB/s 71,694817ms: jni_ys2edn_parse/call_parse 54,314MB/s 71,707664ms: jni_ys2edn_parse 54,304MB/s java:ys2ednBuf: 71.715118ms 54.298MB/s 3894000B java:ys2ednBuf... 0,003304ms: jni_ys2edn_parse/get_jni_data 1178571,392MB/s 0,001898ms: reset + reserve 70,793625ms: parse_in_place 55,005MB/s 2,278467ms: ys2edn_retry_get 73,093513ms: ys2edn_parse 53,274MB/s 73,096558ms: jni_ys2edn_parse/call_parse 53,272MB/s 73,112923ms: jni_ys2edn_parse 53,260MB/s java:ys2ednBuf: 73.125305ms 53.251MB/s 3894000B call:ednBuf: 150.192001ms 25.927MB/s 3894000B call:evt... java:ys2evtBuf... 0,428085ms: jni_ys2evt_parse/GetByteArray(src) 8,114376ms: jni_ys2evt_parse/GetIntArray(dst) 0,001871ms: jni_ys2evt_parse/GetStringUTFChars() 8,565105ms: jni_ys2evt_parse/get_jni 454,635MB/s 0,000878ms: reset + reserve 21,515081ms: parse_in_place 180,989MB/s 21,525742ms: ys2evt_parse 180,900MB/s 21,528959ms: jni_ys2evt_parse/call_parse 180,873MB/s 0,447203ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,624267ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,001076ms: jni_ys2evt_parse/ReleaseStringUTFChars() 6,085247ms: jni_ys2evt_parse/release 36,185322ms: jni_ys2evt_parse 107,613MB/s java:ys2evt: 36.195786ms 107.582MB/s 3894000B call:evt: 40.740360ms 95.581MB/s 3894000B call:evtBuf... java:ys2evtBuf... 0,002188ms: jni_ys2evt_parse/get_jni 1779707,520MB/s 0,000720ms: reset + reserve 21,331545ms: parse_in_place 182,547MB/s 21,341160ms: ys2evt_parse 182,464MB/s 21,343897ms: jni_ys2evt_parse/call_parse 182,441MB/s 21,357212ms: jni_ys2evt_parse 182,327MB/s java:ys2evtBuf: 21.365120ms 182.260MB/s 3894000B call:evtBuf: 27.394588ms 142.145MB/s 3894000B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml ys.length=8784000 call:edn... java:ys2edn... 1,007704ms: jni_ys2edn_parse/GetByteArray(src) 1,068609ms: jni_ys2edn_parse/GetByteArray(dst) 0,001282ms: jni_ys2evt_parse/GetStringUTFChars() 2,093804ms: jni_ys2edn_parse/get_jni_data 4195,235MB/s 0,001659ms: reset + reserve 151,732574ms: parse_in_place 57,891MB/s 0,000019ms: ys2edn_retry_get 151,752396ms: ys2edn_parse 57,884MB/s 2,056400ms: jni_ys2edn_parse/release 155,915070ms: jni_ys2edn_parse 56,338MB/s java:ys2edn: 155.926132ms 56.334MB/s 8784000B java:ys2edn... 0,968508ms: jni_ys2edn_parse/GetByteArray(src) 11,112817ms: jni_ys2edn_parse/GetByteArray(dst) 0,001311ms: jni_ys2evt_parse/GetStringUTFChars() 12,105033ms: jni_ys2edn_parse/get_jni_data 725,649MB/s 0,002236ms: reset + reserve 136,510681ms: parse_in_place 64,347MB/s 4,374031ms: ys2edn_retry_get 140,909515ms: ys2edn_parse 62,338MB/s 5,479193ms: jni_ys2edn_parse/release 158,506912ms: jni_ys2edn_parse 55,417MB/s java:ys2edn: 158.517197ms 55.414MB/s 8784000B call:edn: 336.565643ms 26.099MB/s 8784000B call:ednBuf... java:ys2ednBuf... 0,001840ms: jni_ys2edn_parse/get_jni_data 4773913,088MB/s 0,002572ms: reset + reserve 136,738342ms: parse_in_place 64,239MB/s 0,000035ms: ys2edn_retry_get 136,756714ms: ys2edn_parse 64,231MB/s 136,759857ms: jni_ys2edn_parse/call_parse 64,229MB/s 136,772003ms: jni_ys2edn_parse 64,224MB/s java:ys2ednBuf: 136.783600ms 64.218MB/s 8784000B java:ys2ednBuf... 0,003399ms: jni_ys2edn_parse/get_jni_data 2584289,536MB/s 0,002487ms: reset + reserve 136,512512ms: parse_in_place 64,346MB/s 4,273001ms: ys2edn_retry_get 140,816635ms: ys2edn_parse 62,379MB/s 140,819626ms: jni_ys2edn_parse/call_parse 62,378MB/s 140,835098ms: jni_ys2edn_parse 62,371MB/s java:ys2ednBuf: 140.862976ms 62.358MB/s 8784000B call:ednBuf: 286.745575ms 30.633MB/s 8784000B call:evt... java:ys2evtBuf... 1,004054ms: jni_ys2evt_parse/GetByteArray(src) 8,102132ms: jni_ys2evt_parse/GetIntArray(dst) 0,002797ms: jni_ys2evt_parse/GetStringUTFChars() 9,131552ms: jni_ys2evt_parse/get_jni 961,939MB/s 0,000792ms: reset + reserve 49,813385ms: parse_in_place 176,338MB/s 49,824718ms: ys2evt_parse 176,298MB/s 49,827583ms: jni_ys2evt_parse/call_parse 176,288MB/s 1,033283ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,743564ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,001015ms: jni_ys2evt_parse/ReleaseStringUTFChars() 6,790257ms: jni_ys2evt_parse/release 65,755936ms: jni_ys2evt_parse 133,585MB/s java:ys2evt: 65.765808ms 133.565MB/s 8784000B call:evt: 72.783669ms 120.686MB/s 8784000B call:evtBuf... java:ys2evtBuf... 0,018377ms: jni_ys2evt_parse/get_jni 477988,768MB/s 0,000729ms: reset + reserve 50,390553ms: parse_in_place 174,318MB/s 50,402847ms: ys2evt_parse 174,276MB/s 50,405594ms: jni_ys2evt_parse/call_parse 174,266MB/s 50,435165ms: jni_ys2evt_parse 174,164MB/s java:ys2evtBuf: 50.444641ms 174.131MB/s 8784000B call:evtBuf: 56.956013ms 154.224MB/s 8784000B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json ys.length=3894002 call:edn... java:ys2edn... 0,330166ms: jni_ys2edn_parse/GetByteArray(src) 1,096978ms: jni_ys2edn_parse/GetByteArray(dst) 0,002020ms: jni_ys2evt_parse/GetStringUTFChars() 1,450136ms: jni_ys2edn_parse/get_jni_data 2685,267MB/s 0,002442ms: reset + reserve 71,063118ms: parse_in_place 54,796MB/s 0,000024ms: ys2edn_retry_get 71,079315ms: ys2edn_parse 54,784MB/s 1,544521ms: jni_ys2edn_parse/release 74,087189ms: jni_ys2edn_parse 52,560MB/s java:ys2edn: 74.098160ms 52.552MB/s 3894002B java:ys2edn... 0,426603ms: jni_ys2edn_parse/GetByteArray(src) 1,911429ms: jni_ys2edn_parse/GetByteArray(dst) 0,001644ms: jni_ys2evt_parse/GetStringUTFChars() 2,354758ms: jni_ys2edn_parse/get_jni_data 1653,674MB/s 0,001464ms: reset + reserve 69,939552ms: parse_in_place 55,677MB/s 2,210632ms: ys2edn_retry_get 72,166901ms: ys2edn_parse 53,958MB/s 2,714381ms: jni_ys2edn_parse/release 77,247299ms: jni_ys2edn_parse 50,410MB/s java:ys2edn: 77.256142ms 50.404MB/s 3894002B call:edn: 162.865555ms 23.909MB/s 3894002B call:ednBuf... java:ys2ednBuf... 0,002270ms: jni_ys2edn_parse/get_jni_data 1715419,392MB/s 0,002090ms: reset + reserve 70,507896ms: parse_in_place 55,228MB/s 0,000025ms: ys2edn_retry_get 70,521866ms: ys2edn_parse 55,217MB/s 70,524414ms: jni_ys2edn_parse/call_parse 55,215MB/s 70,535355ms: jni_ys2edn_parse 55,206MB/s java:ys2ednBuf: 70.543930ms 55.200MB/s 3894002B java:ys2ednBuf... 0,001999ms: jni_ys2edn_parse/get_jni_data 1947974,912MB/s 0,017705ms: reset + reserve 71,052269ms: parse_in_place 54,805MB/s 2,230100ms: ys2edn_retry_get 73,315285ms: ys2edn_parse 53,113MB/s 73,318420ms: jni_ys2edn_parse/call_parse 53,111MB/s 73,327438ms: jni_ys2edn_parse 53,104MB/s java:ys2ednBuf: 73.337601ms 53.097MB/s 3894002B call:ednBuf: 147.542007ms 26.392MB/s 3894002B call:evt... java:ys2evtBuf... 0,442496ms: jni_ys2evt_parse/GetByteArray(src) 8,129780ms: jni_ys2evt_parse/GetIntArray(dst) 0,002011ms: jni_ys2evt_parse/GetStringUTFChars() 8,595941ms: jni_ys2evt_parse/get_jni 453,005MB/s 0,000894ms: reset + reserve 21,720827ms: parse_in_place 179,275MB/s 21,730803ms: ys2evt_parse 179,193MB/s 21,734653ms: jni_ys2evt_parse/call_parse 179,161MB/s 0,421911ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,674166ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000743ms: jni_ys2evt_parse/ReleaseStringUTFChars() 6,109232ms: jni_ys2evt_parse/release 36,447289ms: jni_ys2evt_parse 106,839MB/s java:ys2evt: 36.457016ms 106.811MB/s 3894002B call:evt: 40.910328ms 95.184MB/s 3894002B call:evtBuf... java:ys2evtBuf... 0,002744ms: jni_ys2evt_parse/get_jni 1419096,960MB/s 0,000659ms: reset + reserve 21,838436ms: parse_in_place 178,310MB/s 21,850664ms: ys2evt_parse 178,210MB/s 21,853758ms: jni_ys2evt_parse/call_parse 178,185MB/s 21,869469ms: jni_ys2evt_parse 178,057MB/s java:ys2evtBuf: 21.878090ms 177.986MB/s 3894002B call:evtBuf: 28.053461ms 138.806MB/s 3894002B ----- /home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml ys.length=8784002 call:edn... java:ys2edn... 1,888226ms: jni_ys2edn_parse/GetByteArray(src) 2,544243ms: jni_ys2edn_parse/GetByteArray(dst) 0,002253ms: jni_ys2evt_parse/GetStringUTFChars() 4,454822ms: jni_ys2edn_parse/get_jni_data 1971,796MB/s 0,002021ms: reset + reserve 136,622375ms: parse_in_place 64,294MB/s 0,000020ms: ys2edn_retry_get 136,640030ms: ys2edn_parse 64,286MB/s 2,347279ms: jni_ys2edn_parse/release 143,458054ms: jni_ys2edn_parse 61,230MB/s java:ys2edn: 143.469604ms 61.226MB/s 8784002B java:ys2edn... 1,991961ms: jni_ys2edn_parse/GetByteArray(src) 11,108978ms: jni_ys2edn_parse/GetByteArray(dst) 0,002018ms: jni_ys2evt_parse/GetStringUTFChars() 13,122984ms: jni_ys2edn_parse/get_jni_data 669,360MB/s 0,001842ms: reset + reserve 136,570084ms: parse_in_place 64,319MB/s 4,254542ms: ys2edn_retry_get 140,848358ms: ys2edn_parse 62,365MB/s 5,660944ms: jni_ys2edn_parse/release 159,643997ms: jni_ys2edn_parse 55,022MB/s java:ys2edn: 159.653351ms 55.019MB/s 8784002B call:edn: 323.496796ms 27.153MB/s 8784002B call:ednBuf... java:ys2ednBuf... 0,001906ms: jni_ys2edn_parse/get_jni_data 4608605,696MB/s 0,002339ms: reset + reserve 136,883469ms: parse_in_place 64,171MB/s 0,000038ms: ys2edn_retry_get 136,904175ms: ys2edn_parse 64,162MB/s 136,907028ms: jni_ys2edn_parse/call_parse 64,160MB/s 136,920837ms: jni_ys2edn_parse 64,154MB/s java:ys2ednBuf: 136.933243ms 64.148MB/s 8784002B java:ys2ednBuf... 0,002604ms: jni_ys2edn_parse/get_jni_data 3373272,576MB/s 0,002970ms: reset + reserve 136,688339ms: parse_in_place 64,263MB/s 4,348243ms: ys2edn_retry_get 141,061874ms: ys2edn_parse 62,271MB/s 141,064972ms: jni_ys2edn_parse/call_parse 62,269MB/s 141,080063ms: jni_ys2edn_parse 62,263MB/s java:ys2ednBuf: 141.092941ms 62.257MB/s 8784002B call:ednBuf: 287.268158ms 30.578MB/s 8784002B call:evt... java:ys2evtBuf... 0,985668ms: jni_ys2evt_parse/GetByteArray(src) 8,121219ms: jni_ys2evt_parse/GetIntArray(dst) 0,002764ms: jni_ys2evt_parse/GetStringUTFChars() 9,133806ms: jni_ys2evt_parse/get_jni 961,702MB/s 0,000989ms: reset + reserve 50,007858ms: parse_in_place 175,652MB/s 50,019047ms: ys2evt_parse 175,613MB/s 50,022156ms: jni_ys2evt_parse/call_parse 175,602MB/s 0,965973ms: jni_ys2evt_parse/ReleaseByteArray(src) 5,844872ms: jni_ys2evt_parse/ReleaseIntArray(dst) 0,000858ms: jni_ys2evt_parse/ReleaseStringUTFChars() 6,827047ms: jni_ys2evt_parse/release 65,990753ms: jni_ys2evt_parse 133,110MB/s java:ys2evt: 66.000282ms 133.090MB/s 8784002B call:evt: 73.179230ms 120.034MB/s 8784002B call:evtBuf... java:ys2evtBuf... 0,004144ms: jni_ys2evt_parse/get_jni 2119691,648MB/s 0,000963ms: reset + reserve 49,682919ms: parse_in_place 176,801MB/s 49,695457ms: ys2evt_parse 176,757MB/s 49,698326ms: jni_ys2evt_parse/call_parse 176,746MB/s 49,730080ms: jni_ys2evt_parse 176,634MB/s java:ys2evtBuf: 49.739895ms 176.599MB/s 8784002B call:evtBuf: 56.250931ms 156.157MB/s 8784002B --- common/vars.mk | 2 +- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 320 +++++++----- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 52 +- rapidyaml/native/ysparse_common.hpp | 13 +- .../main/java/org/rapidyaml/Rapidyaml.java | 174 +++++-- .../java/org/rapidyaml/cmp/CmpEdnEvt.java | 159 ++++-- .../java/org/rapidyaml/RapidyamlTest.java | 467 ++++++++++++------ 7 files changed, 821 insertions(+), 366 deletions(-) diff --git a/common/vars.mk b/common/vars.mk index 0bae90453..704856e91 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -227,7 +227,7 @@ RAPIDYAML_TAG ?= 8c37616378aefd376690a19459c31a56ce596b5e RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml RAPIDYAML_BUILD_TYPE ?= Release RAPIDYAML_DBG ?= 0 -RAPIDYAML_TIMED ?= 0 +RAPIDYAML_TIMED ?= 1 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \ diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 48d3d8b87..34730c6cc 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -3,47 +3,21 @@ #include "ysparse_evt.hpp" #include -#ifndef _Included_org_rapidyaml_Rapidyaml -#define _Included_org_rapidyaml_Rapidyaml #ifdef __cplusplus extern "C" { #endif -void throw_java_exception(JNIEnv * env, - const char* type, - const char* msg) -{ - jclass clazz = env->FindClass(type); - if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown - env->ThrowNew(clazz, msg); -} +static C4_NO_INLINE void throw_runtime_exception(JNIEnv * env, const char* msg); +static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg); + -void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg) +JNIEXPORT jlong JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) { - // see https://stackoverflow.com/questions/55013243/jni-custom-exceptions-with-more-than-one-parameter - jclass clazz = env->FindClass("org/rapidyaml/YamlParseErrorException"); - if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown - { - jstring jmsg = env->NewStringUTF(msg); - jint joffset = (jint)offset; - jint jline = (jint)line; - jint jcol = (jint)column; - // see https://www.rgagnon.com/javadetails/java-0286.html - // about the proper signature. - // we want (int, int, int, String): - const char * const signature = "(IIILjava/lang/String;)V"; - jmethodID ctor = env->GetMethodID(clazz, "", signature); - jobject jexc = env->NewObject(clazz, ctor, joffset, jline, jcol, jmsg); - env->Throw((jthrowable)jexc); // https://stackoverflow.com/questions/2455668/jni-cast-between-jobect-and-jthrowable - } + Ryml2Evt *obj = ys2evt_init(); + return (jlong)obj; } - -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_init - * Signature: ()Ljava/lang/Object; - */ JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) { @@ -51,61 +25,132 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) return (jlong)obj; } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_init - * Signature: ()J - */ -JNIEXPORT jlong JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) { - Ryml2Evt *obj = ys2evt_init(); - return (jlong)obj; + ys2evt_destroy((Ryml2Evt*)obj); } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_destroy - * Signature: (Ljava/lang/Object;)V - */ JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) { ys2edn_destroy((Ryml2Edn*)obj); } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_destroy - * Signature: (Ljava/lang/Object;)V - */ -JNIEXPORT void JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) + +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jbyteArray src, jint src_len, + jintArray dst, jint dst_len) { - ys2evt_destroy((Ryml2Evt*)obj); + TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + jbyte* src_ = nullptr; + int* dst_ = nullptr; + const char *filename = nullptr; + jboolean dst_is_copy = false; + jboolean src_is_copy = false; + { + TIMED_SECTION("jni_ys2evt_parse/get_jni", (size_type)src_len); + { + TIMED_SECTION("jni_ys2evt_parse/GetByteArray(src)"); + src_ = env->GetByteArrayElements(src, &src_is_copy); + } + { + // TODO this is __S__L__O__W__ + // + // the problem is with GetIntArrayElements(). we should + // use GetDirectBufferAddress(), but that requires a ByteBuffer->jobject + // instead of a int[]->jintArray + // + // see: + // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time + // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string + TIMED_SECTION("jni_ys2evt_parse/GetIntArray(dst)"); + dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + } + { + TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); + filename = env->GetStringUTFChars(jfilename, 0); + } + } + int rc = 0; + { + TIMED_SECTION("jni_ys2evt_parse/call_parse", (size_type)src_len); + try + { + rc = ys2evt_parse((Ryml2Evt*)obj, filename, + (char*)src_, src_len, + dst_, dst_len); + } + catch (YsParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + catch (std::exception const& exc) + { + throw_runtime_exception(env, exc.what()); + } + } + { + TIMED_SECTION("jni_ys2evt_parse/release"); + { + TIMED_SECTION("jni_ys2evt_parse/ReleaseByteArray(src)"); + env->ReleaseByteArrayElements(src, src_, 0); + } + { + // TODO __S__L__O__W__ + TIMED_SECTION("jni_ys2evt_parse/ReleaseIntArray(dst)"); + env->ReleaseIntArrayElements(dst, dst_, 0); + } + { + TIMED_SECTION("jni_ys2evt_parse/ReleaseStringUTFChars()"); + env->ReleaseStringUTFChars(jfilename, filename); + } + } + return rc; } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_retry_get - * Signature: (Ljava/lang/Object;[BI)I - */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, - jlong obj, - jbyteArray dst, jint dst_len) + +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jobject src, jint src_len, + jobject dst, jint dst_len) { - jboolean dst_is_copy; - jbyte* dst_ = env->GetByteArrayElements(dst, &dst_is_copy); - int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); - env->ReleaseByteArrayElements(dst, dst_, 0); - return rc; + TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + char* src_ = nullptr; + int* dst_ = nullptr; + const char *filename = nullptr; + { + TIMED_SECTION("jni_ys2evt_parse/get_jni", (size_type)src_len); + src_ = (char*)env->GetDirectBufferAddress(src); + dst_ = (int*)env->GetDirectBufferAddress(dst); + filename = env->GetStringUTFChars(jfilename, 0); + if(!src_) + throw_runtime_exception(env, "null pointer: src"); + if(!dst_) + throw_runtime_exception(env, "null pointer: dst"); + } + { + TIMED_SECTION("jni_ys2evt_parse/call_parse", (size_type)src_len); + try + { + return ys2evt_parse((Ryml2Evt*)obj, filename, src_, src_len, dst_, dst_len); + } + catch (YsParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + catch (std::exception const& exc) + { + throw_runtime_exception(env, exc.what()); + } + } + return 0; // this is executed even if there is an exception } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_parse - * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I - */ + JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, jlong obj, jstring jfilename, @@ -153,78 +198,97 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, return rc; } -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_parse - * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I - */ + JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, - jlong obj, jstring jfilename, - jbyteArray src, jint src_len, - jintArray dst, jint dst_len) +Java_org_rapidyaml_Rapidyaml_ys2edn_1parse_1buf(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jobject src, jint src_len, + jobject dst, jint dst_len) { - TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); - jbyte* src_ = nullptr; - int* dst_ = nullptr; + TIMED_SECTION("jni_ys2edn_parse", (size_type)src_len); + char* src_ = nullptr; + char* dst_ = nullptr; const char *filename = nullptr; - jboolean dst_is_copy = false; - jboolean src_is_copy = false; { - TIMED_SECTION("jni_ys2evt_parse/get_jni_data", (size_type)src_len); + TIMED_SECTION("jni_ys2edn_parse/get_jni_data", (size_type)src_len); + src_ = (char*)env->GetDirectBufferAddress(src); + dst_ = (char*)env->GetDirectBufferAddress(dst); + filename = env->GetStringUTFChars(jfilename, 0); + if(!src_) + throw_runtime_exception(env, "null pointer: src"); + if(!dst_) + throw_runtime_exception(env, "null pointer: dst"); + } + { + TIMED_SECTION("jni_ys2edn_parse/call_parse", (size_type)src_len); + try { - TIMED_SECTION("jni_ys2evt_parse/GetByteArray(src)"); - src_ = env->GetByteArrayElements(src, &src_is_copy); + return ys2edn_parse((Ryml2Edn*)obj, filename, src_, src_len, dst_, dst_len); } + catch (YsParseError const& exc) { - // TODO this is __S__L__O__W__ - // - // the problem is with GetIntArrayElements(). we should - // use GetDirectBufferAddress(), but that requires a ByteBuffer->jobject - // instead of a int[]->jintArray - // - // see: - // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time - // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string - TIMED_SECTION("jni_ys2evt_parse/GetIntArray(dst)"); - dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); } + catch (std::exception const& exc) { - TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); - filename = env->GetStringUTFChars(jfilename, 0); + throw_runtime_exception(env, exc.what()); } } - int rc = 0; - try - { - rc = ys2evt_parse((Ryml2Evt*)obj, filename, - (char*)src_, src_len, - dst_, dst_len); - } - catch (YsParseError const& exc) + return 0; // this is executed even if there is an exception +} + + +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, + jlong obj, + jobject dst, jint dst_len) +{ + char* dst_ = nullptr; { - throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + TIMED_SECTION("jni_ys2evt_retry_get/get_dst"); + dst_ = (char*)env->GetDirectBufferAddress(dst); + printf("edn: aqui 1 %p\n", dst_); } + int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); + return rc; +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +static C4_NO_INLINE void throw_java_exception(JNIEnv * env, const char* type, const char* msg) +{ + jclass clazz = env->FindClass(type); + if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown + env->ThrowNew(clazz, msg); +} + +static C4_NO_INLINE void throw_runtime_exception(JNIEnv *env, const char* msg) +{ + throw_java_exception(env, "java/lang/RuntimeException", msg); +} + +static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg) +{ + // see https://stackoverflow.com/questions/55013243/jni-custom-exceptions-with-more-than-one-parameter + jclass clazz = env->FindClass("org/rapidyaml/YamlParseErrorException"); + if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown { - TIMED_SECTION("jni_ys2evt_parse/release"); - { - TIMED_SECTION("jni_ys2evt_parse/ReleaseByteArray(src)"); - env->ReleaseByteArrayElements(src, src_, 0); - } - { - // TODO __S__L__O__W__ - TIMED_SECTION("jni_ys2evt_parse/ReleaseIntArray(dst)"); - env->ReleaseIntArrayElements(dst, dst_, 0); - } - { - TIMED_SECTION("jni_ys2evt_parse/ReleaseStringUTFChars()"); - env->ReleaseStringUTFChars(jfilename, filename); - } + jstring jmsg = env->NewStringUTF(msg); + jint joffset = (jint)offset; + jint jline = (jint)line; + jint jcol = (jint)column; + // see https://www.rgagnon.com/javadetails/java-0286.html + // about the proper signature. + // we want (int, int, int, String): + const char * const signature = "(IIILjava/lang/String;)V"; + jmethodID ctor = env->GetMethodID(clazz, "", signature); + jobject jexc = env->NewObject(clazz, ctor, joffset, jline, jcol, jmsg); + env->Throw((jthrowable)jexc); // https://stackoverflow.com/questions/2455668/jni-cast-between-jobect-and-jthrowable } - return rc; } #ifdef __cplusplus } #endif -#endif diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index daba52eff..74c7a5eb4 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -7,6 +7,14 @@ #ifdef __cplusplus extern "C" { #endif +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init + (JNIEnv *, jobject); + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_init @@ -17,51 +25,51 @@ JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_destroy + * Method: ys2evt_destroy * Signature: (J)V */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy (JNIEnv *, jobject, jlong); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_parse - * Signature: (JLjava/lang/String;[BI[BI)I + * Method: ys2edn_destroy + * Signature: (J)V */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse - (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint); +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy + (JNIEnv *, jobject, jlong); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_retry_get - * Signature: (J[BI)I + * Method: ys2evt_parse + * Signature: (JLjava/lang/String;[BI[II)I */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get - (JNIEnv *, jobject, jlong, jbyteArray, jint); +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_init - * Signature: ()J + * Method: ys2evt_parse_buf + * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/IntBuffer;I)I */ -JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init - (JNIEnv *, jobject); +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf + (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_destroy - * Signature: (J)V + * Method: ys2edn_parse + * Signature: (JLjava/lang/String;[BI[BI)I */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy - (JNIEnv *, jobject, jlong); +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_parse - * Signature: (JLjava/lang/String;[BI[II)I + * Method: ys2edn_parse_buf + * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/ByteBuffer;I)I */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse - (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse_1buf + (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); #ifdef __cplusplus } diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp index 9026f951e..ce3c3d1b4 100644 --- a/rapidyaml/native/ysparse_common.hpp +++ b/rapidyaml/native/ysparse_common.hpp @@ -20,7 +20,9 @@ struct YsParseError : public std::exception #ifndef YSPARSE_TIMED #define TIMED_SECTION(...) +#error #else +#include #include #define TIMED_SECTION(...) timed_section C4_XCAT(ts, __LINE__)(__VA_ARGS__) struct timed_section @@ -29,13 +31,18 @@ struct timed_section ryml::csubstr name; size_type len; myclock::time_point start; - timed_section(ryml::csubstr n, size_type len_=0) : name(n), len(len_), start(myclock::now()) {} + timed_section(ryml::csubstr n, size_type len_=0) + : name(n) + , len(len_) + , start(myclock::now()) + { + } ~timed_section() { - const std::chrono::duration t = myclock::now() - start; + const std::chrono::duration t = myclock::now() - start; fprintf(stderr, "%.6fms: %.*s", t.count(), (int)name.len, name.str); if(len) - fprintf(stderr, " %.3fMB/s", (double)len / t.count() * 1.e-3); + fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); fprintf(stderr, "\n"); } }; diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 58d6cb729..f77b519d7 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -3,28 +3,36 @@ import org.rapidyaml.YamlParseErrorException; import java.nio.charset.StandardCharsets; import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.ByteOrder; /** * Interface with the shared librapidyaml library */ -public class Rapidyaml { +public class Rapidyaml +{ public static String RAPIDYAML_VERSION = "0.8.0"; + private native long ys2evt_init(); private native long ys2edn_init(); - private native void ys2edn_destroy(long ryml2edn); - private native int ys2edn_parse(long ryml2edn, String filename, - byte[] ys, int ys_length, - byte[] edn, int edn_length); - private native int ys2edn_retry_get( - long ryml2edn, byte[] edn, int edn_size - ); - private final long ryml2edn; - private native long ys2evt_init(); private native void ys2evt_destroy(long ryml2evt); + private native void ys2edn_destroy(long ryml2edn); + private native int ys2evt_parse(long ryml2evt, String filename, byte[] ys, int ys_length, int[] evt, int evt_length); + private native int ys2evt_parse_buf(long ryml2evt, String filename, + ByteBuffer ys, int ys_length, + IntBuffer evt, int evt_length); + private native int ys2edn_parse(long ryml2edn, String filename, + byte[] ys, int ys_length, + byte[] edn, int edn_length); + private native int ys2edn_parse_buf(long ryml2edn, String filename, + ByteBuffer ys, int ys_length, + ByteBuffer edn, int edn_length); + + private final long ryml2edn; private final long ryml2evt; public Rapidyaml() @@ -33,6 +41,8 @@ public Rapidyaml() System.loadLibrary(library_name); this.ryml2edn = this.ys2edn_init(); this.ryml2evt = this.ys2evt_init(); + // TODO: receive this argument as ctor parameter + timingEnabled(System.getenv("YS_RYML_TIMER") != null); } // Likely bad idea to implement finalize: @@ -50,41 +60,131 @@ protected void finalize() throws Throwable } } - public String parseYsToEdn(String srcstr) - throws RuntimeException, org.rapidyaml.YamlParseErrorException + + //------------------------ + // EVT + //------------------------ + + public int parseYsToEvt(byte[] src, int[] evts) throws Exception + { + return parseYsToEvt("yamlscript", src, evts); + } + + public int parseYsToEvtBuf(ByteBuffer src, IntBuffer evt) throws Exception + { + return parseYsToEvtBuf("yamlscript", src, evt); + } + + public int parseYsToEvt(String filename, byte[] src, int[] evts) throws Exception + { + long t = timingStart("ys2evtBuf"); + int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); + timingStop("ys2evt", t, src.length); + return required_size; + } + + public int parseYsToEvtBuf(String filename, ByteBuffer src, IntBuffer evt) throws Exception + { + if(!src.isDirect()) + throw new RuntimeException("src must be direct"); + if(!evt.isDirect()) + throw new RuntimeException("evt must be direct"); + // the byte order for src does not matter + // but for evt it really does + if(evt.order() != ByteOrder.nativeOrder()) + throw new RuntimeException("evt byte order must be native"); + long t = timingStart("ys2evtBuf"); + evt.position(evt.capacity()); + int reqsize = ys2evt_parse_buf(this.ryml2evt, filename, src, src.position(), evt, evt.capacity()); + if(reqsize <= evt.capacity()) { + evt.position(reqsize); + } + timingStop("ys2evtBuf", t, src.position()); + return reqsize; + } + + public static IntBuffer mkIntBuffer(int numInts) { - String filename = "yamlscript"; // fixme - long t = System.nanoTime(); - byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); - int edn_size = 10 * src.length; - byte[] edn = new byte[edn_size]; - int required_size = ys2edn_parse(this.ryml2edn, filename, src, src.length, edn, edn_size); - if(required_size > edn_size) { - edn_size = required_size; - edn = new byte[edn_size]; - required_size = ys2edn_retry_get(this.ryml2edn, edn, edn_size); - if(required_size != edn_size) { - throw new RuntimeException("inconsistent size"); - } + ByteBuffer bb = ByteBuffer.allocateDirect(/*numBytes*/4 * numInts); + // !!! need to explicitly set the byte order to the native order + return bb.order(ByteOrder.nativeOrder()).asIntBuffer(); + } + + + //------------------------ + // EDN + //------------------------ + + public int parseYsToEdn(byte[] src, byte[] edn) throws Exception + { + return parseYsToEdn("yamlscript", src, edn); + } + + public int parseYsToEdnBuf(ByteBuffer src, ByteBuffer edn) throws Exception + { + return parseYsToEdnBuf("yamlscript", src, edn); + } + + public int parseYsToEdn(String filename, byte[] src, byte[] edn) throws Exception + { + long t = timingStart("ys2edn"); + int ret = ys2edn_parse(this.ryml2edn, filename, src, src.length, edn, edn.length); + timingStop("ys2edn", t, src.length); + return ret; + } + + public int parseYsToEdnBuf(String filename, ByteBuffer src, ByteBuffer edn) throws Exception + { + if(!src.isDirect()) + throw new RuntimeException("src must be direct"); + if(!edn.isDirect()) + throw new RuntimeException("edn must be direct"); + long t = timingStart("ys2ednBuf"); + edn.position(edn.capacity()); + int reqsize = ys2edn_parse_buf(this.ryml2edn, filename, src, src.position(), edn, edn.capacity()); + if(reqsize <= edn.capacity()) { + edn.position(reqsize); } - if (System.getenv("YS_RYML_TIMER") != null) { + timingStop("ys2ednBuf", t, src.position()); + return reqsize; + } + + + //------------------------ + // TIME + //------------------------ + + private boolean showTiming = true; + + public void timingEnabled(boolean yes) + { + showTiming = yes; + } + + private long timingStart(String name) + { + if(showTiming) { + System.out.printf(" java:%s...\n", name); + return System.nanoTime(); + } + return 0; + } + + private void timingStop(String name, long t) + { + if(showTiming) { t = System.nanoTime() - t; - System.out.printf(" edn@java=%.6fms\n", (double)t / 1.e6); + System.out.printf(" java:%s: %.6fms\n", name, (float)t/1.e6f); } - String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); - return ret; } - public int parseYsToEvt(byte[] src, int[] evts) - throws RuntimeException, org.rapidyaml.YamlParseErrorException + private void timingStop(String name, long t, int numBytes) { - String filename = "yamlscript"; // fixme - long t = System.nanoTime(); - int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); - if (System.getenv("YS_RYML_TIMER") != null) { + if(showTiming) { t = System.nanoTime() - t; - System.out.printf(" evt@java=%.6fms\n", (double)t / 1.e6); + float dt = (float)t; + float fb = (float)numBytes; + System.out.printf(" java:%s: %.6fms %.3fMB/s %dB\n", name, dt/1.e6f, 1.e3f*fb/dt, numBytes); } - return required_size; } } diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java index 192ffc9a0..cd86ae189 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java @@ -5,49 +5,144 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; // https://stackoverflow.com/questions/804466/how-do-i-create-executable-java-program public class CmpEdnEvt { - public static void main(String[] args) throws Throwable + public static void main(String[] args) throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); - int evtSize = 10000000; - compareEdnEvt(evtSize, rapidyaml, "./yamllm.ys"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); - compareEdnEvt(evtSize, rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); - } - - public static void compareEdnEvt(int evtSize, Rapidyaml rapidyaml, String path) throws Throwable - { - String ys = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); - int[] evt = new int[evtSize]; + rapidyaml.timingEnabled(true); + compareEdnEvt(rapidyaml, "./yamllm.ys"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); + compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); + } + + public static void compareEdnEvt(Rapidyaml rapidyaml, String path) throws Exception + { + String ys_ = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); + byte[] ys = ys_.getBytes(StandardCharsets.UTF_8); + byte[] ysarr = new byte[ys.length]; + ByteBuffer ysbuf = ByteBuffer.allocateDirect(ys.length); // - long t0 = System.nanoTime(); - byte[] ysBytes = ys.getBytes(StandardCharsets.UTF_8); - long tys2Bytes = System.nanoTime() - t0; System.out.printf("-----\n"); System.out.printf("%s\n", path); - System.out.printf(" ys.length=%d\n", ys.length()); - System.out.printf(" ys2bytes=%fms\n", (double)tys2Bytes / 1.e6); + System.out.printf(" ys.length=%d\n", ys.length); // - System.out.printf(" edn...\n"); - t0 = System.nanoTime(); - String edn = rapidyaml.parseYsToEdn(ys); - long tEdn = System.nanoTime() - t0; - System.out.printf(" edn=%.6fms, length=%d -> %dB @%.3fMB/s\n", (double)tEdn / 1.e6, edn.length(), edn.length(), (double)ysBytes.length / tEdn * 1.e3); + long t = timingStart("edn"); + String ednStr = callEdn(rapidyaml, ys, ysarr); + timingStop("edn", t, ys.length); // - System.out.printf(" evt...\n"); - t0 = System.nanoTime(); - int numEvts = rapidyaml.parseYsToEvt(ysBytes, evt); - long tEvt = System.nanoTime() - t0; - System.out.printf(" evt=%.6fms, length=%d -> %dB @%.3fMB/s\n", (double)tEvt / 1.e6, numEvts, 4*numEvts, (double)ysBytes.length / tEvt * 1.e3); + t = timingStart("ednBuf"); + ByteBuffer edn = callEdnBuf(rapidyaml, ys, ysbuf); + timingStop("ednBuf", t, ys.length); // + t = timingStart("evt"); + int[] evtarr = callEvt(rapidyaml, ys, ysarr); + timingStop("evt", t, ys.length); + // + t = timingStart("evtBuf"); + IntBuffer evtbuf = callEvtBuf(rapidyaml, ys, ysbuf); + timingStop("evtBuf", t, ys.length); + } + + static String callEdn(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception + { + System.arraycopy(src, 0, srcbuf, 0, src.length); + byte[] edn = new byte[10000000]; + int reqsize = rapidyaml.parseYsToEdn(srcbuf, edn); + if(reqsize > edn.length) { + edn = new byte[reqsize]; + System.arraycopy(src, 0, srcbuf, 0, src.length); + int reqsize2 = rapidyaml.parseYsToEdn(srcbuf, edn); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + String ret = new String(edn, 0, reqsize-1, StandardCharsets.UTF_8); + return ret; + } + + static ByteBuffer callEdnBuf(Rapidyaml rapidyaml, byte[] src, ByteBuffer srcbuf) throws Exception + { + srcbuf.position(0); + srcbuf.put(src); + ByteBuffer edn = ByteBuffer.allocateDirect(10000000); + int reqsize = rapidyaml.parseYsToEdnBuf(srcbuf, edn); + if(reqsize > edn.capacity()) { + edn = ByteBuffer.allocateDirect(reqsize); + srcbuf.position(0); + srcbuf.put(src); + int reqsize2 = rapidyaml.parseYsToEdnBuf(srcbuf, edn); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + edn.position(reqsize); + return edn; + } + + static int[] callEvt(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception + { + System.arraycopy(src, 0, srcbuf, 0, src.length); + int[] evt = new int[10000000]; + int reqsize = rapidyaml.parseYsToEvt(srcbuf, evt); + if(reqsize > evt.length) { + evt = new int[reqsize]; + System.arraycopy(src, 0, srcbuf, 0, src.length); + int reqsize2 = rapidyaml.parseYsToEvt(srcbuf, evt); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + return evt; + } + int[] ret = new int[reqsize]; + System.arraycopy(evt, 0, ret, 0, reqsize); + return ret; + } + + static IntBuffer callEvtBuf(Rapidyaml rapidyaml, byte[] src, ByteBuffer srcbuf) throws Exception + { + srcbuf.position(0); + srcbuf.put(src); + IntBuffer evt = Rapidyaml.mkIntBuffer(10000000); + int reqsize = rapidyaml.parseYsToEvtBuf(srcbuf, evt); + if(reqsize > evt.capacity()) { + evt = Rapidyaml.mkIntBuffer(reqsize); + srcbuf.position(0); + srcbuf.put(src); + int reqsize2 = rapidyaml.parseYsToEvtBuf(srcbuf, evt); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + evt.position(reqsize); + return evt; + } + + static private long timingStart(String name) + { + System.out.printf(" call:%s...\n", name); + return System.nanoTime(); + } + static private void timingStop(String name, long t) + { + t = System.nanoTime() - t; + System.out.printf(" call:%s: %.6fms\n", name, (float)t/1.e6f); + } + static private void timingStop(String name, long t, int numBytes) + { + t = System.nanoTime() - t; + float dt = (float)t; + float fb = (float)numBytes; + System.out.printf(" call:%s: %.6fms %.3fMB/s %dB\n", name, dt/1.e6f, 1.e3f*fb/dt, numBytes); } } diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index b78b876be..174dde7aa 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -6,10 +6,13 @@ import junit.framework.TestSuite; import java.nio.charset.StandardCharsets; import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.IntBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; + /** * Unit test for simple App. */ @@ -33,147 +36,6 @@ public static Test suite() return new TestSuite(RapidyamlTest.class); } - private void testEdn_(String ys, String expected) - { - Rapidyaml rapidyaml = new Rapidyaml(); - try { - String actual = rapidyaml.parseYsToEdn(ys); - try { - assertEquals(expected, actual); - assertEquals(expected.length(), actual.length()); - } - catch (Exception e) { - System.err.println("expected:"); - System.err.println(expected); - System.err.println("actual"); - System.err.println(actual); - throw e; - } - } - catch (YamlParseErrorException e) { - fail("parse error:\n" + e.getMessage()); - } - } - - // the result is an array of integers, but we use this to simplify - // running the tests - private class ExpectedEvent - { - int flags; - int str_start; - int str_len; - String str; - ExpectedEvent(int flags) - { - this.flags = flags; - this.str_start = 0; - this.str_len = 0; - this.str = ""; - } - ExpectedEvent(int flags, int str_start, int str_len, String str) - { - this.flags = flags; - this.str_start = str_start; - this.str_len = str_len; - this.str = str; - } - int required_size() - { - return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; - } - }; - private static int required_size_(ExpectedEvent[] evts) - { - int sz = 0; - for(int i = 0; i < evts.length; ++i) { - sz += evts[i].required_size(); - } - return sz; - } - - private void testEvt_(String ys, ExpectedEvent[] expected) - { - boolean dbglog = false; - Rapidyaml rapidyaml = new Rapidyaml(); - try { - int[] actual = new int[2 * required_size_(expected)]; - byte[] src = ys.getBytes(StandardCharsets.UTF_8); - int numEvts = rapidyaml.parseYsToEvt(src, actual); - assertTrue(numEvts < actual.length); - try { - int ia = 0; - int ie = 0; - int status = 1; - while(true) { - if((ia < numEvts) != (ie < expected.length)) { - status = 0; - break; - } - if(ia >= numEvts) - break; - if(ie >= expected.length) - break; - int cmp = 1; - if(dbglog) - System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); - cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; - if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { - cmp &= (ia + 2 < numEvts) ? 1 : 0; - if(cmp != 0) { - cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; - cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; - if(dbglog) - System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); - if(cmp != 0) { - cmp &= (actual[ia + 1] >= 0) ? 1 : 0; - cmp &= (actual[ia + 2] >= 0) ? 1 : 0; - cmp &= (actual[ia + 1] + actual[ia + 2] <= src.length) ? 1 : 0; - if(cmp != 0) { - String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); - cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; - if(dbglog) - System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); - } - else { - if(dbglog) - System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); - } - } - } - } - if(dbglog) - System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); - status &= cmp; - ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; - ++ie; - } - if(required_size_(expected) != numEvts) - status = 0; - assertEquals(1, status); - } - catch (Exception e) { - System.err.println("expected:"); - System.err.println(expected); - System.err.println("actual"); - System.err.println(actual); - throw e; - } - } - catch (YamlParseErrorException e) { - fail("parse error:\n" + e.getMessage()); - } - } - - ExpectedEvent mkev(int flags) - { - return new ExpectedEvent(flags); - } - - ExpectedEvent mkev(int flags, int offs, int len, String ref) - { - return new ExpectedEvent(flags, offs, len, ref); - } - public void testPlainMap() { String ys = "a: 1"; @@ -429,13 +291,13 @@ public void testFilterCase() testEvt_(ys, expected); } - public void testFailure() + public void testFailure() throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); String ys = ": : : :"; boolean gotit = false; try { - rapidyaml.parseYsToEdn(ys); + callEdn(ys); } catch(YamlParseErrorException e) { gotit = true; @@ -451,6 +313,325 @@ public void testFailure() catch(Exception e) { fail("wrong exception type"); } + //catch(Throwable e) { + // throw e; + // //fail("wrong exception type"); + //} assertTrue(gotit); } + + + private void testEdn_(String ys, String expected) + { + String actual; + try { + actual = callEdn(ys); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = ""; + } + try { + cmpEdn_(actual, expected); + } + catch (Exception e) { + System.err.printf("error: edn (no buf)"); + } + //------ + try { + actual = callEdnBuf(ys); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = ""; + } + try { + cmpEdn_(actual, expected); + } + catch (Exception e) { + System.err.printf("error: ednbuf"); + } + } + + private void testEvt_(String ys, ExpectedEvent[] expected) + { + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + byte[] srcbuf = new byte[src.length]; + int[] actual; + try { + actual = callEvt(src, srcbuf); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = new int[1]; + } + try { + cmpEvt_(ys, srcbuf, actual, expected); + } + catch (Exception e) { + System.err.printf("error: evt (no buf)"); + throw e; + } + //------ + src = ys.getBytes(StandardCharsets.UTF_8); + ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); + bbuf.put(src); + IntBuffer buf; + try { + buf = callEvtBuf(src, bbuf); + actual = buf2arr(buf); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = new int[1]; + } + try { + cmpEvt_(ys, srcbuf, actual, expected); + } + catch (Exception e) { + System.err.printf("error: evtbuf"); + throw e; + } + } + + private void cmpEdn_(String actual, String expected) throws Exception + { + try { + assertEquals(expected, actual); + assertEquals(expected.length(), actual.length()); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + + boolean dbglog = true; + private void cmpEvt_(String ys, byte[] src, int[] actual, ExpectedEvent[] expected) + { + if(dbglog) { + System.out.printf("----------------------\n~~~\n%s\n~~~\n", ys); + } + int numEvts = actual.length; + try { + int ia = 0; + int ie = 0; + int status = 1; + while(true) { + if((ia < numEvts) != (ie < expected.length)) { + System.out.printf("status=%d szActual=%d szExpected=%d\n", status, numEvts, ExpectedEvent.required_size_(expected)); + status = 0; + break; + } + if(ia >= numEvts) + break; + if(ie >= expected.length) + break; + int cmp = 1; + if(dbglog) + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; + if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { + cmp &= (ia + 2 < numEvts) ? 1 : 0; + if(cmp != 0) { + cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; + cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; + if(dbglog) + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + if(cmp != 0) { + cmp &= (actual[ia + 1] >= 0) ? 1 : 0; + cmp &= (actual[ia + 2] >= 0) ? 1 : 0; + cmp &= (actual[ia + 1] + actual[ia + 2] <= src.length) ? 1 : 0; + if(cmp != 0) { + String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); + cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; + if(dbglog) + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + } + else { + if(dbglog) + System.out.printf(" BAD RANGE len=%d", src.length); + } + } + } + } + if(dbglog) + System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); + status &= cmp; + ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; + ++ie; + } + if(ExpectedEvent.required_size_(expected) != numEvts) + status = 0; + assertEquals(1, status); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + + public static String buf2str(ByteBuffer edn) + { + int size = edn.position(); + size = size > 0 ? size - 1 : 0; + edn.position(0); + edn.limit(size); + return StandardCharsets.UTF_8.decode(edn).toString(); + } + + public static int[] buf2arr(IntBuffer evt) + { + int[] ret = new int[evt.position()]; + for(int i = 0; i < evt.position(); ++i) { + ret[i] = evt.get(i); + } + return ret; + } + + static String callEdn(String orig) throws Exception + { + byte[] src = orig.getBytes(StandardCharsets.UTF_8); + byte[] srcbuf = new byte[src.length]; + return callEdn(src, srcbuf); + } + + static String callEdnBuf(String orig) throws Exception + { + byte[] src = orig.getBytes(StandardCharsets.UTF_8); + ByteBuffer srcbuf = ByteBuffer.allocateDirect(src.length); + ByteBuffer edn = callEdnBuf(src, srcbuf); + return buf2str(edn); + } + + static String callEdn(byte[] src, byte[] srcbuf) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + System.arraycopy(src, 0, srcbuf, 0, src.length); + byte[] edn = new byte[10 * src.length]; + int reqsize = rapidyaml.parseYsToEdn(srcbuf, edn); + if(reqsize > edn.length) { + edn = new byte[reqsize]; + System.arraycopy(src, 0, srcbuf, 0, src.length); + int reqsize2 = rapidyaml.parseYsToEdn(srcbuf, edn); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + String ret = new String(edn, 0, reqsize-1, StandardCharsets.UTF_8); + return ret; + } + + static ByteBuffer callEdnBuf(byte[] src, ByteBuffer srcbuf) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + srcbuf.position(0); + srcbuf.put(src); + ByteBuffer edn = ByteBuffer.allocateDirect(10000); + int reqsize = rapidyaml.parseYsToEdnBuf(srcbuf, edn); + if(reqsize > edn.capacity()) { + edn = ByteBuffer.allocateDirect(reqsize); + srcbuf.position(0); + srcbuf.put(src); + int reqsize2 = rapidyaml.parseYsToEdnBuf(srcbuf, edn); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + edn.position(reqsize); + return edn; + } + + static int[] callEvt(byte[] src, byte[] srcbuf) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + System.arraycopy(src, 0, srcbuf, 0, src.length); + int[] evt = new int[10000]; + int reqsize = rapidyaml.parseYsToEvt(srcbuf, evt); + if(reqsize > evt.length) { + evt = new int[reqsize]; + System.arraycopy(src, 0, srcbuf, 0, src.length); + int reqsize2 = rapidyaml.parseYsToEvt(srcbuf, evt); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + return evt; + } + int[] ret = new int[reqsize]; + System.arraycopy(evt, 0, ret, 0, reqsize); + return ret; + } + + static IntBuffer callEvtBuf(byte[] src, ByteBuffer srcbuf) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + srcbuf.position(0); + srcbuf.put(src); + IntBuffer evt = Rapidyaml.mkIntBuffer(10000); + int reqsize = rapidyaml.parseYsToEvtBuf(srcbuf, evt); + if(reqsize > evt.capacity()) { + evt = Rapidyaml.mkIntBuffer(reqsize); + srcbuf.position(0); + srcbuf.put(src); + int reqsize2 = rapidyaml.parseYsToEvtBuf(srcbuf, evt); + if(reqsize2 != reqsize) { + throw new RuntimeException("reqsize"); + } + } + evt.position(reqsize); + return evt; + } + + ExpectedEvent mkev(int flags) + { + return new ExpectedEvent(flags); + } + + ExpectedEvent mkev(int flags, int offs, int len, String ref) + { + return new ExpectedEvent(flags, offs, len, ref); + } } + +// the result is an array of integers, but we use this to simplify +// running the tests +class ExpectedEvent +{ + int flags; + int str_start; + int str_len; + String str; + ExpectedEvent(int flags) + { + this.flags = flags; + this.str_start = 0; + this.str_len = 0; + this.str = ""; + } + ExpectedEvent(int flags, int str_start, int str_len, String str) + { + this.flags = flags; + this.str_start = str_start; + this.str_len = str_len; + this.str = str; + } + int required_size() + { + return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; + } + + public static int required_size_(ExpectedEvent[] evts) + { + int sz = 0; + for(int i = 0; i < evts.length; ++i) { + sz += evts[i].required_size(); + } + return sz; + } +}; From 47bd892400f6f80ccbee4b8248824a6aa22f462f Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 23 Feb 2025 23:47:52 +0000 Subject: [PATCH 15/29] rapidyaml: Remove edn parsing --- core/src/yamlscript/parser.clj | 5 +- rapidyaml/native/CMakeLists.txt | 4 - rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 117 --- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 32 - rapidyaml/native/ysparse_edn.cpp | 81 -- rapidyaml/native/ysparse_edn.hpp | 56 -- rapidyaml/native/ysparse_edn_handler.cpp | 80 -- rapidyaml/native/ysparse_edn_handler.hpp | 696 ------------------ rapidyaml/native/ysparse_test.cpp | 253 +------ .../main/java/org/rapidyaml/Rapidyaml.java | 52 -- .../cmp/{CmpEdnEvt.java => CmpEvt.java} | 68 +- .../main/java/org/rapidyaml/cmp/manifest.mf | 2 +- .../src/main/java/org/rapidyaml/cmp/run.sh | 10 +- .../java/org/rapidyaml/RapidyamlTest.java | 210 +----- 14 files changed, 51 insertions(+), 1615 deletions(-) delete mode 100644 rapidyaml/native/ysparse_edn.cpp delete mode 100644 rapidyaml/native/ysparse_edn.hpp delete mode 100644 rapidyaml/native/ysparse_edn_handler.cpp delete mode 100644 rapidyaml/native/ysparse_edn_handler.hpp rename rapidyaml/src/main/java/org/rapidyaml/cmp/{CmpEdnEvt.java => CmpEvt.java} (53%) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 8cfdd70a2..96be8683e 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -233,8 +233,11 @@ "" parse-snakeyaml "snake" parse-snakeyaml "rapid" parse-rapidyaml + "ryml" parse-rapidyaml + ; TODO: + ;"rapid-buf" parse-rapidyaml-buf + ;"ryml-buf" parse-rapidyaml-buf (die "Unknown YS_PARSER value: " parser-name)) - ; parse-rapidyaml-evt parse-snakeyaml)) (comment diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 4f8524e97..aae90d5b5 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -23,10 +23,6 @@ add_library(${libname} # # ysparse files ysparse_common.hpp - ysparse_edn_handler.hpp - ysparse_edn_handler.cpp - ysparse_edn.hpp - ysparse_edn.cpp ysparse_evt_handler.hpp ysparse_evt_handler.cpp ysparse_evt.hpp diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 34730c6cc..02da00d71 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -1,7 +1,5 @@ #include -#include "ysparse_edn.hpp" #include "ysparse_evt.hpp" -#include #ifdef __cplusplus extern "C" { @@ -18,25 +16,12 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) return (jlong)obj; } -JNIEXPORT jlong JNICALL -Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) -{ - Ryml2Edn *obj = ys2edn_init(); - return (jlong)obj; -} - JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) { ys2evt_destroy((Ryml2Evt*)obj); } -JNIEXPORT void JNICALL -Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) -{ - ys2edn_destroy((Ryml2Edn*)obj); -} - JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, @@ -151,108 +136,6 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, } -JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, - jlong obj, jstring jfilename, - jbyteArray src, jint src_len, - jbyteArray dst, jint dst_len) -{ - TIMED_SECTION("jni_ys2edn_parse", (size_type)src_len); - jbyte* src_ = nullptr; - jbyte* dst_ = nullptr; - jboolean src_is_copy = false; - jboolean dst_is_copy = false; - const char *filename = nullptr; - { - TIMED_SECTION("jni_ys2edn_parse/get_jni_data", (size_type)src_len); - { - TIMED_SECTION("jni_ys2edn_parse/GetByteArray(src)"); - src_ = env->GetByteArrayElements(src, &src_is_copy); - } - { - TIMED_SECTION("jni_ys2edn_parse/GetByteArray(dst)"); - dst_ = env->GetByteArrayElements(dst, &dst_is_copy); - } - { - TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); - filename = env->GetStringUTFChars(jfilename, 0); - } - } - int rc = 0; - try - { - rc = ys2edn_parse((Ryml2Edn*)obj, filename, - (char*)src_, src_len, - (char*)dst_, dst_len); - } - catch (YsParseError const& exc) - { - throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); - } - { - TIMED_SECTION("jni_ys2edn_parse/release"); - env->ReleaseByteArrayElements(src, src_, 0); - env->ReleaseByteArrayElements(dst, dst_, 0); - env->ReleaseStringUTFChars(jfilename, filename); - } - return rc; -} - - -JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2edn_1parse_1buf(JNIEnv *env, jobject, - jlong obj, jstring jfilename, - jobject src, jint src_len, - jobject dst, jint dst_len) -{ - TIMED_SECTION("jni_ys2edn_parse", (size_type)src_len); - char* src_ = nullptr; - char* dst_ = nullptr; - const char *filename = nullptr; - { - TIMED_SECTION("jni_ys2edn_parse/get_jni_data", (size_type)src_len); - src_ = (char*)env->GetDirectBufferAddress(src); - dst_ = (char*)env->GetDirectBufferAddress(dst); - filename = env->GetStringUTFChars(jfilename, 0); - if(!src_) - throw_runtime_exception(env, "null pointer: src"); - if(!dst_) - throw_runtime_exception(env, "null pointer: dst"); - } - { - TIMED_SECTION("jni_ys2edn_parse/call_parse", (size_type)src_len); - try - { - return ys2edn_parse((Ryml2Edn*)obj, filename, src_, src_len, dst_, dst_len); - } - catch (YsParseError const& exc) - { - throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); - } - catch (std::exception const& exc) - { - throw_runtime_exception(env, exc.what()); - } - } - return 0; // this is executed even if there is an exception -} - - -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get(JNIEnv *env, jobject, - jlong obj, - jobject dst, jint dst_len) -{ - char* dst_ = nullptr; - { - TIMED_SECTION("jni_ys2evt_retry_get/get_dst"); - dst_ = (char*)env->GetDirectBufferAddress(dst); - printf("edn: aqui 1 %p\n", dst_); - } - int rc = ys2edn_retry_get((Ryml2Edn*)obj, (char*)dst_, dst_len); - return rc; -} - - //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index 74c7a5eb4..d3124b552 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -15,14 +15,6 @@ extern "C" { JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init (JNIEnv *, jobject); -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_init - * Signature: ()J - */ -JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init - (JNIEnv *, jobject); - /* * Class: org_rapidyaml_Rapidyaml * Method: ys2evt_destroy @@ -31,14 +23,6 @@ JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1init JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy (JNIEnv *, jobject, jlong); -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_destroy - * Signature: (J)V - */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy - (JNIEnv *, jobject, jlong); - /* * Class: org_rapidyaml_Rapidyaml * Method: ys2evt_parse @@ -55,22 +39,6 @@ JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_parse - * Signature: (JLjava/lang/String;[BI[BI)I - */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse - (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint); - -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ys2edn_parse_buf - * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/ByteBuffer;I)I - */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse_1buf - (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); - #ifdef __cplusplus } #endif diff --git a/rapidyaml/native/ysparse_edn.cpp b/rapidyaml/native/ysparse_edn.cpp deleted file mode 100644 index 586c3a79e..000000000 --- a/rapidyaml/native/ysparse_edn.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include "ysparse_edn.hpp" - -using namespace ryml; - -#if defined(__cplusplus) -extern "C" { -#endif -// see -// https://stackoverflow.com/questions/230689/best-way-to-throw-exceptions-in-jni-code -// https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method - -namespace { -C4_NORETURN void ys2edn_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) -{ - YsParseError exc; - exc.location = location; - exc.msg.assign(msg, msg_len); - throw exc; -} -} // anon namespace - -RYML_EXPORT Ryml2Edn *ys2edn_init() -{ - TIMED_SECTION("ys2edn_init"); - Callbacks cb = {}; - cb.m_error = &ys2edn_parse_error; - set_callbacks(cb); - Ryml2Edn *ryml2edn = _RYML_CB_ALLOC(get_callbacks(), Ryml2Edn, 1); - _RYML_CB_CHECK(get_callbacks(), ryml2edn != nullptr); - new ((void*)ryml2edn) Ryml2Edn(); - return ryml2edn; -} - -RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn) -{ - TIMED_SECTION("ys2edn_destroy"); - ryml2edn->~Ryml2Edn(); - _RYML_CB_FREE(get_callbacks(), ryml2edn, Ryml2Edn, 1); -} - -RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, - const char *filename, - char *ys, size_type ys_size, - char *edn, size_type edn_size) -{ - TIMED_SECTION("ys2edn_parse", ys_size); - csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; - substr ys_(ys, (size_t)ys_size); - { - TIMED_SECTION("reset + reserve"); - ryml2edn->reset(); - ryml2edn->m_handler.reserve(ys_size > edn_size ? 3 * ys_size : edn_size, 256u); - } - { - TIMED_SECTION("parse_in_place", ys_size); - ryml2edn->m_parser.parse_in_place_ev(filename_, ys_); - } - return ys2edn_retry_get(ryml2edn, edn, edn_size); -} - -RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, - char *edn, size_type edn_size) -{ - TIMED_SECTION("ys2edn_retry_get"); - csubstr result = to_csubstr(ryml2edn->m_sink.result); - size_type required_size = 1 + (size_type)result.len; - if(required_size <= edn_size) - { - memcpy(edn, result.str, (size_t)result.len); - edn[result.len] = '\0'; - } - else if(edn_size > 0) - { - edn[0] = '\0'; - } - return required_size; -} - -#if defined(__cplusplus) -} -#endif diff --git a/rapidyaml/native/ysparse_edn.hpp b/rapidyaml/native/ysparse_edn.hpp deleted file mode 100644 index 3dfe73606..000000000 --- a/rapidyaml/native/ysparse_edn.hpp +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once -#ifndef YSPARSE_EDN_HPP_ -#define YSPARSE_EDN_HPP_ - -#include "ysparse_edn_handler.hpp" -#include "ysparse_common.hpp" - -#if defined(__cplusplus) -extern "C" { -#endif - -struct RYML_EXPORT Ryml2Edn -{ - ys::EventHandlerEdn::EventSink m_sink; - ys::EventHandlerEdn m_handler; - c4::yml::ParseEngine m_parser; - Ryml2Edn() - : m_sink() - , m_handler(&m_sink) - , m_parser(&m_handler) - { - } - void reset() - { - m_sink.reset(); - m_handler.reset(); - } -}; - - -//----------------------------------------------------------------------------- - -/** Initialize the resources */ -RYML_EXPORT Ryml2Edn *ys2edn_init(); - -/** Destroy the resources */ -RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn); - -/** Parse YAML, and return corresponding EDN. Return the number of - * characters needed for edn. The caller must check if the returned - * size is larger than edn_size. If it is, call ys_retry_get() can be - * called afterwards to extract the EDN. */ -RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, - const char *filename, - char *ys, size_type ys_size, - char *edn, size_type edn_size); - -/** Get the edn from the previous call to ys2edn_parse(). */ -RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, - char *edn, size_type edn_size); - -#if defined(__cplusplus) -} -#endif - -#endif /* YSPARSE_EDN_HPP_ */ diff --git a/rapidyaml/native/ysparse_edn_handler.cpp b/rapidyaml/native/ysparse_edn_handler.cpp deleted file mode 100644 index ab295234e..000000000 --- a/rapidyaml/native/ysparse_edn_handler.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include "ysparse_edn_handler.hpp" -#include -#include - - -// instantiate the template -namespace c4 { -namespace yml { -template class ParseEngine; -} // namespace yml -} // namespace c4 - - -namespace ys { -void EventHandlerEdn::EventSink::append_escaped(csubstr val) -{ - #define _c4flush_use_instead(repl, skip) \ - do { \ - this->append(val.range(prev, i)); \ - this->append(repl); \ - prev = i + skip; \ - } \ - while(0) - uint8_t const* C4_RESTRICT s = reinterpret_cast(val.str); - size_t prev = 0; - for(size_t i = 0; i < val.len; ++i) - { - switch(s[i]) - { - case UINT8_C(0x0a): // \n - _c4flush_use_instead("\\n", 1); break; - case UINT8_C(0x5c): // '\\' - _c4flush_use_instead("\\\\", 1); break; - case UINT8_C(0x22): // \" - _c4flush_use_instead("\\\"", 1); break; - case UINT8_C(0x09): // \t - _c4flush_use_instead("\\t", 1); break; - case UINT8_C(0x0d): // \r - _c4flush_use_instead("\\r", 1); break; - case UINT8_C(0x00): // \0 - _c4flush_use_instead("\\0", 1); break; - case UINT8_C(0x0c): // \f (form feed) - _c4flush_use_instead("\\f", 1); break; - case UINT8_C(0x08): // \b (backspace) - _c4flush_use_instead("\\b", 1); break; - case UINT8_C(0x07): // \a (bell) - _c4flush_use_instead("\\a", 1); break; - case UINT8_C(0x0b): // \v (vertical tab) - _c4flush_use_instead("\\v", 1); break; - case UINT8_C(0x1b): // \e (escape) - _c4flush_use_instead("\\e", 1); break; - case UINT8_C(0xc2): - if(i+1 < val.len) - { - const uint8_t np1 = s[i+1]; - if(np1 == UINT8_C(0xa0)) - _c4flush_use_instead("\\_", 2); - else if(np1 == UINT8_C(0x85)) - _c4flush_use_instead("\\N", 2); - } - break; - case UINT8_C(0xe2): - if(i+2 < val.len) - { - if(s[i+1] == UINT8_C(0x80)) - { - if(s[i+2] == UINT8_C(0xa8)) - _c4flush_use_instead("\\L", 3); - else if(s[i+2] == UINT8_C(0xa9)) - _c4flush_use_instead("\\P", 3); - } - } - break; - } - } - // flush the rest - this->append(val.sub(prev)); - #undef _c4flush_use_instead -} -} // namespace ys diff --git a/rapidyaml/native/ysparse_edn_handler.hpp b/rapidyaml/native/ysparse_edn_handler.hpp deleted file mode 100644 index 89c36813f..000000000 --- a/rapidyaml/native/ysparse_edn_handler.hpp +++ /dev/null @@ -1,696 +0,0 @@ -#ifndef _YSPARSE_EDN_HANDLER_HPP_ -#define _YSPARSE_EDN_HANDLER_HPP_ - -#include -#include -#include -#include -#include -#include - -#include - -C4_SUPPRESS_WARNING_GCC_CLANG_PUSH -C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") -C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") - - -namespace ys { - -using c4::csubstr; -using c4::substr; -using c4::to_substr; -using c4::to_csubstr; -using c4::yml::id_type; -using c4::yml::NodeType_e; -using c4::yml::type_bits; -#ifdef RYML_DBG -using c4::_dbg_printf; -#endif - -struct EventHandlerEdnState : public c4::yml::ParserState -{ - c4::yml::NodeData ev_data; -}; - - -struct EventHandlerEdn : public c4::yml::EventHandlerStack -{ - - /** @name types - * @{ */ - - // our internal state must inherit from parser state - using state = EventHandlerEdnState; - - struct EventSink - { - std::string result; - void reset() noexcept { result.clear(); } - void append(csubstr s) noexcept { result.append(s.str, s.len); } - void append(char c) noexcept { result += c; } - void insert(csubstr s, size_t pos) noexcept { result.insert(pos, s.str, s.len); } - void insert(char c, size_t pos) noexcept { result.insert(pos, 1, c); } - csubstr get() const { return csubstr(&result[0], result.size()); } - substr get() { return substr(&result[0], result.size()); } - size_t find_last(csubstr s) const { return result.rfind(s.str, std::string::npos, s.len); } - void append_escaped(csubstr val); - }; - - /** @} */ - -public: - - /** @cond dev */ - EventSink *C4_RESTRICT m_sink; - std::vector m_val_buffers; - char m_key_tag_buf[256]; - char m_val_tag_buf[256]; - std::string m_arena; - bool m_first_doc; - - // undefined at the end - #define _enable_(bits) _enable__() - #define _disable_(bits) _disable__() - #define _has_any_(bits) _has_any__() - /** @endcond */ - -public: - - /** @name construction and resetting - * @{ */ - - EventHandlerEdn(EventSink *sink, c4::yml::Callbacks const& cb) - : EventHandlerStack(cb), m_sink(sink), m_val_buffers(), m_first_doc() - { - reset(); - } - EventHandlerEdn(EventSink *sink) - : EventHandlerEdn(sink, c4::yml::get_callbacks()) - { - } - - void reset() - { - _stack_reset_root(); - m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; - m_val_buffers.resize((size_t)m_stack.size()); - m_arena.clear(); - m_first_doc = true; - } - - void reserve(int edn_size, int arena_size) - { - if(m_val_buffers.empty()) - m_val_buffers.resize((size_t)m_stack.size()); - if(m_sink) - m_sink->result.reserve(edn_size); - for(size_t i = 0; i < m_val_buffers.size(); ++i) - { - int sz = edn_size / (int(1) << (uint32_t)i); - sz = sz >= 128 ? sz : 128; - m_val_buffers[i].result.reserve((size_t)sz); - } - m_arena.reserve(arena_size); - } - - /** @} */ - -public: - - /** @name parse events - * @{ */ - - void start_parse(const char* filename, c4::yml::detail::pfn_relocate_arena relocate_arena, void *relocate_arena_data) - { - this->_stack_start_parse(filename, relocate_arena, relocate_arena_data); - } - - void finish_parse() - { - this->_stack_finish_parse(); - } - - void cancel_parse() - { - while(m_stack.size() > 1) - _pop(); - _buf_flush_(); - } - - /** @} */ - -public: - - /** @name YAML stream events */ - /** @{ */ - - void begin_stream() - { - _send_("(\n"); - } - - void end_stream() - { - _send_(")\n"); - _buf_flush_(); - } - - /** @} */ - -public: - - /** @name YAML document events */ - /** @{ */ - - /** implicit doc start (without ---) */ - void begin_doc() - { - _c4dbgp("begin_doc"); - if(_stack_should_push_on_begin_doc()) - { - _c4dbgp("push!"); - _push(); - _enable_(c4::yml::DOC); - } - m_first_doc = false; - } - /** implicit doc end (without ...) */ - void end_doc() - { - _c4dbgp("end_doc"); - _send_("{:+ \"-DOC\"}\n"); - if(_stack_should_pop_on_end_doc()) - { - _c4dbgp("pop!"); - _pop(); - } - } - - /** explicit doc start, with --- */ - void begin_doc_expl() - { - _c4dbgp("begin_doc_expl"); - if(_stack_should_push_on_begin_doc()) - { - _c4dbgp("push!"); - _push(); - } - if (m_first_doc) - m_first_doc = false; - else - _send_("{:+ \"+DOC\"}\n"); - _enable_(c4::yml::DOC); - } - /** explicit doc end, with ... */ - void end_doc_expl() - { - _c4dbgp("end_doc_expl"); - _send_("{:+ \"-DOC\"}\n"); - if(_stack_should_pop_on_end_doc()) - { - _c4dbgp("pop!"); - _pop(); - } - } - - /** @} */ - -public: - - /** @name YAML map functions */ - /** @{ */ - - void begin_map_key_flow() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - void begin_map_key_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - void begin_map_val_flow() - { - _send_("{:+ \"+MAP\""); - _send_val_props_(); - _send_(", :flow true}\n"); - _mark_parent_with_children_(); - _enable_(c4::yml::MAP|c4::yml::FLOW_SL); - _push(); - } - void begin_map_val_block() - { - _send_("{:+ \"+MAP\""); - _send_val_props_(); - _send_("}\n"); - _mark_parent_with_children_(); - _enable_(c4::yml::MAP|c4::yml::BLOCK); - _push(); - } - - void end_map() - { - _pop(); - _send_("{:+ \"-MAP\"}\n"); - } - - /** @} */ - -public: - - /** @name YAML seq events */ - /** @{ */ - - void begin_seq_key_flow() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - void begin_seq_key_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - void begin_seq_val_flow() - { - _send_("{:+ \"+SEQ\""); - _send_val_props_(); - _send_(", :flow true}\n"); - _mark_parent_with_children_(); - _enable_(c4::yml::SEQ|c4::yml::FLOW_SL); - _push(); - } - void begin_seq_val_block() - { - _send_("{:+ \"+SEQ\""); - _send_val_props_(); - _send_("}\n"); - _mark_parent_with_children_(); - _enable_(c4::yml::SEQ|c4::yml::BLOCK); - _push(); - } - - void end_seq() - { - _pop(); - _send_("{:+ \"-SEQ\"}\n"); - } - - /** @} */ - -public: - - /** @name YAML structure events */ - /** @{ */ - - void add_sibling() - { - _RYML_CB_ASSERT(m_stack.m_callbacks, m_parent); - _buf_flush_to_(m_curr->level, m_parent->level); - m_curr->ev_data = {}; - } - - /** set the previous val as the first key of a new map, with flow style. - * - * See the documentation for @ref doc_event_handlers, which has - * important notes about this event. - */ - void actually_val_is_first_key_of_new_map_flow() - { - // ensure we have a temporary buffer to save the current val - const id_type tmp = m_curr->level + id_type(2); - _buf_ensure_(tmp + id_type(2)); - // save the current val to the temporary buffer - _buf_flush_to_(m_curr->level, tmp); - // create the map. - // this will push a new level, and tmp is one further - begin_map_val_flow(); - _RYML_CB_ASSERT(m_stack.m_callbacks, tmp != m_curr->level); - // now move the saved val as the first key - _buf_flush_to_(tmp, m_curr->level); - } - - void actually_val_is_first_key_of_new_map_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - /** @} */ - -public: - - /** @name YAML scalar events */ - /** @{ */ - - - C4_ALWAYS_INLINE void set_key_scalar_plain_empty() - { - _send_key_scalar_({}, ':'); - _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN|c4::yml::KEYNIL); - } - C4_ALWAYS_INLINE void set_val_scalar_plain_empty() - { - _send_val_scalar_({}, ':'); - _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN|c4::yml::VALNIL); - } - - - C4_ALWAYS_INLINE void set_key_scalar_plain(csubstr scalar) - { - _send_key_scalar_(scalar, '='); - _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN); - } - C4_ALWAYS_INLINE void set_val_scalar_plain(csubstr scalar) - { - _send_val_scalar_(scalar, '='); - _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN); - } - - - C4_ALWAYS_INLINE void set_key_scalar_dquoted(csubstr scalar) - { - _send_key_scalar_(scalar, '$'); - _enable_(c4::yml::KEY|c4::yml::KEY_DQUO); - } - C4_ALWAYS_INLINE void set_val_scalar_dquoted(csubstr scalar) - { - _send_val_scalar_(scalar, '$'); - _enable_(c4::yml::VAL|c4::yml::VAL_DQUO); - } - - - C4_ALWAYS_INLINE void set_key_scalar_squoted(csubstr scalar) - { - _send_key_scalar_(scalar, '\''); - _enable_(c4::yml::KEY|c4::yml::KEY_SQUO); - } - C4_ALWAYS_INLINE void set_val_scalar_squoted(csubstr scalar) - { - _send_val_scalar_(scalar, '\''); - _enable_(c4::yml::VAL|c4::yml::VAL_SQUO); - } - - - C4_ALWAYS_INLINE void set_key_scalar_literal(csubstr scalar) - { - _send_key_scalar_(scalar, '|'); - _enable_(c4::yml::KEY|c4::yml::KEY_LITERAL); - } - C4_ALWAYS_INLINE void set_val_scalar_literal(csubstr scalar) - { - _send_val_scalar_(scalar, '|'); - _enable_(c4::yml::VAL|c4::yml::VAL_LITERAL); - } - - - C4_ALWAYS_INLINE void set_key_scalar_folded(csubstr scalar) - { - _send_key_scalar_(scalar, '>'); - _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); - } - C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) - { - _send_val_scalar_(scalar, '>'); - _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); - } - - - C4_ALWAYS_INLINE void mark_key_scalar_unfiltered() - { - _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); - } - C4_ALWAYS_INLINE void mark_val_scalar_unfiltered() - { - _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); - } - - /** @} */ - -public: - - /** @name YAML anchor/reference events */ - /** @{ */ - - void set_key_anchor(csubstr anchor) - { - _enable_(c4::yml::KEYANCH); - m_curr->ev_data.m_key.anchor = anchor; - } - void set_val_anchor(csubstr anchor) - { - _enable_(c4::yml::VALANCH); - m_curr->ev_data.m_val.anchor = anchor; - } - - void set_key_ref(csubstr ref) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); - _enable_(c4::yml::KEY|c4::yml::KEYREF); - _send_("{:+ \"=ALI\" :* \""); - _send_(ref.sub(1)); - _send_("\"}\n"); - } - void set_val_ref(csubstr ref) - { - _enable_(c4::yml::VAL|c4::yml::VALREF); - _send_("{:+ \"=ALI\" :* \""); - _send_(ref.sub(1)); - _send_("\"}\n"); - } - - /** @} */ - -public: - - /** @name YAML tag events */ - /** @{ */ - - void set_key_tag(csubstr tag) - { - _enable_(c4::yml::KEYTAG); - m_curr->ev_data.m_key.tag = _transform_directive(tag, m_key_tag_buf); - } - void set_val_tag(csubstr tag) - { - _enable_(c4::yml::VALTAG); - m_curr->ev_data.m_val.tag = _transform_directive(tag, m_val_tag_buf); - } - - /** @} */ - -public: - - /** @name YAML directive events */ - /** @{ */ - - void add_directive(csubstr directive) - { - _RYML_CB_ERR(m_stack.m_callbacks, "tag directives not supported"); - } - - /** @} */ - -public: - - /** @name YAML arena events */ - /** @{ */ - - substr alloc_arena(size_t len) - { - const size_t sz = m_arena.size(); - csubstr prev = to_csubstr(m_arena); - m_arena.resize(sz + len); - substr out = to_substr(m_arena).sub(sz); - substr curr = to_substr(m_arena); - if(curr.str != prev.str) - _stack_relocate_to_new_arena(prev, curr); - return out; - } - - substr alloc_arena(size_t len, substr *relocated) - { - csubstr prev = to_csubstr(m_arena); - if(!prev.is_super(*relocated)) - return alloc_arena(len); - substr out = alloc_arena(len); - substr curr = to_substr(m_arena); - if(curr.str != prev.str) - *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); - return out; - } - - /** @} */ - -public: - - /** push a new parent, add a child to the new parent, and set the - * child as the current node */ - void _push() - { - _stack_push(); - _buf_ensure_(m_stack.size() + id_type(1)); - _buf_().reset(); - m_curr->ev_data = {}; - } - - /** end the current scope */ - void _pop() - { - _buf_flush_to_(m_curr->level, m_parent->level); - _stack_pop(); - } - - template C4_ALWAYS_INLINE void _enable__() noexcept - { - m_curr->ev_data.m_type.type = static_cast(m_curr->ev_data.m_type.type | bits); - } - template C4_ALWAYS_INLINE void _disable__() noexcept - { - m_curr->ev_data.m_type.type = static_cast(m_curr->ev_data.m_type.type & (~bits)); - } - template C4_ALWAYS_INLINE bool _has_any__() const noexcept - { - return (m_curr->ev_data.m_type.type & bits) != 0; - } - - void _mark_parent_with_children_() - { - if(m_parent) - m_parent->has_children = true; - } - - EventSink& _buf_() noexcept - { - _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)m_curr->level < m_val_buffers.size()); - return m_val_buffers[(size_t)m_curr->level]; - } - - EventSink& _buf_(id_type level) noexcept - { - _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)level < m_val_buffers.size()); - return m_val_buffers[(size_t)level]; - } - - EventSink const& _buf_(id_type level) const noexcept - { - _RYML_CB_ASSERT(m_stack.m_callbacks, (size_t)level < m_val_buffers.size()); - return m_val_buffers[(size_t)level]; - } - - static void _buf_flush_to_(EventSink &C4_RESTRICT src, EventSink &C4_RESTRICT dst) noexcept - { - dst.append(src.get()); - src.reset(); - } - - void _buf_flush_to_(id_type level_src, id_type level_dst) noexcept - { - auto &src = _buf_(level_src); - auto &dst = _buf_(level_dst); - _buf_flush_to_(src, dst); - } - - void _buf_flush_() noexcept - { - _buf_flush_to_(_buf_(), *m_sink); - } - - void _buf_ensure_(id_type size_needed) noexcept - { - if((size_t)size_needed > m_val_buffers.size()) - m_val_buffers.resize((size_t)size_needed); - } - - C4_ALWAYS_INLINE void _send_(csubstr s) noexcept { _buf_().append(s); } - C4_ALWAYS_INLINE void _send_(char c) noexcept { _buf_().append(c); } - - void _send_key_scalar_(csubstr scalar, char scalar_type_code) - { - _send_("{:+ \"=VAL\""); - _send_key_props_(); - _send_(", :"); - _send_(scalar_type_code); - _send_(" \""); - _buf_().append_escaped(scalar); - _send_("\"}\n"); - } - void _send_val_scalar_(csubstr scalar, char scalar_type_code) - { - _send_("{:+ \"=VAL\""); - _send_val_props_(); - _send_(", :"); - _send_(scalar_type_code); - _send_(" \""); - _buf_().append_escaped(scalar); - _send_("\"}\n"); - } - - void _send_key_props_() - { - if(_has_any_(c4::yml::KEYANCH|c4::yml::KEYREF)) - { - _send_(", :& \""); - _send_(m_curr->ev_data.m_key.anchor); - _send_('\"'); - } - if(_has_any_(c4::yml::KEYTAG)) - { - _send_(", :! \""); - _send_tag_(m_curr->ev_data.m_key.tag); - _send_('\"'); - } - m_curr->ev_data.m_key = {}; - _disable_(c4::yml::KEYANCH|c4::yml::KEYREF|c4::yml::KEYTAG); - } - void _send_val_props_() - { - if(_has_any_(c4::yml::VALANCH|c4::yml::VALREF)) - { - _send_(", :& \""); - _send_(m_curr->ev_data.m_val.anchor); - _send_('\"'); - } - if(m_curr->ev_data.m_type.type & c4::yml::VALTAG) - { - _send_(", :! \""); - _send_tag_(m_curr->ev_data.m_val.tag); - _send_('\"'); - } - m_curr->ev_data.m_val = {}; - _disable_(c4::yml::VALANCH|c4::yml::VALREF|c4::yml::VALTAG); - } - void _send_tag_(csubstr tag) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, !tag.empty()); - if(tag.begins_with('!')) - tag = tag.sub(1); - _send_(tag); - } - - csubstr _transform_directive(csubstr tag, substr output) - { - if(tag.begins_with('!')) - { - if(c4::yml::is_custom_tag(tag)) - { - _RYML_CB_ERR_(m_stack.m_callbacks, "tag not found", m_curr->pos); - } - } - csubstr result = c4::yml::normalize_tag_long(tag, output); - if(result.begins_with('<') && result.ends_with('>')) - result = result.offs(1, 1); - _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); - _RYML_CB_CHECK(m_stack.m_callbacks, result.str); - return result; - } -#undef _enable_ -#undef _disable_ -#undef _has_any_ - -}; - -} // namespace ys - -C4_SUPPRESS_WARNING_GCC_POP - -#endif /* _YSPARSE_EDN_HANDLER_HPP_ */ diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 5688d1ebf..644668b26 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -1,6 +1,6 @@ -#include #include #include +#include using c4::csubstr; using c4::substr; @@ -43,12 +43,6 @@ c4::EnumSymbols const esyms() //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- -struct Ys2EdnScoped -{ - Ryml2Edn *ryml2edn; - Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} - ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } -}; struct Ys2EvtScoped { Ryml2Evt *ryml2evt; @@ -103,7 +97,6 @@ size_t expected_size(std::vector const& evt) struct TestCase { csubstr ys; - csubstr edn; std::vector evt; public: @@ -134,15 +127,9 @@ struct TestCase } \ } while(0) - TestResult test(Ryml2Edn *ryml2edn, Ryml2Evt *ryml2evt) const + TestResult test(Ryml2Evt *ryml2evt) const { TestResult tr = {}; - _runtest(test_edn_large_enough, ); - _runtest(test_edn_too_small, ); - _runtest(test_edn_nullptr, ); - _runtest(test_edn_large_enough_reuse, ryml2edn); - _runtest(test_edn_too_small_reuse, ryml2edn); - _runtest(test_edn_nullptr_reuse, ryml2edn); _runtest(test_evt_large_enough, ); _runtest(test_evt_too_small, ); _runtest(test_evt_nullptr, ); @@ -153,22 +140,6 @@ struct TestCase } // happy path: large-enough destination string - TestResult test_edn_large_enough_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output; - output.resize(2 * edn.len); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - &output[0], (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } TestResult test_evt_large_enough_reuse(Ryml2Evt *ryml2evt) const { if(evt.empty()) return {}; @@ -186,11 +157,6 @@ struct TestCase CHECK(testeq(output, input)); return tr; } - TestResult test_edn_large_enough() const - { - Ys2EdnScoped lib; - return test_edn_large_enough_reuse(lib.ryml2edn); - } TestResult test_evt_large_enough() const { Ys2EvtScoped lib; @@ -198,25 +164,6 @@ struct TestCase } // less-happy path: destination string not large enough - TestResult test_edn_too_small_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output = "?"; - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - CHECK(output != "?"); - output.resize(reqsize); - size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); - CHECK(getsize == reqsize); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } TestResult test_evt_too_small_reuse(Ryml2Evt *ryml2evt) const { TestResult tr = {}; @@ -240,11 +187,6 @@ struct TestCase CHECK(testeq(output, input)); return tr; } - TestResult test_edn_too_small() const - { - Ys2EdnScoped lib; - return test_edn_too_small_reuse(lib.ryml2edn); - } TestResult test_evt_too_small() const { Ys2EvtScoped lib; @@ -252,18 +194,6 @@ struct TestCase } // safe calling with nullptr - TestResult test_edn_nullptr_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - nullptr, 0); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - return tr; - } TestResult test_evt_nullptr_reuse(Ryml2Evt *ryml2evt) const { TestResult tr = {}; @@ -286,11 +216,6 @@ struct TestCase CHECK(testeq(output, input)); return tr; } - TestResult test_edn_nullptr() const - { - Ys2EdnScoped lib; - return test_edn_nullptr_reuse(lib.ryml2edn); - } TestResult test_evt_nullptr() const { Ys2EvtScoped lib; @@ -299,21 +224,6 @@ struct TestCase public: - bool testeq(csubstr actual) const - { - const bool status = (actual == edn); - if(!status) - printf("------\n" - "FAIL:\n" - "input:[%zu]~~~%.*s~~~\n" - "expected:[%zu]~~~%.*s~~~\n" - "actual:[%zu]~~~%.*s~~~\n", - ys.len, (int)ys.len, ys.str, - edn.len, (int)edn.len, edn.str, - actual.len, (int)actual.len, actual.str); - return status; - } - bool testeq(std::vector const& actual, csubstr parsed_source) const { int status = true; @@ -392,21 +302,13 @@ struct TestCase namespace { // make the declarations shorter -#define tc(ys, edn, ...) {ys, edn, std::vector(__VA_ARGS__)} +#define tc(ys, ...) {ys, std::vector(__VA_ARGS__)} #define e(...) EvtWithScalar{__VA_ARGS__} using namespace evt; inline constexpr bool needs_filter = true; const TestCase test_cases[] = { // case ------------------------------------------------- tc("a: 1", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "1"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -419,14 +321,6 @@ const TestCase test_cases[] = { }), // case ------------------------------------------------- tc("say: 2 + 2", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "say"} -{:+ "=VAL", := "2 + 2"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -439,14 +333,6 @@ const TestCase test_cases[] = { }), // case ------------------------------------------------- tc("𝄞: ✅", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "𝄞"} -{:+ "=VAL", := "✅"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -459,15 +345,6 @@ const TestCase test_cases[] = { }), // case ------------------------------------------------- tc("[a, b, c]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "=VAL", := "c"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -481,16 +358,6 @@ const TestCase test_cases[] = { }), // case ------------------------------ tc("[a: b]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "-MAP"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -519,42 +386,6 @@ foo: ! - &anchor-1 !tag-1 foobar --- another: doc -)", - R"(( -{:+ "+MAP", :! "yamlscript/v0"} -{:+ "=VAL", := "foo"} -{:+ "+SEQ", :! ""} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-MAP"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-SEQ"} -{:+ "=VAL", := "foo"} -{:+ "=VAL", :' "foo"} -{:+ "=VAL", :$ "foo"} -{:+ "=VAL", :| "foo\n"} -{:+ "=VAL", :> "foo\n"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "1"} -{:+ "=VAL", := "2"} -{:+ "=VAL", := "true"} -{:+ "=VAL", := "false"} -{:+ "=VAL", := "null"} -{:+ "-SEQ"} -{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} -{:+ "-SEQ"} -{:+ "-MAP"} -{:+ "-DOC"} -{:+ "+DOC"} -{:+ "+MAP"} -{:+ "=VAL", := "another"} -{:+ "=VAL", := "doc"} -{:+ "-MAP"} -{:+ "-DOC"} -) )", { e(BSTR), @@ -613,22 +444,6 @@ fold: > U V W -)", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "plain"} -{:+ "=VAL", := "well a b c"} -{:+ "=VAL", := "squo"} -{:+ "=VAL", :' "single'quote"} -{:+ "=VAL", := "dquo"} -{:+ "=VAL", :$ "x\t\ny"} -{:+ "=VAL", := "lit"} -{:+ "=VAL", :| "X\nY\nZ\n"} -{:+ "=VAL", := "fold"} -{:+ "=VAL", :> "U V W\n"} -{:+ "-MAP"} -{:+ "-DOC"} -) )", { e(BSTR), @@ -650,14 +465,6 @@ fold: > }), // case ------------------------------------------------- tc("- !!seq []", - R"(( -{:+ "+SEQ"} -{:+ "+SEQ", :! "tag:yaml.org,2002:seq", :flow true} -{:+ "-SEQ"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)", { e(BSTR), e(BDOC), @@ -676,20 +483,6 @@ fold: > Q: $(orig-prompt:trim) A ($api-model): $(answer:trim) -)_", - R"_(( -{:+ "+MAP"} -{:+ "=VAL", := "defn run(prompt session=nil)"} -{:+ "+MAP"} -{:+ "=VAL", := "when session"} -{:+ "+MAP"} -{:+ "=VAL", := "write session _ :append true"} -{:+ "=VAL", :| "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "-DOC"} -) )_", { e(BSTR), @@ -734,43 +527,6 @@ defn run(prompt session=nil): A ($api-model): $(answer:trim) -)_", - R"_(( -{:+ "+MAP"} -{:+ "=VAL", := "defn run(prompt session=nil)"} -{:+ "+MAP"} -{:+ "=VAL", := "session-text ="} -{:+ "+MAP"} -{:+ "=VAL", := "when session && session:fs-e"} -{:+ "=VAL", :: ""} -{:+ "-MAP"} -{:+ "=VAL", := "answer ="} -{:+ "+MAP"} -{:+ "=VAL", := "cond"} -{:+ "+MAP"} -{:+ "=VAL", := "api-model =~ /^dall-e/"} -{:+ "=VAL", := "openai-image(prompt).data.0.url"} -{:+ "=VAL", := "api-model.in?(anthropic-models)"} -{:+ "=VAL", := "anthropic(prompt):anthropic-message:format"} -{:+ "=VAL", := "api-model.in?(groq-models)"} -{:+ "=VAL", := "groq(prompt).choices.0.message.content:format"} -{:+ "=VAL", := "api-model.in?(openai-models)"} -{:+ "=VAL", := "openai-chat(prompt).choices.0.message.content:format"} -{:+ "=VAL", := "else"} -{:+ "=VAL", := "die()"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "=VAL", := "say"} -{:+ "=VAL", := "answer"} -{:+ "=VAL", := "when session"} -{:+ "+MAP"} -{:+ "=VAL", := "write session _ :append true"} -{:+ "=VAL", :| "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "-MAP"} -{:+ "-DOC"} -) )_", { e(BSTR), @@ -816,7 +572,6 @@ defn run(prompt session=nil): int main() { - Ys2EdnScoped ys2edn; Ys2EvtScoped ys2evt; TestResult total = {}; size_t failed_cases = {}; @@ -826,7 +581,7 @@ int main() printf("-----------------------------------------\n" "case %zu/%zu ...\n" "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); - const TestResult tr = test_cases[i].test(ys2edn.ryml2edn, ys2evt.ryml2evt); + const TestResult tr = test_cases[i].test(ys2evt.ryml2evt); total.add(tr); failed_cases += (!tr); printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index f77b519d7..a8803482b 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -14,32 +14,20 @@ public class Rapidyaml public static String RAPIDYAML_VERSION = "0.8.0"; private native long ys2evt_init(); - private native long ys2edn_init(); - private native void ys2evt_destroy(long ryml2evt); - private native void ys2edn_destroy(long ryml2edn); - private native int ys2evt_parse(long ryml2evt, String filename, byte[] ys, int ys_length, int[] evt, int evt_length); private native int ys2evt_parse_buf(long ryml2evt, String filename, ByteBuffer ys, int ys_length, IntBuffer evt, int evt_length); - private native int ys2edn_parse(long ryml2edn, String filename, - byte[] ys, int ys_length, - byte[] edn, int edn_length); - private native int ys2edn_parse_buf(long ryml2edn, String filename, - ByteBuffer ys, int ys_length, - ByteBuffer edn, int edn_length); - private final long ryml2edn; private final long ryml2evt; public Rapidyaml() { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); - this.ryml2edn = this.ys2edn_init(); this.ryml2evt = this.ys2evt_init(); // TODO: receive this argument as ctor parameter timingEnabled(System.getenv("YS_RYML_TIMER") != null); @@ -52,7 +40,6 @@ public Rapidyaml() protected void finalize() throws Throwable { try { - this.ys2edn_destroy(this.ryml2edn); this.ys2evt_destroy(this.ryml2evt); } finally { @@ -111,45 +98,6 @@ public static IntBuffer mkIntBuffer(int numInts) } - //------------------------ - // EDN - //------------------------ - - public int parseYsToEdn(byte[] src, byte[] edn) throws Exception - { - return parseYsToEdn("yamlscript", src, edn); - } - - public int parseYsToEdnBuf(ByteBuffer src, ByteBuffer edn) throws Exception - { - return parseYsToEdnBuf("yamlscript", src, edn); - } - - public int parseYsToEdn(String filename, byte[] src, byte[] edn) throws Exception - { - long t = timingStart("ys2edn"); - int ret = ys2edn_parse(this.ryml2edn, filename, src, src.length, edn, edn.length); - timingStop("ys2edn", t, src.length); - return ret; - } - - public int parseYsToEdnBuf(String filename, ByteBuffer src, ByteBuffer edn) throws Exception - { - if(!src.isDirect()) - throw new RuntimeException("src must be direct"); - if(!edn.isDirect()) - throw new RuntimeException("edn must be direct"); - long t = timingStart("ys2ednBuf"); - edn.position(edn.capacity()); - int reqsize = ys2edn_parse_buf(this.ryml2edn, filename, src, src.position(), edn, edn.capacity()); - if(reqsize <= edn.capacity()) { - edn.position(reqsize); - } - timingStop("ys2ednBuf", t, src.position()); - return reqsize; - } - - //------------------------ // TIME //------------------------ diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java similarity index 53% rename from rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java rename to rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java index cd86ae189..19b6fab7c 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEdnEvt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java @@ -9,24 +9,24 @@ import java.nio.IntBuffer; // https://stackoverflow.com/questions/804466/how-do-i-create-executable-java-program -public class CmpEdnEvt +public class CmpEvt { public static void main(String[] args) throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); rapidyaml.timingEnabled(true); - compareEdnEvt(rapidyaml, "./yamllm.ys"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); - compareEdnEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); + compareEvt(rapidyaml, "./yamllm.ys"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); + compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); } - public static void compareEdnEvt(Rapidyaml rapidyaml, String path) throws Exception + public static void compareEvt(Rapidyaml rapidyaml, String path) throws Exception { String ys_ = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); byte[] ys = ys_.getBytes(StandardCharsets.UTF_8); @@ -37,15 +37,7 @@ public static void compareEdnEvt(Rapidyaml rapidyaml, String path) throws Except System.out.printf("%s\n", path); System.out.printf(" ys.length=%d\n", ys.length); // - long t = timingStart("edn"); - String ednStr = callEdn(rapidyaml, ys, ysarr); - timingStop("edn", t, ys.length); - // - t = timingStart("ednBuf"); - ByteBuffer edn = callEdnBuf(rapidyaml, ys, ysbuf); - timingStop("ednBuf", t, ys.length); - // - t = timingStart("evt"); + long t = timingStart("evt"); int[] evtarr = callEvt(rapidyaml, ys, ysarr); timingStop("evt", t, ys.length); // @@ -54,42 +46,6 @@ public static void compareEdnEvt(Rapidyaml rapidyaml, String path) throws Except timingStop("evtBuf", t, ys.length); } - static String callEdn(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception - { - System.arraycopy(src, 0, srcbuf, 0, src.length); - byte[] edn = new byte[10000000]; - int reqsize = rapidyaml.parseYsToEdn(srcbuf, edn); - if(reqsize > edn.length) { - edn = new byte[reqsize]; - System.arraycopy(src, 0, srcbuf, 0, src.length); - int reqsize2 = rapidyaml.parseYsToEdn(srcbuf, edn); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - String ret = new String(edn, 0, reqsize-1, StandardCharsets.UTF_8); - return ret; - } - - static ByteBuffer callEdnBuf(Rapidyaml rapidyaml, byte[] src, ByteBuffer srcbuf) throws Exception - { - srcbuf.position(0); - srcbuf.put(src); - ByteBuffer edn = ByteBuffer.allocateDirect(10000000); - int reqsize = rapidyaml.parseYsToEdnBuf(srcbuf, edn); - if(reqsize > edn.capacity()) { - edn = ByteBuffer.allocateDirect(reqsize); - srcbuf.position(0); - srcbuf.put(src); - int reqsize2 = rapidyaml.parseYsToEdnBuf(srcbuf, edn); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - edn.position(reqsize); - return edn; - } - static int[] callEvt(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception { System.arraycopy(src, 0, srcbuf, 0, src.length); diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf index 97704e895..aa44b6579 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf @@ -1 +1 @@ -Main-class: cmp.CmpEdnEvt +Main-class: cmp.CmpEvt diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh index da951ab24..4f655cfe2 100755 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh +++ b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh @@ -11,10 +11,12 @@ make -C $rymldir test RAPIDYAML_TIMED=1 cd $thisdir -wget https://raw.githubusercontent.com/yaml/yamllm/refs/heads/main/bin/yamllm.ys +if [ ! -f yamllm.ys ] ; then + wget https://raw.githubusercontent.com/yaml/yamllm/refs/heads/main/bin/yamllm.ys +fi ls -lFhp jd=${jd:-/usr/lib/jvm/java-23-openjdk/bin} $jd/javac -d . ../*.java -$jd/javac -d . -cp . CmpEdnEvt.java -$jd/jar -cmf manifest.mf CmpEdnEvt.jar cmp org -$jd/java -jar -Djava.library.path=$nativedir CmpEdnEvt.jar +$jd/javac -d . -cp . CmpEvt.java +$jd/jar -cmf manifest.mf CmpEvt.jar cmp org +$jd/java -jar -Djava.library.path=$nativedir CmpEvt.jar diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index 174dde7aa..7a2af04d7 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -39,15 +39,6 @@ public static Test suite() public void testPlainMap() { String ys = "a: 1"; - testEdn_(ys, - "(\n" + - "{:+ \"+MAP\"}\n" + - "{:+ \"=VAL\", := \"a\"}\n" + - "{:+ \"=VAL\", := \"1\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -64,15 +55,6 @@ public void testPlainMap() public void testUtf8() { String ys = "𝄞: ✅"; - testEdn_(ys, - "(\n" + - "{:+ \"+MAP\"}\n" + - "{:+ \"=VAL\", := \"𝄞\"}\n" + - "{:+ \"=VAL\", := \"✅\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -89,14 +71,6 @@ public void testUtf8() public void testTaggedInt() { String ys = "- !!int 42"; - testEdn_(ys, - "(\n" + - "{:+ \"+SEQ\"}\n" + - "{:+ \"=VAL\", :! \"tag:yaml.org,2002:int\", := \"42\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -113,15 +87,6 @@ public void testTaggedInt() public void testTaggedSeq() { String ys = "- !!seq []"; - testEdn_(ys, - "(\n" + - "{:+ \"+SEQ\"}\n" + - "{:+ \"+SEQ\", :! \"tag:yaml.org,2002:seq\", :flow true}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -155,43 +120,6 @@ public void testLargeCase() "- &anchor-1 !tag-1 foobar\n" + "---\n" + "another: doc\n"; - testEdn_(ys, - "(\n" + - "{:+ \"+MAP\", :! \"yamlscript/v0\"}\n" + - "{:+ \"=VAL\", := \"foo\"}\n" + - "{:+ \"+SEQ\", :! \"\"}\n" + - "{:+ \"+MAP\", :flow true}\n" + - "{:+ \"=VAL\", := \"x\"}\n" + - "{:+ \"=VAL\", := \"y\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"+SEQ\", :flow true}\n" + - "{:+ \"=VAL\", := \"x\"}\n" + - "{:+ \"=VAL\", := \"y\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"=VAL\", := \"foo\"}\n" + - "{:+ \"=VAL\", :' \"foo\"}\n" + - "{:+ \"=VAL\", :$ \"foo\"}\n" + - "{:+ \"=VAL\", :| \"foo\\nliteral\\n\"}\n" + - "{:+ \"=VAL\", :> \"foo folded\\n\"}\n" + - "{:+ \"+SEQ\", :flow true}\n" + - "{:+ \"=VAL\", := \"1\"}\n" + - "{:+ \"=VAL\", := \"2\"}\n" + - "{:+ \"=VAL\", := \"true\"}\n" + - "{:+ \"=VAL\", := \"false\"}\n" + - "{:+ \"=VAL\", := \"null\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"=VAL\", :& \"anchor-1\", :! \"tag-1\", := \"foobar\"}\n" + - "{:+ \"-SEQ\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - "{:+ \"+DOC\"}\n" + - "{:+ \"+MAP\"}\n" + - "{:+ \"=VAL\", := \"another\"}\n" + - "{:+ \"=VAL\", := \"doc\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n" - ); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC|Evt.EXPL), @@ -254,22 +182,6 @@ public void testFilterCase() " U\n" + " V\n" + " W\n"; - testEdn_(ys, - "(\n" + - "{:+ \"+MAP\"}\n" + - "{:+ \"=VAL\", := \"plain\"}\n" + - "{:+ \"=VAL\", := \"well a b c\"}\n" + - "{:+ \"=VAL\", := \"squo\"}\n" + - "{:+ \"=VAL\", :' \"single'quote\"}\n" + - "{:+ \"=VAL\", := \"dquo\"}\n" + - "{:+ \"=VAL\", :$ \"x\\t\\ny\"}\n" + - "{:+ \"=VAL\", := \"lit\"}\n" + - "{:+ \"=VAL\", :| \"X\\nY\\nZ\\n\"}\n" + - "{:+ \"=VAL\", := \"fold\"}\n" + - "{:+ \"=VAL\", :> \"U V W\\n\"}\n" + - "{:+ \"-MAP\"}\n" + - "{:+ \"-DOC\"}\n" + - ")\n"); ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), @@ -295,9 +207,11 @@ public void testFailure() throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); String ys = ": : : :"; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + byte[] srcbuf = new byte[src.length]; boolean gotit = false; try { - callEdn(ys); + callEvt(src, srcbuf); } catch(YamlParseErrorException e) { gotit = true; @@ -313,46 +227,38 @@ public void testFailure() throws Exception catch(Exception e) { fail("wrong exception type"); } - //catch(Throwable e) { - // throw e; - // //fail("wrong exception type"); - //} assertTrue(gotit); } - - private void testEdn_(String ys, String expected) + public void testFailureBuf() throws Exception { - String actual; - try { - actual = callEdn(ys); - } - catch (Exception e) { - fail("parse error:\n" + e.getMessage()); - actual = ""; - } - try { - cmpEdn_(actual, expected); - } - catch (Exception e) { - System.err.printf("error: edn (no buf)"); - } - //------ + Rapidyaml rapidyaml = new Rapidyaml(); + String ys = ": : : :"; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); + bbuf.put(src); + boolean gotit = false; try { - actual = callEdnBuf(ys); + callEvtBuf(src, bbuf); } - catch (Exception e) { - fail("parse error:\n" + e.getMessage()); - actual = ""; + catch(YamlParseErrorException e) { + gotit = true; + assertEquals(2, e.offset); + assertEquals(1, e.line); + assertEquals(3, e.column); + assertTrue(e.getMessage() != null); + assertFalse(e.getMessage().isEmpty()); } - try { - cmpEdn_(actual, expected); + catch(RuntimeException e) { + fail("wrong exception type"); } - catch (Exception e) { - System.err.printf("error: ednbuf"); + catch(Exception e) { + fail("wrong exception type"); } + assertTrue(gotit); } + private void testEvt_(String ys, ExpectedEvent[] expected) { byte[] src = ys.getBytes(StandardCharsets.UTF_8); @@ -394,21 +300,6 @@ private void testEvt_(String ys, ExpectedEvent[] expected) } } - private void cmpEdn_(String actual, String expected) throws Exception - { - try { - assertEquals(expected, actual); - assertEquals(expected.length(), actual.length()); - } - catch (Exception e) { - System.err.println("expected:"); - System.err.println(expected); - System.err.println("actual"); - System.err.println(actual); - throw e; - } - } - boolean dbglog = true; private void cmpEvt_(String ys, byte[] src, int[] actual, ExpectedEvent[] expected) { @@ -495,59 +386,6 @@ public static int[] buf2arr(IntBuffer evt) return ret; } - static String callEdn(String orig) throws Exception - { - byte[] src = orig.getBytes(StandardCharsets.UTF_8); - byte[] srcbuf = new byte[src.length]; - return callEdn(src, srcbuf); - } - - static String callEdnBuf(String orig) throws Exception - { - byte[] src = orig.getBytes(StandardCharsets.UTF_8); - ByteBuffer srcbuf = ByteBuffer.allocateDirect(src.length); - ByteBuffer edn = callEdnBuf(src, srcbuf); - return buf2str(edn); - } - - static String callEdn(byte[] src, byte[] srcbuf) throws Exception - { - Rapidyaml rapidyaml = new Rapidyaml(); - System.arraycopy(src, 0, srcbuf, 0, src.length); - byte[] edn = new byte[10 * src.length]; - int reqsize = rapidyaml.parseYsToEdn(srcbuf, edn); - if(reqsize > edn.length) { - edn = new byte[reqsize]; - System.arraycopy(src, 0, srcbuf, 0, src.length); - int reqsize2 = rapidyaml.parseYsToEdn(srcbuf, edn); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - String ret = new String(edn, 0, reqsize-1, StandardCharsets.UTF_8); - return ret; - } - - static ByteBuffer callEdnBuf(byte[] src, ByteBuffer srcbuf) throws Exception - { - Rapidyaml rapidyaml = new Rapidyaml(); - srcbuf.position(0); - srcbuf.put(src); - ByteBuffer edn = ByteBuffer.allocateDirect(10000); - int reqsize = rapidyaml.parseYsToEdnBuf(srcbuf, edn); - if(reqsize > edn.capacity()) { - edn = ByteBuffer.allocateDirect(reqsize); - srcbuf.position(0); - srcbuf.put(src); - int reqsize2 = rapidyaml.parseYsToEdnBuf(srcbuf, edn); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - edn.position(reqsize); - return edn; - } - static int[] callEvt(byte[] src, byte[] srcbuf) throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); From 328e3997a9b16ff36d8ed607c5903247a7d959d1 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 24 Feb 2025 00:14:41 +0000 Subject: [PATCH 16/29] rapidyaml: Add run-time timing toggle --- rapidyaml/native/CMakeLists.txt | 5 +++ rapidyaml/native/Makefile | 10 +++-- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 18 ++++++++ rapidyaml/native/org_rapidyaml_Rapidyaml.h | 8 ++++ rapidyaml/native/ysparse_common.hpp | 42 ++++++++++++++----- rapidyaml/native/ysparse_test.cpp | 8 +++- .../main/java/org/rapidyaml/Rapidyaml.java | 5 ++- 7 files changed, 80 insertions(+), 16 deletions(-) diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index aae90d5b5..968d7cb9d 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -64,3 +64,8 @@ add_custom_target(${libname}-test-run COMMAND $ COMMENT "running C++ tests" ) +add_custom_target(${libname}-test-run-timing + DEPENDS ${libname}-test + COMMAND $ --timing + COMMENT "running C++ tests, with timing" +) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index ec0037293..e656be03e 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -63,17 +63,21 @@ cfg-static: rapidyaml $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF test-static: build-static $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run +test-static-timing: build-static + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run-timing $(RAPIDYAML_LIB): cfg-static $(RAPIDYAML_DEPS) - $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --target rapidyaml --parallel --verbose + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --parallel --target rapidyaml cp -fv $(BDIR)-static/*.a $@ build-shared: $(RAPIDYAML_SO) cfg-shared: rapidyaml $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON test-shared: build-shared - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --target rapidyaml-test-run --verbose + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run +test-shared-timing: build-shared + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run-timing $(RAPIDYAML_SO): cfg-shared $(RAPIDYAML_DEPS) - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --target rapidyaml --parallel --verbose + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --parallel --target rapidyaml cp -fv $(BDIR)-shared/*.so $@ ln -fs $@ librapidyaml.so diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 02da00d71..0a337082f 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -5,10 +5,17 @@ extern "C" { #endif + static C4_NO_INLINE void throw_runtime_exception(JNIEnv * env, const char* msg); static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg); +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set(JNIEnv *, jobject, jboolean yes) +{ + ysparse_timing_set(yes); +} + JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) { @@ -16,6 +23,7 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) return (jlong)obj; } + JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) { @@ -140,6 +148,16 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- +static bool s_timing_enabled = false; +RYML_EXPORT bool ysparse_timing_get() +{ + return s_timing_enabled; +} +RYML_EXPORT void ysparse_timing_set(bool yes) +{ + s_timing_enabled = yes; +} + static C4_NO_INLINE void throw_java_exception(JNIEnv * env, const char* type, const char* msg) { jclass clazz = env->FindClass(type); diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index d3124b552..54c2aad02 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -7,6 +7,14 @@ #ifdef __cplusplus extern "C" { #endif +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_timing_set + * Signature: (Z)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set + (JNIEnv *, jobject, jboolean); + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2evt_init diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp index ce3c3d1b4..15bb2caf6 100644 --- a/rapidyaml/native/ysparse_common.hpp +++ b/rapidyaml/native/ysparse_common.hpp @@ -3,6 +3,7 @@ #define YSPARSE_COMMON_HPP_ #include +#include namespace ryml { using namespace c4; @@ -18,6 +19,19 @@ struct YsParseError : public std::exception const char* what() const noexcept override { return msg.c_str(); } }; + +//----------------------------------------------------------------------------- +// timing + +#ifdef __cplusplus +extern "C" { +#endif +RYML_EXPORT bool ysparse_timing_get(); +RYML_EXPORT void ysparse_timing_set(bool yes); +#ifdef __cplusplus +} +#endif + #ifndef YSPARSE_TIMED #define TIMED_SECTION(...) #error @@ -28,22 +42,28 @@ struct YsParseError : public std::exception struct timed_section { using myclock = std::chrono::steady_clock; - ryml::csubstr name; + const char* name; size_type len; myclock::time_point start; - timed_section(ryml::csubstr n, size_type len_=0) - : name(n) - , len(len_) - , start(myclock::now()) + C4_NO_INLINE timed_section(const char* n, size_type len_=0) { + if(ysparse_timing_get()) + { + name = n; + len = len_; + start = myclock::now(); + } } - ~timed_section() + C4_NO_INLINE ~timed_section() { - const std::chrono::duration t = myclock::now() - start; - fprintf(stderr, "%.6fms: %.*s", t.count(), (int)name.len, name.str); - if(len) - fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); - fprintf(stderr, "\n"); + if(ysparse_timing_get()) + { + const std::chrono::duration t = myclock::now() - start; + fprintf(stderr, "%.6fms: %s", t.count(), name); + if(len) + fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); + fprintf(stderr, "\n"); + } } }; #endif // YSPARSE_TIMED diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 644668b26..77f31c5e2 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -570,8 +570,14 @@ defn run(prompt session=nil): }; } // namespace -int main() +int main(int argc, const char *argv[]) { + for(int i = 1; i < argc; ++i) + { + csubstr arg = ryml::to_csubstr(argv[i]); + if(arg == "--timing" || arg == "-t") + ysparse_timing_set(true); + } Ys2EvtScoped ys2evt; TestResult total = {}; size_t failed_cases = {}; diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index a8803482b..4c7bc2fd4 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -13,6 +13,8 @@ public class Rapidyaml { public static String RAPIDYAML_VERSION = "0.8.0"; + private native void ysparse_timing_set(boolean yes); + // TODO: rename these to ysparse_init() etc private native long ys2evt_init(); private native void ys2evt_destroy(long ryml2evt); private native int ys2evt_parse(long ryml2evt, String filename, @@ -64,7 +66,7 @@ public int parseYsToEvtBuf(ByteBuffer src, IntBuffer evt) throws Exception public int parseYsToEvt(String filename, byte[] src, int[] evts) throws Exception { - long t = timingStart("ys2evtBuf"); + long t = timingStart("ys2evt"); int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); timingStop("ys2evt", t, src.length); return required_size; @@ -107,6 +109,7 @@ public static IntBuffer mkIntBuffer(int numInts) public void timingEnabled(boolean yes) { showTiming = yes; + ysparse_timing_set(yes); } private long timingStart(String name) From 586f7bf742900366f235144240c100576357f1b7 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 24 Feb 2025 00:59:52 +0000 Subject: [PATCH 17/29] rapidyaml: Rename ryml2evt/ys2evt to ysparse --- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 56 +++++++++---------- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 16 +++--- rapidyaml/native/ysparse_common.hpp | 3 +- rapidyaml/native/ysparse_evt.cpp | 42 +++++++------- rapidyaml/native/ysparse_evt.hpp | 16 +++--- rapidyaml/native/ysparse_test.cpp | 24 ++++---- .../main/java/org/rapidyaml/Rapidyaml.java | 34 +++++------ 7 files changed, 93 insertions(+), 98 deletions(-) diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 0a337082f..050c75aad 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -17,62 +17,56 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set(JNIEnv *, jobject, jboolean ye } JNIEXPORT jlong JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) +Java_org_rapidyaml_Rapidyaml_ysparse_1init(JNIEnv *env, jobject) { - Ryml2Evt *obj = ys2evt_init(); + ysparse *obj = ysparse_init(); return (jlong)obj; } JNIEXPORT void JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) +Java_org_rapidyaml_Rapidyaml_ysparse_1destroy(JNIEnv *, jobject, jlong obj) { - ys2evt_destroy((Ryml2Evt*)obj); + ysparse_destroy((ysparse*)obj); } JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, +Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject, jlong obj, jstring jfilename, jbyteArray src, jint src_len, jintArray dst, jint dst_len) { - TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + TIMED_SECTION("jni:ysparse", (size_type)src_len); jbyte* src_ = nullptr; int* dst_ = nullptr; const char *filename = nullptr; jboolean dst_is_copy = false; jboolean src_is_copy = false; { - TIMED_SECTION("jni_ys2evt_parse/get_jni", (size_type)src_len); + TIMED_SECTION("jni:ysparse/get_jni", (size_type)src_len); + // this is __S__L__O__W__ + // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time + // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string { - TIMED_SECTION("jni_ys2evt_parse/GetByteArray(src)"); + TIMED_SECTION("jni:ysparse/GetByteArray(src)"); src_ = env->GetByteArrayElements(src, &src_is_copy); } { - // TODO this is __S__L__O__W__ - // - // the problem is with GetIntArrayElements(). we should - // use GetDirectBufferAddress(), but that requires a ByteBuffer->jobject - // instead of a int[]->jintArray - // - // see: - // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time - // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string - TIMED_SECTION("jni_ys2evt_parse/GetIntArray(dst)"); + TIMED_SECTION("jni:ysparse/GetIntArray(dst)"); dst_ = env->GetIntArrayElements(dst, &dst_is_copy); } { - TIMED_SECTION("jni_ys2evt_parse/GetStringUTFChars()"); + TIMED_SECTION("jni:ysparse/GetStringUTFChars()"); filename = env->GetStringUTFChars(jfilename, 0); } } int rc = 0; { - TIMED_SECTION("jni_ys2evt_parse/call_parse", (size_type)src_len); + TIMED_SECTION("jni:ysparse/parse", (size_type)src_len); try { - rc = ys2evt_parse((Ryml2Evt*)obj, filename, + rc = ysparse_parse((ysparse*)obj, filename, (char*)src_, src_len, dst_, dst_len); } @@ -86,18 +80,18 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, } } { - TIMED_SECTION("jni_ys2evt_parse/release"); + TIMED_SECTION("jni:ysparse/release"); + // __S__L__O__W__ { - TIMED_SECTION("jni_ys2evt_parse/ReleaseByteArray(src)"); + TIMED_SECTION("jni:ysparse/ReleaseByteArray(src)"); env->ReleaseByteArrayElements(src, src_, 0); } { - // TODO __S__L__O__W__ - TIMED_SECTION("jni_ys2evt_parse/ReleaseIntArray(dst)"); + TIMED_SECTION("jni:ysparse/ReleaseIntArray(dst)"); env->ReleaseIntArrayElements(dst, dst_, 0); } { - TIMED_SECTION("jni_ys2evt_parse/ReleaseStringUTFChars()"); + TIMED_SECTION("jni:ysparse/ReleaseStringUTFChars()"); env->ReleaseStringUTFChars(jfilename, filename); } } @@ -106,17 +100,17 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, JNIEXPORT jint JNICALL -Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, +Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf(JNIEnv *env, jobject, jlong obj, jstring jfilename, jobject src, jint src_len, jobject dst, jint dst_len) { - TIMED_SECTION("jni_ys2evt_parse", (size_type)src_len); + TIMED_SECTION("jni:ysparse_buf", (size_type)src_len); char* src_ = nullptr; int* dst_ = nullptr; const char *filename = nullptr; { - TIMED_SECTION("jni_ys2evt_parse/get_jni", (size_type)src_len); + TIMED_SECTION("jni:ysparse_buf/get_jni", (size_type)src_len); src_ = (char*)env->GetDirectBufferAddress(src); dst_ = (int*)env->GetDirectBufferAddress(dst); filename = env->GetStringUTFChars(jfilename, 0); @@ -126,10 +120,10 @@ Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf(JNIEnv *env, jobject, throw_runtime_exception(env, "null pointer: dst"); } { - TIMED_SECTION("jni_ys2evt_parse/call_parse", (size_type)src_len); + TIMED_SECTION("jni:ysparse_buf/parse", (size_type)src_len); try { - return ys2evt_parse((Ryml2Evt*)obj, filename, src_, src_len, dst_, dst_len); + return ysparse_parse((ysparse*)obj, filename, src_, src_len, dst_, dst_len); } catch (YsParseError const& exc) { diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index 54c2aad02..021ccff69 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -17,34 +17,34 @@ JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_init + * Method: ysparse_init * Signature: ()J */ -JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1init (JNIEnv *, jobject); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_destroy + * Method: ysparse_destroy * Signature: (J)V */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1destroy (JNIEnv *, jobject, jlong); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_parse + * Method: ysparse_parse * Signature: (JLjava/lang/String;[BI[II)I */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); /* * Class: org_rapidyaml_Rapidyaml - * Method: ys2evt_parse_buf + * Method: ysparse_parse_buf * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/IntBuffer;I)I */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse_1buf +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); #ifdef __cplusplus diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp index 15bb2caf6..48e4dc090 100644 --- a/rapidyaml/native/ysparse_common.hpp +++ b/rapidyaml/native/ysparse_common.hpp @@ -52,6 +52,7 @@ struct timed_section name = n; len = len_; start = myclock::now(); + //fprintf(stderr, "%10s : %s...\n", " ", name); } } C4_NO_INLINE ~timed_section() @@ -59,7 +60,7 @@ struct timed_section if(ysparse_timing_get()) { const std::chrono::duration t = myclock::now() - start; - fprintf(stderr, "%.6fms: %s", t.count(), name); + fprintf(stderr, "%10.6fms: %s", t.count(), name); if(len) fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); fprintf(stderr, "\n"); diff --git a/rapidyaml/native/ysparse_evt.cpp b/rapidyaml/native/ysparse_evt.cpp index 9ef064b3f..0c3497666 100644 --- a/rapidyaml/native/ysparse_evt.cpp +++ b/rapidyaml/native/ysparse_evt.cpp @@ -10,7 +10,7 @@ extern "C" { // https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method namespace { -C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) +C4_NORETURN void ysparse_error(const char* msg, size_t msg_len, Location location, void *user_data) { YsParseError exc; exc.location = location; @@ -19,43 +19,43 @@ C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location lo } } // anon namespace -RYML_EXPORT Ryml2Evt *ys2evt_init() +RYML_EXPORT ysparse *ysparse_init() { - TIMED_SECTION("ys2evt_init"); + TIMED_SECTION("cpp:ysparse_init"); Callbacks cb = {}; - cb.m_error = &ys2evt_parse_error; + cb.m_error = &ysparse_error; set_callbacks(cb); - Ryml2Evt *ryml2evt = _RYML_CB_ALLOC(get_callbacks(), Ryml2Evt, 1); + ysparse *ryml2evt = _RYML_CB_ALLOC(get_callbacks(), ysparse, 1); _RYML_CB_CHECK(get_callbacks(), ryml2evt != nullptr); - new ((void*)ryml2evt) Ryml2Evt(); + new ((void*)ryml2evt) ysparse(); return ryml2evt; } -RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt) +RYML_EXPORT void ysparse_destroy(ysparse *obj) { - TIMED_SECTION("ys2evt_destroy"); - ryml2evt->~Ryml2Evt(); - _RYML_CB_FREE(get_callbacks(), ryml2evt, Ryml2Evt, 1); + TIMED_SECTION("cpp:ysparse_destroy"); + obj->~ysparse(); + _RYML_CB_FREE(get_callbacks(), obj, ysparse, 1); } -RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, - const char *filename, - char *ys, size_type ys_size, - evt::DataType *events, size_type evt_size) +RYML_EXPORT size_type ysparse_parse(ysparse *obj, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *events, size_type evt_size) { - TIMED_SECTION("ys2evt_parse", ys_size); + TIMED_SECTION("cpp:ysparse", ys_size); csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); { - TIMED_SECTION("reset + reserve"); - ryml2evt->reset(ys_, events, evt_size); - ryml2evt->m_handler.reserve(256u); + TIMED_SECTION("cpp:ysparse/reset"); + obj->reset(ys_, events, evt_size); + obj->m_handler.reserve(256u); } { - TIMED_SECTION("parse_in_place", ys_size); - ryml2evt->m_parser.parse_in_place_ev(filename_, ys_); + TIMED_SECTION("cpp:ysparse/parse", ys_size); + obj->m_parser.parse_in_place_ev(filename_, ys_); } - return (size_type)ryml2evt->m_handler.m_evt_curr; + return (size_type)obj->m_handler.m_evt_curr; } #if defined(__cplusplus) diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp index cc231ae1a..9b0c9dbb8 100644 --- a/rapidyaml/native/ysparse_evt.hpp +++ b/rapidyaml/native/ysparse_evt.hpp @@ -9,11 +9,11 @@ extern "C" { #endif -struct RYML_EXPORT Ryml2Evt +struct RYML_EXPORT ysparse { ys::EventHandlerEvt m_handler; c4::yml::ParseEngine m_parser; - Ryml2Evt() + ysparse() : m_handler() , m_parser(&m_handler) { @@ -29,10 +29,10 @@ struct RYML_EXPORT Ryml2Evt //----------------------------------------------------------------------------- /** Initialize the resources */ -RYML_EXPORT Ryml2Evt *ys2evt_init(); +RYML_EXPORT ysparse *ysparse_init(); /** Destroy the resources */ -RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt); +RYML_EXPORT void ysparse_destroy(ysparse *ryml2evt); /** Parse YAML in the string `ys` of size `ys_size`, and write the * result into the array of (integer) events `evt` of size @@ -82,10 +82,10 @@ RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt); * in-place in the input string, and the extra integers will pertain * to the resulting filtered string. */ -RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, - const char *filename, - char *ys, size_type ys_size, - evt::DataType *evt, size_type evt_size); +RYML_EXPORT size_type ysparse_parse(ysparse *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *evt, size_type evt_size); #if defined(__cplusplus) } diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 77f31c5e2..8567e3342 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -45,9 +45,9 @@ c4::EnumSymbols const esyms() struct Ys2EvtScoped { - Ryml2Evt *ryml2evt; - Ys2EvtScoped() : ryml2evt(ys2evt_init()) {} - ~Ys2EvtScoped() { if(ryml2evt) ys2evt_destroy(ryml2evt); } + ysparse *ryml2evt; + Ys2EvtScoped() : ryml2evt(ysparse_init()) {} + ~Ys2EvtScoped() { if(ryml2evt) ysparse_destroy(ryml2evt); } }; @@ -127,7 +127,7 @@ struct TestCase } \ } while(0) - TestResult test(Ryml2Evt *ryml2evt) const + TestResult test(ysparse *ryml2evt) const { TestResult tr = {}; _runtest(test_evt_large_enough, ); @@ -140,7 +140,7 @@ struct TestCase } // happy path: large-enough destination string - TestResult test_evt_large_enough_reuse(Ryml2Evt *ryml2evt) const + TestResult test_evt_large_enough_reuse(ysparse *ryml2evt) const { if(evt.empty()) return {}; TestResult tr = {}; @@ -148,7 +148,7 @@ struct TestCase substr input = c4::to_substr(input_); std::vector output; output.resize(2 * expected_size(evt)); - size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, &output[0], (size_type)output.size()); CHECK_MSG((size_t)reqsize == expected_size(evt), "%d vs %zu", reqsize, expected_size(evt)); @@ -164,14 +164,14 @@ struct TestCase } // less-happy path: destination string not large enough - TestResult test_evt_too_small_reuse(Ryml2Evt *ryml2evt) const + TestResult test_evt_too_small_reuse(ysparse *ryml2evt) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); std::vector output; output.resize(expected_size(evt)); - size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, output.data(), (size_type)output.size()); CHECK(reqsize == expected_size(evt)); @@ -179,7 +179,7 @@ struct TestCase output.resize(reqsize); input_.assign(ys.begin(), ys.end()); // FIXME input = c4::to_substr(input_); - size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize2 = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, output.data(), (size_type)output.size()); CHECK(reqsize2 == reqsize); @@ -194,12 +194,12 @@ struct TestCase } // safe calling with nullptr - TestResult test_evt_nullptr_reuse(Ryml2Evt *ryml2evt) const + TestResult test_evt_nullptr_reuse(ysparse *ryml2evt) const { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); - size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, nullptr, 0); CHECK(reqsize == expected_size(evt)); @@ -208,7 +208,7 @@ struct TestCase output.resize(reqsize); input_.assign(ys.begin(), ys.end()); // FIXME input = c4::to_substr(input_); - size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + size_type reqsize2 = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, output.data(), (size_type)output.size()); CHECK(reqsize2 == reqsize); diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 4c7bc2fd4..f071e1ac9 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -15,22 +15,22 @@ public class Rapidyaml private native void ysparse_timing_set(boolean yes); // TODO: rename these to ysparse_init() etc - private native long ys2evt_init(); - private native void ys2evt_destroy(long ryml2evt); - private native int ys2evt_parse(long ryml2evt, String filename, - byte[] ys, int ys_length, - int[] evt, int evt_length); - private native int ys2evt_parse_buf(long ryml2evt, String filename, - ByteBuffer ys, int ys_length, - IntBuffer evt, int evt_length); + private native long ysparse_init(); + private native void ysparse_destroy(long ysparse); + private native int ysparse_parse(long ysparse, String filename, + byte[] ys, int ys_length, + int[] evt, int evt_length); + private native int ysparse_parse_buf(long ysparse, String filename, + ByteBuffer ys, int ys_length, + IntBuffer evt, int evt_length); - private final long ryml2evt; + private final long ysparse; public Rapidyaml() { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); - this.ryml2evt = this.ys2evt_init(); + this.ysparse = this.ysparse_init(); // TODO: receive this argument as ctor parameter timingEnabled(System.getenv("YS_RYML_TIMER") != null); } @@ -42,7 +42,7 @@ public Rapidyaml() protected void finalize() throws Throwable { try { - this.ys2evt_destroy(this.ryml2evt); + this.ysparse_destroy(this.ysparse); } finally { super.finalize(); @@ -66,9 +66,9 @@ public int parseYsToEvtBuf(ByteBuffer src, IntBuffer evt) throws Exception public int parseYsToEvt(String filename, byte[] src, int[] evts) throws Exception { - long t = timingStart("ys2evt"); - int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); - timingStop("ys2evt", t, src.length); + long t = timingStart("ysparse"); + int required_size = ysparse_parse(this.ysparse, filename, src, src.length, evts, evts.length); + timingStop("ysparse", t, src.length); return required_size; } @@ -82,13 +82,13 @@ public int parseYsToEvtBuf(String filename, ByteBuffer src, IntBuffer evt) throw // but for evt it really does if(evt.order() != ByteOrder.nativeOrder()) throw new RuntimeException("evt byte order must be native"); - long t = timingStart("ys2evtBuf"); + long t = timingStart("ysparseBuf"); evt.position(evt.capacity()); - int reqsize = ys2evt_parse_buf(this.ryml2evt, filename, src, src.position(), evt, evt.capacity()); + int reqsize = ysparse_parse_buf(this.ysparse, filename, src, src.position(), evt, evt.capacity()); if(reqsize <= evt.capacity()) { evt.position(reqsize); } - timingStop("ys2evtBuf", t, src.position()); + timingStop("ysparseBuf", t, src.position()); return reqsize; } From dd51414e568e109fe8e892bbc46a4ea0ea22cd6c Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 24 Feb 2025 19:14:27 +0000 Subject: [PATCH 18/29] rapidyaml: Working with native-image --- common/native.mk | 1 + common/vars.mk | 8 +- core/src/yamlscript/parser.clj | 89 ++++++++-- mvp/Makefile | 20 +++ mvp/pom.xml | 75 ++++++++ mvp/src/main/java/com/example/App.java | 12 ++ mvp/src/test/java/com/example/AppTest.java | 13 ++ rapidyaml/Makefile | 46 ++--- rapidyaml/manifest.txt | 1 + rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 14 +- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 16 +- rapidyaml/native/ysparse_test.cpp | 163 +++++++++++++++--- .../java/org/rapidyaml/NativeLibLoader.java | 59 +++++++ .../main/java/org/rapidyaml/Rapidyaml.java | 138 ++++++++------- .../java/org/rapidyaml/RapidyamlTest.java | 58 +++++-- ys/Makefile | 12 ++ 16 files changed, 575 insertions(+), 150 deletions(-) create mode 100644 mvp/Makefile create mode 100644 mvp/pom.xml create mode 100644 mvp/src/main/java/com/example/App.java create mode 100644 mvp/src/test/java/com/example/AppTest.java create mode 100644 rapidyaml/manifest.txt create mode 100644 rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java diff --git a/common/native.mk b/common/native.mk index 34c300961..62a98c696 100644 --- a/common/native.mk +++ b/common/native.mk @@ -9,6 +9,7 @@ NATIVE_OPTS := \ --emit=build-report \ -march=compatibility \ -H:IncludeResources=SCI_VERSION \ + -H:IncludeResources='librapidyaml\.0\.8\.0\.so' \ -H:ReflectionConfigurationFiles=reflection.json \ -H:+ReportExceptionStackTraces \ -H:Log=registerResource: \ diff --git a/common/vars.mk b/common/vars.mk index 704856e91..27e80c42b 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -223,21 +223,23 @@ RAPIDYAML := $(ROOT)/rapidyaml RAPIDYAML_VERSION := 0.8.0 #RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) -RAPIDYAML_TAG ?= 8c37616378aefd376690a19459c31a56ce596b5e +RAPIDYAML_TAG ?= d3132a25ec21c65e27ff46ab8c3d61c72a321302 RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml RAPIDYAML_BUILD_TYPE ?= Release RAPIDYAML_DBG ?= 0 RAPIDYAML_TIMED ?= 1 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java RAPIDYAML_JNI_H := $(ROOT)/rapidyaml/native/org_rapidyaml_Rapidyaml.h -RAPIDYAML_SO := $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_SO_NAME := librapidyaml.$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_SO := $(ROOT)/rapidyaml/native/$(RAPIDYAML_SO_NAME) RAPIDYAML_LIB := $(ROOT)/rapidyaml/native/librapidyaml.$(DOTLIB) RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar RAPIDYAML_INSTALLED_DIR := \ - $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)/ + $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION) RAPIDYAML_INSTALLED := \ $(RAPIDYAML_INSTALLED_DIR)/rapidyaml-$(RAPIDYAML_VERSION).jar diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 96be8683e..9f1e36b2a 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -10,6 +10,7 @@ [yamlscript.common]) (:import (java.util Optional) + (java.nio ByteBuffer) (java.nio.charset StandardCharsets) (org.rapidyaml Evt Rapidyaml) (org.snakeyaml.engine.v2.api LoadSettings) @@ -31,17 +32,20 @@ (declare parse-fn) +(defn TIMER [] (System/getenv "YS_TIMER")) + (def shebang-ys #"^#!.*/env ys-0(?:\.d+\.\d+)?\n") (def shebang-bash #"^#!.*[/ ]bash\n+source +<\(") + (defn parse "Parse a YAML string into a sequence of event objects." [yaml-string] (let [has-code-mode-shebang (or (re-find shebang-ys yaml-string) (re-find shebang-bash yaml-string)) - events (if (System/getenv "YS_PARSER_TIME") - (time (parse-fn yaml-string)) - (parse-fn yaml-string)) + events (if (TIMER) + (time ((parse-fn) yaml-string)) + ((parse-fn) yaml-string)) [first-event & rest-events] events first-event-tag (:! first-event) first-event (if (and has-code-mode-shebang @@ -184,12 +188,16 @@ (defn parse-rapidyaml [^String yaml-string] (rest (let [parser ^Rapidyaml (new Rapidyaml) + _ (when (TIMER) + (.timingEnabled parser true)) buffer (.getBytes yaml-string StandardCharsets/UTF_8) masks (int-array 5) needed (.parseYsToEvt parser buffer masks) buffer (.getBytes yaml-string StandardCharsets/UTF_8) masks (int-array needed) _ (.parseYsToEvt parser buffer masks) + ;; TODO: aget slow? + ;; https://stackoverflow.com/questions/10133094/clojure-why-is-aget-so-slow get-str (fn [i] (let [off (aget masks (inc i)) len (aget masks (+ i 2))] @@ -228,17 +236,70 @@ (recur i tag anchor events))) events))))) -(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")] - (condp = parser-name - "" parse-snakeyaml - "snake" parse-snakeyaml - "rapid" parse-rapidyaml - "ryml" parse-rapidyaml - ; TODO: - ;"rapid-buf" parse-rapidyaml-buf - ;"ryml-buf" parse-rapidyaml-buf - (die "Unknown YS_PARSER value: " parser-name)) - parse-snakeyaml)) +(defn parse-rapidyaml-buf [^String yaml-string] + (rest + (let [parser (new Rapidyaml) + _ (when (TIMER) + (.timingEnabled parser true)) + srcbytes (.getBytes yaml-string StandardCharsets/UTF_8) + srcbuffer (ByteBuffer/allocateDirect (alength srcbytes)) + _ (.put srcbuffer srcbytes) + masks (Rapidyaml/mkIntBuffer 5) + needed (.parseYsToEvtBuf parser srcbuffer masks) + _ (.position srcbuffer 0) + _ (.put srcbuffer srcbytes) + masks (Rapidyaml/mkIntBuffer needed) + _ (.parseYsToEvtBuf parser srcbuffer masks) + get-str (fn [i] + (let [off (.get masks ^Long (inc i)) + len (.get masks ^Long (+ i 2))] + (reduce + (fn [slice i] (str slice + (char (.get srcbuffer ^Long i)))) + "" (range off (+ off len)))))] + + (loop [i 0, tag nil, anchor nil, events []] + (if (< i needed) + (let [mask (.get masks i) + type (event-type mask) + ; _ (WWW (Integer/toString mask 2) type) + sval (when (flag? HAS_STR mask) (get-str i)) + tag (if (flag? TAG_ mask) sval tag) + anchor (if (flag? ANCH mask) sval anchor) + event (when type + (let [event {:+ type} + event (if (flag? FLOW mask) + (assoc event :flow true) event) + event (if anchor (assoc event :& anchor) event) + event (if tag + (let [tag (str/replace tag + #"^!!" + "tag:yaml.org,2002:")] + (assoc event :! tag)) event) + event (if sval (assoc event + (get-skey mask) sval) event) + event (if (= type "=ALI") + {:+ "=ALI" :* sval} + event)] + event)) + events (if event (conj events event) events) + i (+ i (if sval 3 1))] + (if event + (recur i nil nil events) + (recur i tag anchor events))) + events))))) + +(defn parse-fn [] + (if-let [parser-name (System/getenv "YS_PARSER")] + (condp = parser-name + "" parse-snakeyaml + "snake" parse-snakeyaml + "rapid" parse-rapidyaml + "rapid-buf" parse-rapidyaml-buf + "ryml" parse-rapidyaml + "ryml-buf" parse-rapidyaml-buf + (die "Unknown YS_PARSER value: " parser-name)) + parse-snakeyaml)) (comment ) diff --git a/mvp/Makefile b/mvp/Makefile new file mode 100644 index 000000000..50d26712b --- /dev/null +++ b/mvp/Makefile @@ -0,0 +1,20 @@ +include ../common/base.mk +include $(COMMON)/java.mk + +JAVA_FILE := src/main/java/com/example/App.java +JAR_FILE := target/my-app-1.0-SNAPSHOT.jar + + +default:: + which mvn + +build:: $(JAR_FILE) + +$(JAR_FILE): $(JAVA_FILE) + mvn clean package + +run:: build + java -jar target/my-app-1.0-SNAPSHOT.jar + +test clean package:: + mvn $@ diff --git a/mvp/pom.xml b/mvp/pom.xml new file mode 100644 index 000000000..9ef65ac16 --- /dev/null +++ b/mvp/pom.xml @@ -0,0 +1,75 @@ + + + + 4.0.0 + + com.example + my-app + 1.0-SNAPSHOT + + + 11 + 11 + UTF-8 + + + + + + + + junit + junit + 4.13.2 + test + + + + org.rapidyaml + rapidyaml + 0.8.0 + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + + true + com.example.App + + + + + + + + + diff --git a/mvp/src/main/java/com/example/App.java b/mvp/src/main/java/com/example/App.java new file mode 100644 index 000000000..2714e45e8 --- /dev/null +++ b/mvp/src/main/java/com/example/App.java @@ -0,0 +1,12 @@ +package com.example; + +// import org.apache.commons.lang3.StringUtils; +import org.rapidyaml.Rapidyaml; + +public class App { + public static void main(String[] args) { + String text = "Hello, World!"; + System.out.println("Original text: " + text); + // System.out.println("Reversed text: " + StringUtils.reverse(text)); + } +} diff --git a/mvp/src/test/java/com/example/AppTest.java b/mvp/src/test/java/com/example/AppTest.java new file mode 100644 index 000000000..0a7ec0b34 --- /dev/null +++ b/mvp/src/test/java/com/example/AppTest.java @@ -0,0 +1,13 @@ +package com.example; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class AppTest { + @Test + public void testApp() { + // Simple test to verify the App class can be instantiated + App app = new App(); + assertNotNull(app); + } +} diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index af0d7ea2b..31f5d09c3 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -2,39 +2,31 @@ include ../common/base.mk include $(COMMON)/clojure.mk include $(COMMON)/java.mk -RAPIDYAML_JAR_DEPS := \ - $(wildcard \ - build \ - $(JAVA_INSTALLED) \ - Makefile \ - pom.xml \ - ) - RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class) +RAPIDYAML_CLASSES := \ + $(RAPIDYAML_CLASS:$(RAPIDYAML)/src/main/java/%=-C src/main/java %) #------------------------------------------------------------------------------ default:: - echo $(RAPIDYAML_SO) - echo $(RAPIDYAML_LIB) - -x: - YS -ce 'foo: var' -build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) +build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) $(RAPIDYAML_JAR) @: jar: $(RAPIDYAML_JAR) +jar-test: $(RAPIDYAML_JAR) + java -jar $< + install:: $(RAPIDYAML_INSTALLED) test:: build $(JAVA_INSTALLED) $(MAKE) -C native $@ - $(MVN) $@ + YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) $@ test-x: build $(JAVA_INSTALLED) - $(MAKE) -C native $@ - $(MVN) -X -e test + $(MAKE) -C native test + YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) -X -e test clean:: $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @@ -43,6 +35,7 @@ clean:: realclean:: clean $(MAKE) -C native $@ + $(RM) -r $(HOME)/.m2/repository/org/rapidyaml/ sysclean:: $(RM) -r $(MAVEN_REPOSITORY)/org/rapidyaml @@ -53,14 +46,23 @@ jni: $(RAPIDYAML_JNI_H) #------------------------------------------------------------------------------ $(RAPIDYAML_SO): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ + +# XXX Probably don't need this: $(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ -$(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED) - $(MVN) install - -$(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED) - $(JAR) -cvf $@ $< +$(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) +ifdef YS_MAVEN_TEST + YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) install +else + $(MVN) install -Dmaven.test.skip +endif + touch $@ + +$(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) + $(JAR) -cf $@ $(RAPIDYAML_CLASSES) + $(JAR) -uf $@ -C $(RAPIDYAML)/native/ $(RAPIDYAML_SO_NAME) + $(JAR) -umf manifest.txt $@ $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) @# this doesn't work: diff --git a/rapidyaml/manifest.txt b/rapidyaml/manifest.txt new file mode 100644 index 000000000..280e5a906 --- /dev/null +++ b/rapidyaml/manifest.txt @@ -0,0 +1 @@ +Main-Class: org.rapidyaml.Rapidyaml diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 050c75aad..d02df8fb4 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -154,9 +154,9 @@ RYML_EXPORT void ysparse_timing_set(bool yes) static C4_NO_INLINE void throw_java_exception(JNIEnv * env, const char* type, const char* msg) { - jclass clazz = env->FindClass(type); - if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown - env->ThrowNew(clazz, msg); + jclass cls = env->FindClass(type); + if (cls != NULL) // if it is null, a NoClassDefFoundError was already thrown + env->ThrowNew(cls, msg); } static C4_NO_INLINE void throw_runtime_exception(JNIEnv *env, const char* msg) @@ -167,8 +167,8 @@ static C4_NO_INLINE void throw_runtime_exception(JNIEnv *env, const char* msg) static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg) { // see https://stackoverflow.com/questions/55013243/jni-custom-exceptions-with-more-than-one-parameter - jclass clazz = env->FindClass("org/rapidyaml/YamlParseErrorException"); - if (clazz != NULL) // if it is null, a NoClassDefFoundError was already thrown + jclass cls = env->FindClass("org/rapidyaml/YamlParseErrorException"); + if (cls != NULL) // if it is null, a NoClassDefFoundError was already thrown { jstring jmsg = env->NewStringUTF(msg); jint joffset = (jint)offset; @@ -178,8 +178,8 @@ static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t li // about the proper signature. // we want (int, int, int, String): const char * const signature = "(IIILjava/lang/String;)V"; - jmethodID ctor = env->GetMethodID(clazz, "", signature); - jobject jexc = env->NewObject(clazz, ctor, joffset, jline, jcol, jmsg); + jmethodID ctor = env->GetMethodID(cls, "", signature); + jobject jexc = env->NewObject(cls, ctor, joffset, jline, jcol, jmsg); env->Throw((jthrowable)jexc); // https://stackoverflow.com/questions/2455668/jni-cast-between-jobect-and-jthrowable } } diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index 021ccff69..ea8355bc0 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -7,14 +7,6 @@ #ifdef __cplusplus extern "C" { #endif -/* - * Class: org_rapidyaml_Rapidyaml - * Method: ysparse_timing_set - * Signature: (Z)V - */ -JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set - (JNIEnv *, jobject, jboolean); - /* * Class: org_rapidyaml_Rapidyaml * Method: ysparse_init @@ -31,6 +23,14 @@ JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1init JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1destroy (JNIEnv *, jobject, jlong); +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_timing_set + * Signature: (Z)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set + (JNIEnv *, jobject, jboolean); + /* * Class: org_rapidyaml_Rapidyaml * Method: ysparse_parse diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 8567e3342..63de41d49 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -4,6 +4,8 @@ using c4::csubstr; using c4::substr; +using ryml::Location; + namespace c4 { @@ -94,6 +96,47 @@ size_t expected_size(std::vector const& evt) return exp; } +#define _runtest(name, ...) \ + do { \ + printf("[ RUN ] %s ... \n", #name); \ + TestResult tr_ = name(__VA_ARGS__); \ + tr.add(tr_); \ + printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ + } while(0) + +#define CHECK(cond) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s\n", __FILE__, __LINE__, #cond); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + +#define CHECK_EQ(lhs, rhs) \ + do { \ + bool pass = !!(lhs == rhs); \ + ++tr.num_assertions; \ + if(!pass) { \ + std::string slhs = c4::catrs(lhs); \ + std::string srhs = c4::catrs(rhs); \ + printf("%s:%d: fail! %s=%s == %s=%s\n", __FILE__, __LINE__, #lhs, slhs.c_str(), #rhs, srhs.c_str()); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + +#define CHECK_MSG(cond, fmt, ...) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s:" fmt "\n", __FILE__, __LINE__, #cond, ## __VA_ARGS__); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + + struct TestCase { csubstr ys; @@ -101,32 +144,6 @@ struct TestCase public: - #define _runtest(name, ...) \ - do { \ - printf("[ RUN ] %s ... \n", #name); \ - TestResult tr_ = name(__VA_ARGS__); \ - tr.add(tr_); \ - printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ - } while(0) - #define CHECK(cond) \ - do { \ - bool pass = !!(cond); \ - ++tr.num_assertions; \ - if(!pass) { \ - printf("%s:%d: fail! %s\n", __FILE__, __LINE__, #cond); \ - ++tr.num_failed_assertions; \ - } \ - } while(0) - #define CHECK_MSG(cond, fmt, ...) \ - do { \ - bool pass = !!(cond); \ - ++tr.num_assertions; \ - if(!pass) { \ - printf("%s:%d: fail! %s:" fmt "\n", __FILE__, __LINE__, #cond, ## __VA_ARGS__); \ - ++tr.num_failed_assertions; \ - } \ - } while(0) - TestResult test(ysparse *ryml2evt) const { TestResult tr = {}; @@ -298,6 +315,65 @@ struct TestCase }; +//----------------------------------------------------------------------------- + +struct TestCaseErr +{ + csubstr ys; + bool is_parse_err; + ryml::Location loc; + + TestCaseErr(csubstr ys_) : ys(ys_), is_parse_err(false), loc() {} + TestCaseErr(csubstr ys_, ryml::Location loc_) : ys(ys_), is_parse_err(true), loc(loc_) {} + + TestResult test(ysparse *ryml2evt) const + { + TestResult tr = {}; + _runtest(test_err, ); + _runtest(test_err_reuse, ryml2evt); + return tr; + } + + TestResult test_err_reuse(ysparse *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + bool gotit = false; + try + { + size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + } + catch(YsParseError const& exc) + { + if(is_parse_err) + { + gotit = true; + CHECK_EQ(exc.location.name, "ysfilename"); + CHECK_EQ(exc.location.line, loc.line); + CHECK_EQ(exc.location.col, loc.col); + CHECK_EQ(exc.location.offset, loc.offset); + } + } + catch(std::exception const& exc) + { + if(!is_parse_err) + gotit = true; + } + CHECK(gotit); + return tr; + return tr; + } + TestResult test_err() const + { + Ys2EvtScoped lib; + return test_err_reuse(lib.ryml2evt); + } +}; + + //----------------------------------------------------------------------------- namespace { @@ -308,6 +384,20 @@ using namespace evt; inline constexpr bool needs_filter = true; const TestCase test_cases[] = { // case ------------------------------------------------- + tc("!yamlscript/v0/bare\n--- !code\n42\n", + { + e(BSTR), + e(BDOC), + e(VAL_|TAG_, 1, 18, "yamlscript/v0/bare"), + e(VAL_|SCLR|PLAI, 0, 0, ""), + e(EDOC), + e(BDOC|EXPL), + e(VAL_|TAG_, 25, 4, "code"), + e(VAL_|SCLR|PLAI, 30, 2, "42"), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- tc("a: 1", { e(BSTR), @@ -568,8 +658,16 @@ defn run(prompt session=nil): e(ESTR), }), }; +#define tcf(...) TestCaseErr(__VA_ARGS__) +const TestCaseErr test_cases_err[] = { + tcf("- !!str, xxx\n", Location(13, 2, 1)), + //FIXME tcf(": : : :", Location(2, 1, 3)), +}; } // namespace + +//----------------------------------------------------------------------------- + int main(int argc, const char *argv[]) { for(int i = 1; i < argc; ++i) @@ -592,9 +690,20 @@ int main(int argc, const char *argv[]) failed_cases += (!tr); printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); } + size_t num_cases_err = C4_COUNTOF(test_cases_err); + for(size_t i = 0; i < C4_COUNTOF(test_cases_err); ++i) + { + printf("-----------------------------------------\n" + "errcase %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", i, num_cases_err, test_cases_err[i].ys.len, (int)test_cases_err[i].ys.len, test_cases_err[i].ys.str); + const TestResult tr = test_cases_err[i].test(ys2evt.ryml2evt); + total.add(tr); + failed_cases += (!tr); + printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + } printf("assertions: %u/%u pass %u/%u fail\n", total.num_assertions - total.num_failed_assertions, total.num_assertions, total.num_failed_assertions, total.num_assertions); printf("tests: %u/%u pass %u/%u fail\n", total.num_tests - total.num_failed_tests, total.num_tests, total.num_failed_tests, total.num_tests); - printf("cases: %zu/%zu pass %zu/%zu fail\n", num_cases-failed_cases, num_cases, failed_cases, num_cases); + printf("cases: %zu/%zu pass %zu/%zu fail\n", num_cases-failed_cases, num_cases+num_cases_err, failed_cases, num_cases); if(total) printf("TESTS SUCCEED!\n"); return total ? 0 : -1; diff --git a/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java b/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java new file mode 100644 index 000000000..68b4b35d7 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java @@ -0,0 +1,59 @@ +package org.rapidyaml; + +import java.io.File; +import java.io.InputStream; +import java.io.IOException; + +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +class NativeLibLoader { + public static void loadLibraryFromResource(String libraryName) + throws IOException + { +// System.out.println("libraryName: " + libraryName); + ClassLoader classLoader = + Thread.currentThread().getContextClassLoader(); + + InputStream inputStream = + classLoader.getResourceAsStream(libraryName); + + if (inputStream == null) + throw new IOException( + "Failed to load library resource '" + + libraryName + "' from NativeLibLoader"); + +// System.out.println("inputStream: " + inputStream); + + File tempDir = createTempDir(); + // tempDir.deleteOnExit(); + File tempFile = new File(tempDir, libraryName); +// System.out.println("tempFile: " + tempFile); + + Files.copy( + inputStream, + tempFile.toPath(), + StandardCopyOption.REPLACE_EXISTING); + + inputStream.close(); + + try { + System.load(tempFile.getAbsolutePath()); + } + finally { + // tempFile.delete(); + } + +// System.out.println("Loaded library from temp file"); + } + + private static File createTempDir() throws IOException { + String tempBase = System.getProperty("java.io.tmpdir"); + File tempDir = new File(tempBase, "" + System.nanoTime()); + if (! tempDir.mkdir()) + throw new IOException( + "Failed to create temp directory " + + tempDir.getName()); + return tempDir; + } +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index f071e1ac9..1d16a8893 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -1,119 +1,140 @@ package org.rapidyaml; -import org.rapidyaml.YamlParseErrorException; -import java.nio.charset.StandardCharsets; +import org.rapidyaml.NativeLibLoader; + +import java.io.IOException; + import java.nio.ByteBuffer; import java.nio.IntBuffer; import java.nio.ByteOrder; /** - * Interface with the shared librapidyaml library + * Interface with the librapidyaml shared library */ -public class Rapidyaml -{ - public static String RAPIDYAML_VERSION = "0.8.0"; +public class Rapidyaml { + public static String RAPIDYAML_NAME = "rapidyaml.0.8.0"; + public static String RAPIDYAML_LIBNAME = "librapidyaml.0.8.0.so"; - private native void ysparse_timing_set(boolean yes); - // TODO: rename these to ysparse_init() etc private native long ysparse_init(); private native void ysparse_destroy(long ysparse); - private native int ysparse_parse(long ysparse, String filename, - byte[] ys, int ys_length, - int[] evt, int evt_length); - private native int ysparse_parse_buf(long ysparse, String filename, - ByteBuffer ys, int ys_length, - IntBuffer evt, int evt_length); - + private native void ysparse_timing_set(boolean yes); + private native int ysparse_parse( + long ysparse, String filename, + byte[] ys, int ys_length, + int[] evt, int evt_length); + private native int ysparse_parse_buf( + long ysparse, String filename, + ByteBuffer ys, int ys_length, + IntBuffer evt, int evt_length); private final long ysparse; - public Rapidyaml() - { - String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; - System.loadLibrary(library_name); + + + // XXX this 'main' is for testing + public static void main( + String[] args + ) throws Exception, IOException { + (new Rapidyaml()).timingEnabled(true); + System.out.printf("It works!\n"); + } + + + + //------------------------ + // CTOR/DTOR + //------------------------ + + public Rapidyaml() throws Exception, IOException { + if (System.getenv("YS_RAPIDYAML_MAVEN_TEST") != null) + System.loadLibrary(RAPIDYAML_NAME); + else + NativeLibLoader.loadLibraryFromResource(RAPIDYAML_LIBNAME); + this.ysparse = this.ysparse_init(); - // TODO: receive this argument as ctor parameter - timingEnabled(System.getenv("YS_RYML_TIMER") != null); + timingEnabled(false); } // Likely bad idea to implement finalize: // // https://stackoverflow.com/questions/158174/why-would-you-ever-implement-finalize // - protected void finalize() throws Throwable - { - try { - this.ysparse_destroy(this.ysparse); - } - finally { - super.finalize(); - } - } +// protected void finalize() throws Throwable { +// try { +// this.ysparse_destroy(this.ysparse); +// } +// finally { +// super.finalize(); +// } +// } //------------------------ // EVT //------------------------ - public int parseYsToEvt(byte[] src, int[] evts) throws Exception - { + public int parseYsToEvt( + byte[] src, int[] evts + ) throws Exception { return parseYsToEvt("yamlscript", src, evts); } - public int parseYsToEvtBuf(ByteBuffer src, IntBuffer evt) throws Exception - { + public int parseYsToEvtBuf( + ByteBuffer src, IntBuffer evt + ) throws Exception { return parseYsToEvtBuf("yamlscript", src, evt); } - public int parseYsToEvt(String filename, byte[] src, int[] evts) throws Exception - { + public int parseYsToEvt( + String filename, byte[] src, int[] evts + ) throws Exception { long t = timingStart("ysparse"); - int required_size = ysparse_parse(this.ysparse, filename, src, src.length, evts, evts.length); + int required_size = ysparse_parse( + this.ysparse, filename, + src, src.length, evts, evts.length); timingStop("ysparse", t, src.length); return required_size; } - public int parseYsToEvtBuf(String filename, ByteBuffer src, IntBuffer evt) throws Exception - { - if(!src.isDirect()) + public int parseYsToEvtBuf( + String filename, ByteBuffer src, IntBuffer evt + ) throws Exception { + if (! src.isDirect()) throw new RuntimeException("src must be direct"); - if(!evt.isDirect()) + if (! evt.isDirect()) throw new RuntimeException("evt must be direct"); // the byte order for src does not matter // but for evt it really does - if(evt.order() != ByteOrder.nativeOrder()) + if (evt.order() != ByteOrder.nativeOrder()) throw new RuntimeException("evt byte order must be native"); long t = timingStart("ysparseBuf"); evt.position(evt.capacity()); - int reqsize = ysparse_parse_buf(this.ysparse, filename, src, src.position(), evt, evt.capacity()); - if(reqsize <= evt.capacity()) { + int reqsize = ysparse_parse_buf( + this.ysparse, filename, + src, src.position(), evt, evt.capacity()); + if (reqsize <= evt.capacity()) evt.position(reqsize); - } timingStop("ysparseBuf", t, src.position()); return reqsize; } - public static IntBuffer mkIntBuffer(int numInts) - { + public static IntBuffer mkIntBuffer(int numInts) { ByteBuffer bb = ByteBuffer.allocateDirect(/*numBytes*/4 * numInts); // !!! need to explicitly set the byte order to the native order return bb.order(ByteOrder.nativeOrder()).asIntBuffer(); } - //------------------------ // TIME //------------------------ private boolean showTiming = true; - public void timingEnabled(boolean yes) - { + public void timingEnabled(boolean yes) { showTiming = yes; ysparse_timing_set(yes); } - private long timingStart(String name) - { + private long timingStart(String name) { if(showTiming) { System.out.printf(" java:%s...\n", name); return System.nanoTime(); @@ -121,21 +142,22 @@ private long timingStart(String name) return 0; } - private void timingStop(String name, long t) - { + private void timingStop(String name, long t) { if(showTiming) { t = System.nanoTime() - t; - System.out.printf(" java:%s: %.6fms\n", name, (float)t/1.e6f); + System.out.printf( + " java:%s: %.6fms\n", name, (float) t/1.e6f); } } - private void timingStop(String name, long t, int numBytes) - { + private void timingStop(String name, long t, int numBytes) { if(showTiming) { t = System.nanoTime() - t; float dt = (float)t; float fb = (float)numBytes; - System.out.printf(" java:%s: %.6fms %.3fMB/s %dB\n", name, dt/1.e6f, 1.e3f*fb/dt, numBytes); + System.out.printf( + " java:%s: %.6fms %.3fMB/s %dB\n", + name, dt/1.e6f, 1.e3f*fb/dt, numBytes); } } } diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index 7a2af04d7..a5117d676 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -1,6 +1,6 @@ package org.rapidyaml; -import org.rapidyaml.*; +import org.rapidyaml.Rapidyaml; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; @@ -36,6 +36,8 @@ public static Test suite() return new TestSuite(RapidyamlTest.class); } + public void testPass() { } + public void testPlainMap() { String ys = "a: 1"; @@ -68,6 +70,22 @@ public void testUtf8() testEvt_(ys, expected); } + public void testUtf8_2() + { + String ys = "star: ★"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "star"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "★"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + public void testTaggedInt() { String ys = "- !!int 42"; @@ -100,6 +118,24 @@ public void testTaggedSeq() }; testEvt_(ys, expected); } + + public void testDocTag() + { + String ys = "!yamlscript/v0/bare\n--- !code\n42\n"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.TAG_, 1, 18, "yamlscript/v0/bare"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 0, 0, ""), + mkev(Evt.EDOC), + mkev(Evt.BDOC|Evt.EXPL), + mkev(Evt.VAL_|Evt.TAG_, 25, 4, "code"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 30, 2, "42"), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } public void testLargeCase() { @@ -164,7 +200,7 @@ public void testLargeCase() }; testEvt_(ys, expected); } - + public void testFilterCase() { String ys = "" + @@ -202,11 +238,11 @@ public void testFilterCase() }; testEvt_(ys, expected); } - + public void testFailure() throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); - String ys = ": : : :"; + String ys = "{a: b"; byte[] src = ys.getBytes(StandardCharsets.UTF_8); byte[] srcbuf = new byte[src.length]; boolean gotit = false; @@ -215,9 +251,9 @@ public void testFailure() throws Exception } catch(YamlParseErrorException e) { gotit = true; - assertEquals(2, e.offset); + assertEquals(5, e.offset); assertEquals(1, e.line); - assertEquals(3, e.column); + assertEquals(6, e.column); assertTrue(e.getMessage() != null); assertFalse(e.getMessage().isEmpty()); } @@ -229,11 +265,11 @@ public void testFailure() throws Exception } assertTrue(gotit); } - + public void testFailureBuf() throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); - String ys = ": : : :"; + String ys = "{a: b"; byte[] src = ys.getBytes(StandardCharsets.UTF_8); ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); bbuf.put(src); @@ -243,9 +279,9 @@ public void testFailureBuf() throws Exception } catch(YamlParseErrorException e) { gotit = true; - assertEquals(2, e.offset); + assertEquals(5, e.offset); assertEquals(1, e.line); - assertEquals(3, e.column); + assertEquals(6, e.column); assertTrue(e.getMessage() != null); assertFalse(e.getMessage().isEmpty()); } @@ -300,7 +336,7 @@ private void testEvt_(String ys, ExpectedEvent[] expected) } } - boolean dbglog = true; + boolean dbglog = false; private void cmpEvt_(String ys, byte[] src, int[] actual, ExpectedEvent[] expected) { if(dbglog) { diff --git a/ys/Makefile b/ys/Makefile index 778c0a231..251d48b65 100644 --- a/ys/Makefile +++ b/ys/Makefile @@ -12,7 +12,13 @@ test := test/ YS_BUILD_DEPS := \ $(YAMLSCRIPT_CLI_BIN) \ $(YAMLSCRIPT_CLI_BIN_BASH) \ + $(COMMON/base.mk) \ + $(COMMON/clojure.mk) \ + $(COMMON/docker.mk) \ + $(COMMON/native.mk) \ + $(lastword $(MAKEFILE_LIST)) \ +ifdef YS_NATIVE_BUILD_STATIC ifeq (true,$(IS_LINUX)) ifeq (true,$(IS_INTEL)) YS_BUILD_DEPS := $(MUSL_GCC) $(YS_BUILD_DEPS) @@ -22,6 +28,7 @@ NATIVE_OPTS += \ --libc=musl endif endif +endif #------------------------------------------------------------------------------ @@ -37,7 +44,12 @@ install: $(CLI_DEPS) install -m 755 $(CLI_BIN_BASH) \ $(PREFIX)/bin/ +<<<<<<< HEAD # jar: $(CLI_JAR) +======= +jar: $(YAMLSCRIPT_CLI_JAR_PATH) + @$(MAKE) -s -C $(RAPIDYAML) $(RAPIDYAML_INSTALLED) +>>>>>>> a27837f5 (rapidyaml: Working with native-image) test: test-run From 54b6993b788e6e3aa9dac9cc8b09db6e19c63ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sun, 23 Mar 2025 21:45:36 -0700 Subject: [PATCH 19/29] rapidyaml: Makefile major refactoring --- common/binding.mk | 1 + common/vars-core.mk | 4 +- common/vars-rapidyaml.mk | 26 +++++++ common/vars.mk | 28 ------- core/Makefile | 9 ++- libyamlscript/Makefile | 4 - mvp/Makefile | 20 ----- mvp/pom.xml | 75 ------------------- mvp/src/main/java/com/example/App.java | 12 --- mvp/src/test/java/com/example/AppTest.java | 13 ---- rapidyaml/Makefile | 17 +++-- rapidyaml/native/Makefile | 1 + .../main/java/org/rapidyaml/Rapidyaml.java | 2 +- ys/Makefile | 28 +------ 14 files changed, 50 insertions(+), 190 deletions(-) create mode 100644 common/vars-rapidyaml.mk delete mode 100644 mvp/Makefile delete mode 100644 mvp/pom.xml delete mode 100644 mvp/src/main/java/com/example/App.java delete mode 100644 mvp/src/test/java/com/example/AppTest.java diff --git a/common/binding.mk b/common/binding.mk index cc8cd1f64..5a6c1a7ab 100644 --- a/common/binding.mk +++ b/common/binding.mk @@ -1,3 +1,4 @@ +include $(COMMON)/vars-rapidyaml.mk include $(COMMON)/vars-libys.mk test:: $(LIBYS_SO_FQNP) diff --git a/common/vars-core.mk b/common/vars-core.mk index c89b28902..f96fbd848 100644 --- a/common/vars-core.mk +++ b/common/vars-core.mk @@ -1,6 +1,8 @@ +include $(COMMON)/vars-rapidyaml.mk + CORE_DIR := $(ROOT)/core CORE_JAR := $(CORE_DIR)/target/core-$(YAMLSCRIPT_VERSION)-standalone.jar -CORE_DEPS := $(LEIN) +CORE_DEPS := $(LEIN) $(RAPIDYAML_INSTALLED) CORE_INSTALLED := \ $(MAVEN_REPOSITORY)/yamlscript/core/$(YAMLSCRIPT_VERSION) diff --git a/common/vars-rapidyaml.mk b/common/vars-rapidyaml.mk new file mode 100644 index 000000000..33ea7d11c --- /dev/null +++ b/common/vars-rapidyaml.mk @@ -0,0 +1,26 @@ +RAPIDYAML_DIR := $(ROOT)/rapidyaml +NATIVE_DIR := $(RAPIDYAML_DIR)/native + +export LD_LIBRARY_PATH := $(NATIVE_DIR):$(LD_LIBRARY_PATH) +export $(DY)LD_LIBRARY_PATH := $(LD_LIBRARY_PATH) + +RAPIDYAML_VERSION := 0.8.0 +RAPIDYAML_TAG ?= d3132a25ec21c65e27ff46ab8c3d61c72a321302 +RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_BUILD_TYPE ?= Release +RAPIDYAML_DBG ?= 0 +RAPIDYAML_TIMED ?= 1 +RAPIDYAML_JAVA := \ + $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/NativeLibLoader.java \ + $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/Evt.java \ + $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/YamlParseErrorException.java +RAPIDYAML_JNI_H := $(NATIVE_DIR)/org_rapidyaml_Rapidyaml.h +RAPIDYAML_SO_NAME := librapidyaml.$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_SO := $(NATIVE_DIR)/$(RAPIDYAML_SO_NAME) +RAPIDYAML_LIB := $(NATIVE_DIR)/librapidyaml.$(DOTLIB) +RAPIDYAML_JAR := $(RAPIDYAML_DIR)/target/rapidyaml-$(RAPIDYAML_VERSION).jar +RAPIDYAML_INSTALLED := \ + $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION) +RAPIDYAML_INSTALLED := \ + $(RAPIDYAML_INSTALLED)/rapidyaml-$(RAPIDYAML_VERSION).jar diff --git a/common/vars.mk b/common/vars.mk index 27e80c42b..e6eb3fd33 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -216,34 +216,6 @@ RELEASE_LYS_NAME := libyamlscript-$(YS_VERSION)-$(GRAALVM_ARCH) RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz -#------------------------------------------------------------------------------ -# RapidYAML variables: -#------------------------------------------------------------------------------ -RAPIDYAML := $(ROOT)/rapidyaml - -RAPIDYAML_VERSION := 0.8.0 -#RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) -RAPIDYAML_TAG ?= d3132a25ec21c65e27ff46ab8c3d61c72a321302 -RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml -RAPIDYAML_BUILD_TYPE ?= Release -RAPIDYAML_DBG ?= 0 -RAPIDYAML_TIMED ?= 1 -RAPIDYAML_JAVA := \ - $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ - $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java \ - $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \ - $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java -RAPIDYAML_JNI_H := $(ROOT)/rapidyaml/native/org_rapidyaml_Rapidyaml.h -RAPIDYAML_SO_NAME := librapidyaml.$(RAPIDYAML_VERSION).$(SO) -RAPIDYAML_SO := $(ROOT)/rapidyaml/native/$(RAPIDYAML_SO_NAME) -RAPIDYAML_LIB := $(ROOT)/rapidyaml/native/librapidyaml.$(DOTLIB) -RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar -RAPIDYAML_INSTALLED_DIR := \ - $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION) -RAPIDYAML_INSTALLED := \ - $(RAPIDYAML_INSTALLED_DIR)/rapidyaml-$(RAPIDYAML_VERSION).jar - - #------------------------------------------------------------------------------ # Programs #------------------------------------------------------------------------------ diff --git a/core/Makefile b/core/Makefile index c001566df..fe22a1403 100644 --- a/core/Makefile +++ b/core/Makefile @@ -2,18 +2,20 @@ include ../common/base.mk include $(COMMON)/java.mk include $(COMMON)/clojure.mk include $(COMMON)/vars-core.mk +include $(COMMON)/vars-rapidyaml.mk export PATH := $(ROOT)/core/bin:$(PATH) #------------------------------------------------------------------------------ -build:: $(RAPIDYAML_SO) $(RAPIDYAML_INSTALLED) test:: $(CORE_DEPS) $(LEIN) $@ install: $(CORE_INSTALLED) -$(CORE_INSTALLED): $(CORE_JAR) +repl-deps:: $(RAPIDYAML_INSTALLED) + +$(CORE_INSTALLED): $(CORE_JAR) $(RAPIDYAML_INSTALLED) $(LEIN) install touch $@ @@ -21,6 +23,9 @@ $(CORE_JAR): $(CORE_DEPS) $(LEIN) uberjar touch $@ +$(RAPIDYAML_INSTALLED): + $(MAKE) -C $(RAPIDYAML_DIR) $@ + clean:: $(RM) pom.xml $(RM) -r target/ diff --git a/libyamlscript/Makefile b/libyamlscript/Makefile index 34c577ca4..46e66453e 100644 --- a/libyamlscript/Makefile +++ b/libyamlscript/Makefile @@ -8,11 +8,7 @@ include $(COMMON)/vars-libys.mk #------------------------------------------------------------------------------ build:: $(LIBYS_DEPS) -<<<<<<< HEAD # jar: $(LIBYS_JAR_PATH) -======= -jar: $(LIBYAMLSCRIPT_JAR_PATH) $(RAPIDYAML_INSTALLED) ->>>>>>> 8a8f78d0 (core: Add rapidyaml parser support) install:: $(LIBYS_DEPS) mkdir -p $(PREFIX)/include/ diff --git a/mvp/Makefile b/mvp/Makefile deleted file mode 100644 index 50d26712b..000000000 --- a/mvp/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -include ../common/base.mk -include $(COMMON)/java.mk - -JAVA_FILE := src/main/java/com/example/App.java -JAR_FILE := target/my-app-1.0-SNAPSHOT.jar - - -default:: - which mvn - -build:: $(JAR_FILE) - -$(JAR_FILE): $(JAVA_FILE) - mvn clean package - -run:: build - java -jar target/my-app-1.0-SNAPSHOT.jar - -test clean package:: - mvn $@ diff --git a/mvp/pom.xml b/mvp/pom.xml deleted file mode 100644 index 9ef65ac16..000000000 --- a/mvp/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 4.0.0 - - com.example - my-app - 1.0-SNAPSHOT - - - 11 - 11 - UTF-8 - - - - - - - - junit - junit - 4.13.2 - test - - - - org.rapidyaml - rapidyaml - 0.8.0 - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.2.0 - - - - true - com.example.App - - - - - - - - - diff --git a/mvp/src/main/java/com/example/App.java b/mvp/src/main/java/com/example/App.java deleted file mode 100644 index 2714e45e8..000000000 --- a/mvp/src/main/java/com/example/App.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.example; - -// import org.apache.commons.lang3.StringUtils; -import org.rapidyaml.Rapidyaml; - -public class App { - public static void main(String[] args) { - String text = "Hello, World!"; - System.out.println("Original text: " + text); - // System.out.println("Reversed text: " + StringUtils.reverse(text)); - } -} diff --git a/mvp/src/test/java/com/example/AppTest.java b/mvp/src/test/java/com/example/AppTest.java deleted file mode 100644 index 0a7ec0b34..000000000 --- a/mvp/src/test/java/com/example/AppTest.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.example; - -import org.junit.Test; -import static org.junit.Assert.*; - -public class AppTest { - @Test - public void testApp() { - // Simple test to verify the App class can be instantiated - App app = new App(); - assertNotNull(app); - } -} diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index 31f5d09c3..f1b79ff19 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -1,10 +1,11 @@ include ../common/base.mk include $(COMMON)/clojure.mk include $(COMMON)/java.mk +include $(COMMON)/vars-rapidyaml.mk RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class) RAPIDYAML_CLASSES := \ - $(RAPIDYAML_CLASS:$(RAPIDYAML)/src/main/java/%=-C src/main/java %) + $(RAPIDYAML_CLASS:$(ROOT)/rapidyaml/src/main/java/%=-C src/main/java %) #------------------------------------------------------------------------------ @@ -18,15 +19,15 @@ jar: $(RAPIDYAML_JAR) jar-test: $(RAPIDYAML_JAR) java -jar $< -install:: $(RAPIDYAML_INSTALLED) +# install:: $(RAPIDYAML_INSTALLED) test:: build $(JAVA_INSTALLED) $(MAKE) -C native $@ - YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) $@ + YS_TESTING=1 $(MVN) $@ test-x: build $(JAVA_INSTALLED) $(MAKE) -C native test - YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) -X -e test + YS_TESTING=1 $(MVN) -X -e test clean:: $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @@ -52,16 +53,16 @@ $(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ $(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) -ifdef YS_MAVEN_TEST - YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) install -else +ifdef YS_MAVEN_TEST_SKIP $(MVN) install -Dmaven.test.skip +else + YS_TESTING=1 $(MVN) install endif touch $@ $(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(JAR) -cf $@ $(RAPIDYAML_CLASSES) - $(JAR) -uf $@ -C $(RAPIDYAML)/native/ $(RAPIDYAML_SO_NAME) + $(JAR) -uf $@ -C $(ROOT)/rapidyaml/native/ $(RAPIDYAML_SO_NAME) $(JAR) -umf manifest.txt $@ $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index e656be03e..b2b5a387f 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -2,6 +2,7 @@ include ../../common/base.mk include $(COMMON)/clojure.mk include $(COMMON)/java.mk include $(COMMON)/python.mk +include $(COMMON)/vars-rapidyaml.mk # TODO change to static library! # https://www.graalvm.org/latest/reference-manual/native-image/guides/build-static-executables/ diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 1d16a8893..85502f140 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -45,7 +45,7 @@ public static void main( //------------------------ public Rapidyaml() throws Exception, IOException { - if (System.getenv("YS_RAPIDYAML_MAVEN_TEST") != null) + if (System.getenv("YS_TESTING") != null) System.loadLibrary(RAPIDYAML_NAME); else NativeLibLoader.loadLibraryFromResource(RAPIDYAML_LIBNAME); diff --git a/ys/Makefile b/ys/Makefile index 251d48b65..511fe0a8e 100644 --- a/ys/Makefile +++ b/ys/Makefile @@ -9,27 +9,6 @@ BPAN_REPO_URL := https://github.com/bpan-org/bpan test := test/ -YS_BUILD_DEPS := \ - $(YAMLSCRIPT_CLI_BIN) \ - $(YAMLSCRIPT_CLI_BIN_BASH) \ - $(COMMON/base.mk) \ - $(COMMON/clojure.mk) \ - $(COMMON/docker.mk) \ - $(COMMON/native.mk) \ - $(lastword $(MAKEFILE_LIST)) \ - -ifdef YS_NATIVE_BUILD_STATIC -ifeq (true,$(IS_LINUX)) -ifeq (true,$(IS_INTEL)) -YS_BUILD_DEPS := $(MUSL_GCC) $(YS_BUILD_DEPS) -NATIVE_OPTS += \ - -H:CCompilerOption=-Wl,-z,stack-size=2097152 \ - --static \ - --libc=musl -endif -endif -endif - #------------------------------------------------------------------------------ build:: $(CLI_DEPS) @@ -44,12 +23,7 @@ install: $(CLI_DEPS) install -m 755 $(CLI_BIN_BASH) \ $(PREFIX)/bin/ -<<<<<<< HEAD # jar: $(CLI_JAR) -======= -jar: $(YAMLSCRIPT_CLI_JAR_PATH) - @$(MAKE) -s -C $(RAPIDYAML) $(RAPIDYAML_INSTALLED) ->>>>>>> a27837f5 (rapidyaml: Working with native-image) test: test-run @@ -96,6 +70,8 @@ $(CLI_BIN_BASH): $(CLI_BIN_BASH_SRC) $(CLI_JAR): $(CLI_JAR_DEPS) $(LEIN) uberjar + jar -uf $@ \ + -C $(RAPIDYAML_DIR)/native/ librapidyaml.0.8.0.so touch $@ $(CORE_INSTALLED): From 772fb1a8da7683084da670bbdf0fedb29f70e284 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 27 Mar 2025 00:08:48 +0000 Subject: [PATCH 20/29] POC: bytebuffer 2 slice 2 str --- .../java/org/rapidyaml/RapidyamlTest.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index a5117d676..1da786510 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -36,6 +36,54 @@ public static Test suite() return new TestSuite(RapidyamlTest.class); } + public void testByteBuffer2Slice2Str() + { + String str = "0123456789𝄞✅★"; + byte[] arr = str.getBytes(StandardCharsets.UTF_8); + ByteBuffer buf = ByteBuffer.allocateDirect(arr.length); + buf.put(arr); + assertEquals("012", StandardCharsets.UTF_8.decode(buf.slice(0, 3)).toString()); + assertEquals("123", StandardCharsets.UTF_8.decode(buf.slice(1, 3)).toString()); + assertEquals("234", StandardCharsets.UTF_8.decode(buf.slice(2, 3)).toString()); + assertEquals("345", StandardCharsets.UTF_8.decode(buf.slice(3, 3)).toString()); + assertEquals("456", StandardCharsets.UTF_8.decode(buf.slice(4, 3)).toString()); + assertEquals("0123", StandardCharsets.UTF_8.decode(buf.slice(0, 4)).toString()); + assertEquals("1234", StandardCharsets.UTF_8.decode(buf.slice(1, 4)).toString()); + assertEquals("2345", StandardCharsets.UTF_8.decode(buf.slice(2, 4)).toString()); + assertEquals("3456", StandardCharsets.UTF_8.decode(buf.slice(3, 4)).toString()); + assertEquals("4567", StandardCharsets.UTF_8.decode(buf.slice(4, 4)).toString()); + assertEquals("01234", StandardCharsets.UTF_8.decode(buf.slice(0, 5)).toString()); + assertEquals("12345", StandardCharsets.UTF_8.decode(buf.slice(1, 5)).toString()); + assertEquals("23456", StandardCharsets.UTF_8.decode(buf.slice(2, 5)).toString()); + assertEquals("34567", StandardCharsets.UTF_8.decode(buf.slice(3, 5)).toString()); + assertEquals("45678", StandardCharsets.UTF_8.decode(buf.slice(4, 5)).toString()); + assertEquals("𝄞", StandardCharsets.UTF_8.decode(buf.slice(10, 4)).toString()); + assertEquals("𝄞✅", StandardCharsets.UTF_8.decode(buf.slice(10, 7)).toString()); + assertEquals("𝄞✅★", StandardCharsets.UTF_8.decode(buf.slice(10, 10)).toString()); + assertEquals("✅", StandardCharsets.UTF_8.decode(buf.slice(14, 3)).toString()); + assertEquals("✅★", StandardCharsets.UTF_8.decode(buf.slice(14, 6)).toString()); + assert("012".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(0, 3)))); + assert("123".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(1, 3)))); + assert("234".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(2, 3)))); + assert("345".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(3, 3)))); + assert("456".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(4, 3)))); + assert("0123".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(0, 4)))); + assert("1234".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(1, 4)))); + assert("2345".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(2, 4)))); + assert("3456".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(3, 4)))); + assert("4567".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(4, 4)))); + assert("01234".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(0, 5)))); + assert("12345".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(1, 5)))); + assert("23456".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(2, 5)))); + assert("34567".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(3, 5)))); + assert("45678".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(4, 5)))); + assert("𝄞".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(10, 4)))); + assert("𝄞✅".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(10, 7)))); + assert("𝄞✅★".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(10, 10)))); + assert("✅".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(14, 3)))); + assert("✅★".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(14, 6)))); + } + public void testPass() { } public void testPlainMap() From 87959095dbf650ba4a92d3d6a86357647fbacc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Thu, 27 Mar 2025 10:10:07 -0700 Subject: [PATCH 21/29] small ryml refactors --- core/src/yamlscript/parser.clj | 8 ++++---- util/YS | 2 +- ys/Makefile | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 9f1e36b2a..7a3b16c8a 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -185,7 +185,7 @@ Evt/FOLD :> nil)) -(defn parse-rapidyaml [^String yaml-string] +(defn parse-rapidyaml-arr [^String yaml-string] (rest (let [parser ^Rapidyaml (new Rapidyaml) _ (when (TIMER) @@ -253,6 +253,7 @@ get-str (fn [i] (let [off (.get masks ^Long (inc i)) len (.get masks ^Long (+ i 2))] + ;; need string not bytes (reduce (fn [slice i] (str slice (char (.get srcbuffer ^Long i)))) @@ -289,15 +290,14 @@ (recur i tag anchor events))) events))))) + (defn parse-fn [] (if-let [parser-name (System/getenv "YS_PARSER")] (condp = parser-name "" parse-snakeyaml "snake" parse-snakeyaml - "rapid" parse-rapidyaml + "rapid-arr" parse-rapidyaml-arr "rapid-buf" parse-rapidyaml-buf - "ryml" parse-rapidyaml - "ryml-buf" parse-rapidyaml-buf (die "Unknown YS_PARSER value: " parser-name)) parse-snakeyaml)) diff --git a/util/YS b/util/YS index 9e7184ca3..53eff8a5f 100755 --- a/util/YS +++ b/util/YS @@ -10,6 +10,6 @@ set -euo pipefail export PATH=$JAVA_HOME/bin:$PATH root_ys=$root/ys jar=yamlscript.cli-$libyamlscript_version-SNAPSHOT-standalone.jar - make --no-print-directory -C "$root_ys" jar + make --quiet --no-print-directory -C "$root_ys" jar java -jar "$root_ys/target/uberjar/$jar" "$@" ) diff --git a/ys/Makefile b/ys/Makefile index 511fe0a8e..a2e6537b6 100644 --- a/ys/Makefile +++ b/ys/Makefile @@ -23,7 +23,7 @@ install: $(CLI_DEPS) install -m 755 $(CLI_BIN_BASH) \ $(PREFIX)/bin/ -# jar: $(CLI_JAR) +jar: $(CLI_JAR) test: test-run From 5ef0dd857b707ba52673356207e1f13b7b726ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Thu, 27 Mar 2025 10:44:37 -0700 Subject: [PATCH 22/29] ryml: everything works (on linux) --- core/src/yamlscript/parser.clj | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 7a3b16c8a..a002d4c2e 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -201,9 +201,7 @@ get-str (fn [i] (let [off (aget masks (inc i)) len (aget masks (+ i 2))] - (reduce - (fn [slice i] (str slice (char (aget buffer i)))) - "" (range off (+ off len)))))] + (String. buffer off len StandardCharsets/UTF_8)))] (loop [i 0, tag nil, anchor nil, events []] (if (< i needed) @@ -253,11 +251,9 @@ get-str (fn [i] (let [off (.get masks ^Long (inc i)) len (.get masks ^Long (+ i 2))] - ;; need string not bytes - (reduce - (fn [slice i] (str slice - (char (.get srcbuffer ^Long i)))) - "" (range off (+ off len)))))] + (.toString + (.decode StandardCharsets/UTF_8 + (.slice srcbuffer (int off) (int len))))))] (loop [i 0, tag nil, anchor nil, events []] (if (< i needed) From dc0e93e37a4416c816ac73964022adf75d8f90ad Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 27 Mar 2025 18:12:14 +0000 Subject: [PATCH 23/29] ryml: no need to upcast to ^Long for annotation --- core/src/yamlscript/parser.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index a002d4c2e..37aff31f4 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -249,11 +249,11 @@ masks (Rapidyaml/mkIntBuffer needed) _ (.parseYsToEvtBuf parser srcbuffer masks) get-str (fn [i] - (let [off (.get masks ^Long (inc i)) - len (.get masks ^Long (+ i 2))] + (let [off (.get masks ^int (inc i)) + len (.get masks ^int (+ i 2))] (.toString (.decode StandardCharsets/UTF_8 - (.slice srcbuffer (int off) (int len))))))] + (.slice srcbuffer off len)))))] (loop [i 0, tag nil, anchor nil, events []] (if (< i needed) From 86950cdb16b1a9498e84fa4dfd19863cf74c3cf3 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 27 Mar 2025 18:20:38 +0000 Subject: [PATCH 24/29] ryml: set 'rapid' as default for rapidyaml-buf --- core/src/yamlscript/parser.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 37aff31f4..4eabd34a1 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -292,8 +292,9 @@ (condp = parser-name "" parse-snakeyaml "snake" parse-snakeyaml - "rapid-arr" parse-rapidyaml-arr + "rapid" parse-rapidyaml-buf "rapid-buf" parse-rapidyaml-buf + "rapid-arr" parse-rapidyaml-arr (die "Unknown YS_PARSER value: " parser-name)) parse-snakeyaml)) From c4ff3f6856a94590f7c0e3cbf4e56f81e611510d Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 27 Mar 2025 18:18:56 +0000 Subject: [PATCH 25/29] ryml: add github workflows --- .github/workflows/rapidyaml.yml | 107 ++++++++++++++++++++++++++++++++ .github/workflows/ys.yml | 65 +++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 .github/workflows/rapidyaml.yml create mode 100644 .github/workflows/ys.yml diff --git a/.github/workflows/rapidyaml.yml b/.github/workflows/rapidyaml.yml new file mode 100644 index 000000000..b89fd865c --- /dev/null +++ b/.github/workflows/rapidyaml.yml @@ -0,0 +1,107 @@ +name: rapidyaml + +defaults: + run: + shell: bash -xeo pipefail {0} +'on': + workflow_dispatch: null + push: + branches: + - main + pull_request: + +jobs: + + # check that the spec'ed version of rapidyaml passes its own tests + ryml: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - bt: Debug + - bt: Release + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: checkout rapidyaml + run: | + cd rapidyaml/native + make rapidyaml + - name: configure + run: | + cd rapidyaml/native/rapidyaml + cmake -B build -D CMAKE_BUILD_TYPE=${{matrix.bt}} -D RYML_BUILD_TESTS=ON + - name: build + run: | + cd rapidyaml/native/rapidyaml + cmake --build build --target ryml-test-build --parallel --verbose + - name: run tests + run: | + cd rapidyaml/native/rapidyaml + cmake --build build --target ryml-test-run + + # run the c++ tests, also in Debug to test with assertions + cpp: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - bt: Debug + - bt: Release + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: check jni header up to date + run: make -C rapidyaml/native -B jni jnicheck + - name: get rapidyaml + run: make -C rapidyaml/native rapidyaml + - name: run c++ tests, static ----------------------------- + run: echo + - name: cfg c++, static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-static + - name: build c++ tests, static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-static + - name: run c++ tests, static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static + - name: run c++ tests, static with timing + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static-timing + - name: run c++ tests, shared ----------------------------- + run: echo + - name: cfg c++, shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-shared + - name: build c++ tests, shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-shared + - name: run c++ tests, shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared + - name: run c++ tests, shared with timing + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared-timing + + # run the java tests, also in Debug to test with assertions + java: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - bt: Debug + - bt: Release + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: build lib + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml build + - name: build jar + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml jar + - name: run java tests + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml test diff --git a/.github/workflows/ys.yml b/.github/workflows/ys.yml new file mode 100644 index 000000000..ff35b0bc2 --- /dev/null +++ b/.github/workflows/ys.yml @@ -0,0 +1,65 @@ +name: ys + +defaults: + run: + shell: bash -xeo pipefail {0} +'on': + workflow_dispatch: null + push: + branches: + - main + pull_request: + +jobs: + + # run ys tests, also in Debug to test with assertions. + # TODO: add a job to do all the downloads and build graalvm, and + # then make the other jobs depend on that + ys: + name: ys/${{matrix.ysparser}}/${{matrix.bt}} + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - {v: 1, ysparser: snake} + - {v: 1, ysparser: rapid-arr, bt: Debug} + - {v: 1, ysparser: rapid-arr, bt: Release} + - {v: 1, ysparser: rapid-buf, bt: Debug} + - {v: 1, ysparser: rapid-buf, bt: Release} + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: run core tests + run: | + . .profile + make test-core \ + v=${{matrix.v}} \ + YS_TESTING=1 \ + YS_PARSER=${{matrix.ysparser}} \ + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} + - name: run ys/test-run + run: | + . .profile + make -C ys test-run \ + v=${{matrix.v}} \ + YS_PARSER=${{matrix.ysparser}} \ + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} + - name: run test-ys + run: | + . .profile + make test-ys \ + v=${{matrix.v}} \ + YS_PARSER=${{matrix.ysparser}} \ + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} + - name: run all tests + run: | + . .profile + make test \ + v=${{matrix.v}} \ + YS_TESTING=1 \ + YS_PARSER=${{matrix.ysparser}} \ + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} From fa5d9493e447490e59750d234be4cce4d6716d33 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 27 Sep 2025 14:45:51 +0100 Subject: [PATCH 26/29] int events from ryml WIP: update c++ code --- common/vars-rapidyaml.mk | 10 +- rapidyaml/native/CMakeLists.txt | 47 +- rapidyaml/native/Makefile | 16 +- rapidyaml/native/org_rapidyaml_Rapidyaml.cpp | 22 +- rapidyaml/native/org_rapidyaml_Rapidyaml.h | 28 +- rapidyaml/native/ysparse_common.hpp | 1 - rapidyaml/native/ysparse_evt.cpp | 25 +- rapidyaml/native/ysparse_evt.hpp | 51 +- rapidyaml/native/ysparse_evt_handler.cpp | 12 - rapidyaml/native/ysparse_evt_handler.hpp | 725 ------------------- rapidyaml/native/ysparse_test.cpp | 398 +++++----- 11 files changed, 342 insertions(+), 993 deletions(-) delete mode 100644 rapidyaml/native/ysparse_evt_handler.cpp delete mode 100644 rapidyaml/native/ysparse_evt_handler.hpp diff --git a/common/vars-rapidyaml.mk b/common/vars-rapidyaml.mk index 33ea7d11c..43d3b2cde 100644 --- a/common/vars-rapidyaml.mk +++ b/common/vars-rapidyaml.mk @@ -4,8 +4,8 @@ NATIVE_DIR := $(RAPIDYAML_DIR)/native export LD_LIBRARY_PATH := $(NATIVE_DIR):$(LD_LIBRARY_PATH) export $(DY)LD_LIBRARY_PATH := $(LD_LIBRARY_PATH) -RAPIDYAML_VERSION := 0.8.0 -RAPIDYAML_TAG ?= d3132a25ec21c65e27ff46ab8c3d61c72a321302 +RAPIDYAML_VERSION := 0.10.0 +RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml RAPIDYAML_BUILD_TYPE ?= Release RAPIDYAML_DBG ?= 0 @@ -15,10 +15,12 @@ RAPIDYAML_JAVA := \ $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/NativeLibLoader.java \ $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/Evt.java \ $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/YamlParseErrorException.java +RAPIDYAML_NAME := ysparse +RAPIDYAML_LIBNAME := lib$(RAPIDYAML_NAME) RAPIDYAML_JNI_H := $(NATIVE_DIR)/org_rapidyaml_Rapidyaml.h -RAPIDYAML_SO_NAME := librapidyaml.$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_SO_NAME := $(RAPIDYAML_LIBNAME).$(RAPIDYAML_VERSION).$(SO) RAPIDYAML_SO := $(NATIVE_DIR)/$(RAPIDYAML_SO_NAME) -RAPIDYAML_LIB := $(NATIVE_DIR)/librapidyaml.$(DOTLIB) +RAPIDYAML_LIB := $(NATIVE_DIR)/$(RAPIDYAML_LIBNAME).$(DOTLIB) RAPIDYAML_JAR := $(RAPIDYAML_DIR)/target/rapidyaml-$(RAPIDYAML_VERSION).jar RAPIDYAML_INSTALLED := \ $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION) diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 968d7cb9d..e70ae2a68 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -13,9 +13,8 @@ if(UNIX) set(CMAKE_SHARED_LIBRARY_SUFFIX .so) endif() -set(libname rapidyaml) # TODO rename to ysparse -add_library(${libname} +add_library(ysparse # # JNI bridge org_rapidyaml_Rapidyaml.h @@ -23,49 +22,59 @@ add_library(${libname} # # ysparse files ysparse_common.hpp - ysparse_evt_handler.hpp - ysparse_evt_handler.cpp ysparse_evt.hpp ysparse_evt.cpp # # files required from rapidyaml + rapidyaml/src/c4/yml/common.hpp rapidyaml/src/c4/yml/common.cpp + rapidyaml/src/c4/yml/node_type.hpp rapidyaml/src/c4/yml/node_type.cpp - rapidyaml/src/c4/yml/parse.cpp - rapidyaml/src/c4/yml/tree.cpp rapidyaml/src/c4/yml/tag.cpp - rapidyaml/src/c4/yml/reference_resolver.cpp + rapidyaml/src_extra/c4/yml/extra/event_handler_ints.hpp + rapidyaml/src_extra/c4/yml/extra/event_handler_ints.cpp # # files required from rapidyaml/ext/c4core + rapidyaml/ext/c4core/src/c4/substr.hpp + rapidyaml/ext/c4core/src/c4/base64.hpp rapidyaml/ext/c4core/src/c4/base64.cpp + rapidyaml/ext/c4core/src/c4/error.hpp rapidyaml/ext/c4core/src/c4/error.cpp + rapidyaml/ext/c4core/src/c4/language.hpp rapidyaml/ext/c4core/src/c4/language.cpp + rapidyaml/ext/c4core/src/c4/utf.hpp rapidyaml/ext/c4core/src/c4/utf.cpp ) -target_include_directories(${libname} PUBLIC +target_include_directories(ysparse PUBLIC ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src_extra ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/ext/c4core/src ) -target_compile_definitions(${libname} PUBLIC +target_compile_definitions(ysparse PUBLIC RYML_WITH_TAB_TOKENS RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS $<$:YSPARSE_TIMED> $<$:RYML_DBG> ) -set_target_properties(${libname} PROPERTIES CXX_STANDARD 17) +set_target_properties(ysparse PROPERTIES CXX_STANDARD 17) -target_include_directories(${libname} PUBLIC ${JNI_INCLUDE_DIRS}) +target_include_directories(ysparse PUBLIC ${JNI_INCLUDE_DIRS}) -add_executable(${libname}-test ysparse_test.cpp) -target_link_libraries(${libname}-test ${libname}) -add_custom_target(${libname}-test-run - DEPENDS ${libname}-test - COMMAND $ +add_executable(ysparse-test + ysparse_test.cpp + rapidyaml/src_extra/c4/yml/extra/ints_utils.hpp + rapidyaml/src_extra/c4/yml/extra/ints_utils.cpp + rapidyaml/src_extra/c4/yml/extra/scalar.cpp +) +target_link_libraries(ysparse-test ysparse) +add_custom_target(ysparse-test-run + DEPENDS ysparse-test + COMMAND $ COMMENT "running C++ tests" ) -add_custom_target(${libname}-test-run-timing - DEPENDS ${libname}-test - COMMAND $ --timing +add_custom_target(ysparse-test-run-timing + DEPENDS ysparse-test + COMMAND $ --timing COMMENT "running C++ tests, with timing" ) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index b2b5a387f..e1d8ebafe 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -10,7 +10,7 @@ include $(COMMON)/vars-rapidyaml.mk # https://stackoverflow.com/questions/24493337/linking-static-library-with-jni THIS_DIR := $(shell pwd) -BDIR := $(THIS_DIR)/_build/$(RAPIDYAML_BUILD_TYPE)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) +BDIR := $(THIS_DIR)/build/$(RAPIDYAML_BUILD_TYPE)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) RAPIDYAML_DEPS := \ Makefile \ @@ -63,24 +63,24 @@ build-static: $(RAPIDYAML_LIB) cfg-static: rapidyaml $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF test-static: build-static - $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target $(RAPIDYAML_NAME)-test-run test-static-timing: build-static - $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target rapidyaml-test-run-timing + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target $(RAPIDYAML_NAME)-test-run-timing $(RAPIDYAML_LIB): cfg-static $(RAPIDYAML_DEPS) - $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --parallel --target rapidyaml + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --parallel --target $(RAPIDYAML_NAME) cp -fv $(BDIR)-static/*.a $@ build-shared: $(RAPIDYAML_SO) cfg-shared: rapidyaml $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON test-shared: build-shared - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target $(RAPIDYAML_NAME)-test-run test-shared-timing: build-shared - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target rapidyaml-test-run-timing + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target $(RAPIDYAML_NAME)-test-run-timing $(RAPIDYAML_SO): cfg-shared $(RAPIDYAML_DEPS) - $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --parallel --target rapidyaml + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --parallel --target $(RAPIDYAML_NAME) cp -fv $(BDIR)-shared/*.so $@ - ln -fs $@ librapidyaml.so + ln -fs $@ $(RAPIDYAML_LIBNAME).$(SO) $(RAPIDYAML_JNI_H): $(JAVAC) $(RAPIDYAML_JAVA) $(JAVAC) -h . $(RAPIDYAML_JAVA) # $^ doesn't work diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index d02df8fb4..47eeb2b0a 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -35,14 +35,17 @@ JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject, jlong obj, jstring jfilename, jbyteArray src, jint src_len, + jbyteArray arena, jint arena_len, jintArray dst, jint dst_len) { TIMED_SECTION("jni:ysparse", (size_type)src_len); + jbyte* arena_ = nullptr; jbyte* src_ = nullptr; int* dst_ = nullptr; const char *filename = nullptr; jboolean dst_is_copy = false; jboolean src_is_copy = false; + jboolean arena_is_copy = false; { TIMED_SECTION("jni:ysparse/get_jni", (size_type)src_len); // this is __S__L__O__W__ @@ -52,6 +55,10 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject, TIMED_SECTION("jni:ysparse/GetByteArray(src)"); src_ = env->GetByteArrayElements(src, &src_is_copy); } + { + TIMED_SECTION("jni:ysparse/GetByteArray(arena)"); + arena_ = env->GetByteArrayElements(arena, &arena_is_copy); + } { TIMED_SECTION("jni:ysparse/GetIntArray(dst)"); dst_ = env->GetIntArrayElements(dst, &dst_is_copy); @@ -68,6 +75,7 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject, { rc = ysparse_parse((ysparse*)obj, filename, (char*)src_, src_len, + (char*)arena_, arena_len, dst_, dst_len); } catch (YsParseError const& exc) @@ -86,6 +94,10 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject, TIMED_SECTION("jni:ysparse/ReleaseByteArray(src)"); env->ReleaseByteArrayElements(src, src_, 0); } + { + TIMED_SECTION("jni:ysparse/ReleaseByteArray(arena)"); + env->ReleaseByteArrayElements(arena, arena_, 0); + } { TIMED_SECTION("jni:ysparse/ReleaseIntArray(dst)"); env->ReleaseIntArrayElements(dst, dst_, 0); @@ -103,19 +115,24 @@ JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf(JNIEnv *env, jobject, jlong obj, jstring jfilename, jobject src, jint src_len, + jobject arena, jint arena_len, jobject dst, jint dst_len) { TIMED_SECTION("jni:ysparse_buf", (size_type)src_len); + char* arena_ = nullptr; char* src_ = nullptr; int* dst_ = nullptr; const char *filename = nullptr; { TIMED_SECTION("jni:ysparse_buf/get_jni", (size_type)src_len); src_ = (char*)env->GetDirectBufferAddress(src); + arena_ = (char*)env->GetDirectBufferAddress(arena); dst_ = (int*)env->GetDirectBufferAddress(dst); filename = env->GetStringUTFChars(jfilename, 0); if(!src_) throw_runtime_exception(env, "null pointer: src"); + if(!arena_) + throw_runtime_exception(env, "null pointer: arena"); if(!dst_) throw_runtime_exception(env, "null pointer: dst"); } @@ -123,7 +140,10 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf(JNIEnv *env, jobject, TIMED_SECTION("jni:ysparse_buf/parse", (size_type)src_len); try { - return ysparse_parse((ysparse*)obj, filename, src_, src_len, dst_, dst_len); + return ysparse_parse((ysparse*)obj, filename, + src_, src_len, + arena_, arena_len, + dst_, dst_len); } catch (YsParseError const& exc) { diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index ea8355bc0..9c84ba2bf 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -34,18 +34,34 @@ JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set /* * Class: org_rapidyaml_Rapidyaml * Method: ysparse_parse - * Signature: (JLjava/lang/String;[BI[II)I + * Signature: (JLjava/lang/String;[BI[BI[II)Z */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse - (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); +JNIEXPORT jboolean JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint, jintArray, jint); /* * Class: org_rapidyaml_Rapidyaml * Method: ysparse_parse_buf - * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/IntBuffer;I)I + * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/ByteBuffer;ILjava/nio/IntBuffer;I)Z */ -JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf - (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint); +JNIEXPORT jboolean JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf + (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint, jobject, jint); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_reqsize_evt + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1reqsize_1evt + (JNIEnv *, jobject, jlong); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_reqsize_arena + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1reqsize_1arena + (JNIEnv *, jobject, jlong); #ifdef __cplusplus } diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp index 48e4dc090..17f7bb7de 100644 --- a/rapidyaml/native/ysparse_common.hpp +++ b/rapidyaml/native/ysparse_common.hpp @@ -34,7 +34,6 @@ RYML_EXPORT void ysparse_timing_set(bool yes); #ifndef YSPARSE_TIMED #define TIMED_SECTION(...) -#error #else #include #include diff --git a/rapidyaml/native/ysparse_evt.cpp b/rapidyaml/native/ysparse_evt.cpp index 0c3497666..bce48e16e 100644 --- a/rapidyaml/native/ysparse_evt.cpp +++ b/rapidyaml/native/ysparse_evt.cpp @@ -38,24 +38,35 @@ RYML_EXPORT void ysparse_destroy(ysparse *obj) _RYML_CB_FREE(get_callbacks(), obj, ysparse, 1); } -RYML_EXPORT size_type ysparse_parse(ysparse *obj, - const char *filename, - char *ys, size_type ys_size, - evt::DataType *events, size_type evt_size) +RYML_EXPORT bool ysparse_parse(ysparse *obj, + const char *filename, + char *ys, size_type ys_size, + char *arena, size_type arena_size, + int *events, size_type evt_size) { TIMED_SECTION("cpp:ysparse", ys_size); csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; substr ys_(ys, (size_t)ys_size); + substr arena_(arena, (size_t)arena_size); { TIMED_SECTION("cpp:ysparse/reset"); - obj->reset(ys_, events, evt_size); - obj->m_handler.reserve(256u); + obj->reset(ys_, arena_, events, evt_size); } { TIMED_SECTION("cpp:ysparse/parse", ys_size); obj->m_parser.parse_in_place_ev(filename_, ys_); } - return (size_type)obj->m_handler.m_evt_curr; + return obj->m_handler.fits_buffers(); +} + +RYML_EXPORT int ysparse_reqsize_evt(ysparse *obj) +{ + return obj->m_handler.required_size_events(); +} + +RYML_EXPORT int ysparse_reqsize_arena(ysparse *obj) +{ + return (int)obj->m_handler.required_size_arena(); } #if defined(__cplusplus) diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp index 9b0c9dbb8..dcb1fc490 100644 --- a/rapidyaml/native/ysparse_evt.hpp +++ b/rapidyaml/native/ysparse_evt.hpp @@ -2,7 +2,9 @@ #ifndef YSPARSE_EVT_HPP_ #define YSPARSE_EVT_HPP_ -#include "ysparse_evt_handler.hpp" +#include "c4/yml/parse_engine.hpp" +#include "c4/yml/parse_engine.def.hpp" +#include "c4/yml/extra/event_handler_ints.hpp" #include "ysparse_common.hpp" #if defined(__cplusplus) @@ -11,17 +13,17 @@ extern "C" { struct RYML_EXPORT ysparse { - ys::EventHandlerEvt m_handler; - c4::yml::ParseEngine m_parser; + c4::yml::extra::EventHandlerInts m_handler; + c4::yml::ParseEngine m_parser; ysparse() : m_handler() , m_parser(&m_handler) { RYML_CHECK(m_parser.options().scalar_filtering()); } - void reset(c4::csubstr src, evt::DataType *evt, int32_t evt_size) + void reset(c4::substr src, c4::substr arena, int32_t *evt, int32_t evt_size) { - m_handler.reset(src, evt, evt_size); + m_handler.reset(src, arena, evt, evt_size); } }; @@ -43,18 +45,20 @@ RYML_EXPORT void ysparse_destroy(ysparse *ryml2evt); * length of the string in the `ys` string. The `ys` string is mutated * during parsing. * - * @return the size needed for `evt`. The caller must check if the - * returned size is larger than `evt_size`. If so, this means that - * `evt` could not accomodate all events produced from `ys`, and is - * incomplete. The caller must then (1) resize `evt` to at least the - * return value, (2) re-copy the original YS into `ys` and (3) call - * again this function, passing in the resized `evt` and the fresh - * copy in `ys`. + * @return true if the `evt` and `arena` buffers were large enough to + * accomodate the result The caller must check this value. When false, + * it means that at least one of the buffers could not accomodate the + * result. The caller must then (1) resize `evt` to at least + * the return value, (2) re-copy the original YS into `ys` and (3) + * call again this function, passing in the resized `evt` and the + * fresh copy in `ys`. * - * @note nothing is written beyond `evt_size`. This means that when - * `evt_size` is 0, then `evt` can be null. This function can be - * safely called for any valid pair of `evt` and `evt_size`, and will - * always return the same required size. + * @note nothing is written beyond `evt_size` or `arena_size`. This + * means that when `evt_size`/`arena_size` is 0, then `evt`/`arena` + * can be null. This function can be safely called for any valid pair + * of `evt`+`evt_size` and `arena`/`arena_size`, and the same required + * size will always be reported. The same applies for + * `arena`+`arena_size`. * * For example, the YAML `say: 2 + 2` produces the following sequence of * 12 integers: @@ -82,10 +86,17 @@ RYML_EXPORT void ysparse_destroy(ysparse *ryml2evt); * in-place in the input string, and the extra integers will pertain * to the resulting filtered string. */ -RYML_EXPORT size_type ysparse_parse(ysparse *ryml2evt, - const char *filename, - char *ys, size_type ys_size, - evt::DataType *evt, size_type evt_size); +RYML_EXPORT bool ysparse_parse(ysparse *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + char *arena, size_type arena_size, + int *evt, size_type evt_size); + +/** Get the required size for the event buffer, from the last parse call */ +RYML_EXPORT int ysparse_reqsize_evt(ysparse *ryml2evt); + +/** Get the required size for the arena buffer, from the last parse call */ +RYML_EXPORT int ysparse_reqsize_arena(ysparse *ryml2evt); #if defined(__cplusplus) } diff --git a/rapidyaml/native/ysparse_evt_handler.cpp b/rapidyaml/native/ysparse_evt_handler.cpp deleted file mode 100644 index 24e5fd662..000000000 --- a/rapidyaml/native/ysparse_evt_handler.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include "ysparse_evt_handler.hpp" -#include -#include - -namespace c4 { -namespace yml { - -// instantiate the template -template class ParseEngine; - -} // namespace yml -} // namespace c4 diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp deleted file mode 100644 index a8f098b99..000000000 --- a/rapidyaml/native/ysparse_evt_handler.hpp +++ /dev/null @@ -1,725 +0,0 @@ -#ifndef _YSPARSE_EVT_HANDLER_HPP_ -#define _YSPARSE_EVT_HANDLER_HPP_ - -#include -#include -#include -#include -#include -#include - -C4_SUPPRESS_WARNING_GCC_CLANG_PUSH -C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") -C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") - -namespace evt { -using DataType = int32_t; -typedef enum : DataType { - // Event types - BSTR = 1 << 0, // +STR - ESTR = 1 << 1, // -STR - BDOC = 1 << 2, // +DOC - EDOC = 1 << 3, // -DOC - BMAP = 1 << 4, // +MAP - EMAP = 1 << 5, // -MAP - BSEQ = 1 << 6, // +SEQ - ESEQ = 1 << 7, // -SEQ - SCLR = 1 << 8, // =VAL - ALIA = 1 << 9, // =ALI - - // Style flags - PLAI = 1 << 16, // : (plain scalar) - SQUO = 1 << 17, // ' (single-quoted scalar) - DQUO = 1 << 18, // " (double-quoted scalar) - LITL = 1 << 19, // | (block literal scalar) - FOLD = 1 << 20, // > (block folded scalar) - FLOW = 1 << 21, // flow container: - // [] for seqs or {} for maps - BLCK = 1 << 22, // block container - - // Modifiers - ANCH = 1 << 24, // anchor - TAG_ = 1 << 25, // tag - - // Structure flags - KEY_ = 1 << 26, // as key - VAL_ = 1 << 27, // as value - EXPL = 1 << 28, // --- (with BDOC) or - // ... (with EDOC) - // (may be fused with FLOW - // if needed) - - // Utility flags - LAST = EXPL, - MASK = (LAST << 1) - 1, - // the event requires a string. the next two integers will provide - // respectively the string's offset and length - HAS_STR = SCLR|ALIA|ANCH|TAG_ -} EventFlags; -} // namespace evt - - -namespace ys { - -using c4::csubstr; -using c4::substr; -using c4::to_substr; -using c4::to_csubstr; -#ifdef RYML_DBG -using c4::_dbg_printf; -#endif - -struct EventHandlerEvtState : public c4::yml::ParserState -{ - c4::yml::type_bits evt_type; - int32_t evt_id; -}; - - -struct EventHandlerEvt : public c4::yml::EventHandlerStack -{ - - /** @name types - * @{ */ - - // our internal state must inherit from parser state - using state = EventHandlerEvtState; - - /** @} */ - -public: - - /** @cond dev */ - csubstr m_str; - evt::DataType * m_evt; - int32_t m_evt_curr; - int32_t m_evt_prev; - int32_t m_evt_size; - char m_key_tag_buf[256]; - char m_val_tag_buf[256]; - std::string m_arena; - - // undefined at the end - #define _enable_(bits) _enable__() - #define _disable_(bits) _disable__() - #define _has_any_(bits) _has_any__() - /** @endcond */ - -public: - - /** @name construction and resetting - * @{ */ - - EventHandlerEvt(c4::yml::Callbacks const& cb) - : EventHandlerStack(cb) - { - reset({}, nullptr, 0); - } - EventHandlerEvt() - : EventHandlerEvt(c4::yml::get_callbacks()) - { - } - - void reset(csubstr str, evt::DataType *dst, int32_t dst_size) - { - _stack_reset_root(); - m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; - m_curr->evt_type = {}; - m_curr->evt_id = 0; - m_arena.clear(); - m_str = str; - m_evt = dst; - m_evt_size = dst_size; - m_evt_curr = 0; - m_evt_prev = 0; - } - - void reserve(int arena_size) - { - m_arena.reserve(arena_size); - } - - /** @} */ - -public: - - /** @name parse events - * @{ */ - - void start_parse(const char* filename, c4::yml::detail::pfn_relocate_arena relocate_arena, void *relocate_arena_data) - { - this->_stack_start_parse(filename, relocate_arena, relocate_arena_data); - } - - void finish_parse() - { - this->_stack_finish_parse(); - } - - void cancel_parse() - { - while(m_stack.size() > 1) - _pop(); - } - - /** @} */ - -public: - - /** @name YAML stream events */ - /** @{ */ - - void begin_stream() - { - _send_flag_only_(evt::BSTR); - } - - void end_stream() - { - _send_flag_only_(evt::ESTR); - } - - /** @} */ - -public: - - /** @name YAML document events */ - /** @{ */ - - /** implicit doc start (without ---) */ - void begin_doc() - { - _c4dbgpf("{}/{}: begin_doc", m_evt_curr, m_evt_size); - _send_flag_only_(evt::BDOC); - if(_stack_should_push_on_begin_doc()) - { - _c4dbgp("push!"); - _push(); - } - } - /** implicit doc end (without ...) */ - void end_doc() - { - _c4dbgpf("{}/{}: end_doc", m_evt_curr, m_evt_size); - _send_flag_only_(evt::EDOC); - if(_stack_should_pop_on_end_doc()) - { - _c4dbgp("pop!"); - _pop(); - } - } - - /** explicit doc start, with --- */ - void begin_doc_expl() - { - _c4dbgpf("{}/{}: begin_doc_expl", m_evt_curr, m_evt_size); - _send_flag_only_(evt::BDOC|evt::EXPL); - if(_stack_should_push_on_begin_doc()) - { - _c4dbgp("push!"); - _push(); - } - } - /** explicit doc end, with ... */ - void end_doc_expl() - { - _c4dbgpf("{}/{}: end_doc_expl", m_evt_curr, m_evt_size); - _send_flag_only_(evt::EDOC|evt::EXPL); - if(_stack_should_pop_on_end_doc()) - { - _c4dbgp("pop!"); - _pop(); - } - } - - /** @} */ - -public: - - /** @name YAML map functions */ - /** @{ */ - - void begin_map_key_flow() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - void begin_map_key_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - void begin_map_val_flow() - { - _c4dbgpf("{}/{}: bmap flow", m_evt_curr, m_evt_size); - _send_flag_only_(evt::VAL_|evt::BMAP|evt::FLOW); - _mark_parent_with_children_(); - _enable_(c4::yml::MAP|c4::yml::FLOW_SL); - _push(); - } - void begin_map_val_block() - { - _c4dbgpf("{}/{}: bmap block", m_evt_curr, m_evt_size); - _send_flag_only_(evt::VAL_|evt::BMAP|evt::BLCK); - _mark_parent_with_children_(); - _enable_(c4::yml::MAP|c4::yml::BLOCK); - _push(); - } - - void end_map() - { - _pop(); - _send_flag_only_(evt::EMAP); - } - - /** @} */ - -public: - - /** @name YAML seq events */ - /** @{ */ - - void begin_seq_key_flow() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - void begin_seq_key_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - void begin_seq_val_flow() - { - _c4dbgpf("{}/{}: bseq flow", m_evt_curr, m_evt_size); - _send_flag_only_(evt::VAL_|evt::BSEQ|evt::FLOW); - _mark_parent_with_children_(); - _enable_(c4::yml::SEQ|c4::yml::FLOW_SL); - _push(); - } - void begin_seq_val_block() - { - _c4dbgpf("{}/{}: bseq block", m_evt_curr, m_evt_size); - _send_flag_only_(evt::VAL_|evt::BSEQ|evt::BLCK); - _mark_parent_with_children_(); - _enable_(c4::yml::SEQ|c4::yml::BLOCK); - _push(); - } - - void end_seq() - { - _pop(); - _send_flag_only_(evt::ESEQ); - } - - /** @} */ - -public: - - /** @name YAML structure events */ - /** @{ */ - - void add_sibling() - { - _RYML_CB_ASSERT(m_stack.m_callbacks, m_parent); - m_curr->evt_type = {}; - } - - /** set the previous val as the first key of a new map, with flow style. - * - * See the documentation for @ref doc_event_handlers, which has - * important notes about this event. - */ - void actually_val_is_first_key_of_new_map_flow() - { - _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt_curr > 2); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt_prev > 0); - _c4dbgpf("{}/{}: prev={} actually_val_is_first_key_of_new_map_flow", m_evt_curr, m_evt_size, m_evt_prev); - // BEFORE - // ... flag start len (free) - // | | - // prev curr - // AFTER - // ... flag flag start len (free) - // | | - // prev curr - if(m_evt_prev < m_evt_size) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, (m_evt[m_evt_prev] & evt::HAS_STR) || m_evt_curr >= m_evt_size); - if(m_evt_curr < m_evt_size) - { - // watchout: it must be in this order! - m_evt[m_evt_curr ] = m_evt[m_evt_prev + 2]; - m_evt[m_evt_curr - 1] = m_evt[m_evt_prev + 1]; - m_evt[m_evt_curr - 2] = m_evt[m_evt_prev] | evt::KEY_; - m_evt[m_evt_curr - 2] &= ~evt::VAL_; - } - m_evt[m_evt_prev] = evt::BMAP|evt::FLOW|evt::VAL_; - } - m_curr->evt_id = m_evt_curr - 2; - ++m_evt_prev; - ++m_evt_curr; - _enable_(c4::yml::MAP|c4::yml::FLOW); - _push(); - } - - void actually_val_is_first_key_of_new_map_block() - { - _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); - } - - /** @} */ - -public: - - /** @name YAML scalar events */ - /** @{ */ - - - C4_ALWAYS_INLINE void set_key_scalar_plain_empty() - { - _c4dbgpf("{}/{}: set_key_scalar_plain_empty", m_evt_curr, m_evt_size); - _send_key_scalar_(_get_latest_empty_scalar(), evt::PLAI); - _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN|c4::yml::KEYNIL); - } - C4_ALWAYS_INLINE void set_val_scalar_plain_empty() - { - _c4dbgpf("{}/{}: set_val_scalar_plain_empty", m_evt_curr, m_evt_size); - _send_val_scalar_(_get_latest_empty_scalar(), evt::PLAI); - _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN|c4::yml::VALNIL); - } - C4_ALWAYS_INLINE csubstr _get_latest_empty_scalar() const - { - // ideally we should search back in the latest event that has - // a scalar, than select a zero-length scalar immediately - // after that scalar. But this also works for now: - return m_str.first(0); - } - - - C4_ALWAYS_INLINE void set_key_scalar_plain(csubstr scalar) - { - _c4dbgpf("{}/{}: set_key_scalar_plain: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_key_scalar_(scalar, evt::PLAI); - _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN); - } - C4_ALWAYS_INLINE void set_val_scalar_plain(csubstr scalar) - { - _c4dbgpf("{}/{}: set_val_scalar_plain: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_val_scalar_(scalar, evt::PLAI); - _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN); - } - - - C4_ALWAYS_INLINE void set_key_scalar_dquoted(csubstr scalar) - { - _c4dbgpf("{}/{}: set_key_scalar_dquo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_key_scalar_(scalar, evt::DQUO); - _enable_(c4::yml::KEY|c4::yml::KEY_DQUO); - } - C4_ALWAYS_INLINE void set_val_scalar_dquoted(csubstr scalar) - { - _c4dbgpf("{}/{}: set_val_scalar_dquo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_val_scalar_(scalar, evt::DQUO); - _enable_(c4::yml::VAL|c4::yml::VAL_DQUO); - } - - - C4_ALWAYS_INLINE void set_key_scalar_squoted(csubstr scalar) - { - _c4dbgpf("{}/{}: set_key_scalar_squo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_key_scalar_(scalar, evt::SQUO); - _enable_(c4::yml::KEY|c4::yml::KEY_SQUO); - } - C4_ALWAYS_INLINE void set_val_scalar_squoted(csubstr scalar) - { - _c4dbgpf("{}/{}: set_val_scalar_squo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_val_scalar_(scalar, evt::SQUO); - _enable_(c4::yml::VAL|c4::yml::VAL_SQUO); - } - - - C4_ALWAYS_INLINE void set_key_scalar_literal(csubstr scalar) - { - _c4dbgpf("{}/{}: set_key_scalar_literal: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_key_scalar_(scalar, evt::LITL); - _enable_(c4::yml::KEY|c4::yml::KEY_LITERAL); - } - C4_ALWAYS_INLINE void set_val_scalar_literal(csubstr scalar) - { - _c4dbgpf("{}/{}: set_val_scalar_literal: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_val_scalar_(scalar, evt::LITL); - _enable_(c4::yml::VAL|c4::yml::VAL_LITERAL); - } - - - C4_ALWAYS_INLINE void set_key_scalar_folded(csubstr scalar) - { - _c4dbgpf("{}/{}: set_key_scalar_folded: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_key_scalar_(scalar, evt::FOLD); - _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); - } - C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) - { - _c4dbgpf("{}/{}: set_val_scalar_folded: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); - _send_val_scalar_(scalar, evt::FOLD); - _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); - } - - - C4_ALWAYS_INLINE void mark_key_scalar_unfiltered() - { - _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); - } - C4_ALWAYS_INLINE void mark_val_scalar_unfiltered() - { - _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); - } - - /** @} */ - -public: - -#define _add_scalar_(i, scalar) \ - _c4dbgpf("{}/{}: scalar!", i, m_evt_size); \ - _RYML_CB_ASSERT(m_stack.m_callbacks, scalar.is_sub(m_str)); \ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt[i] & evt::HAS_STR); \ - _RYML_CB_ASSERT(m_stack.m_callbacks, i + 2 < m_evt_size); \ - m_evt[i + 1] = (evt::DataType)(scalar.str - m_str.str); \ - m_evt[i + 2] = (evt::DataType)scalar.len - - /** @name YAML anchor/reference events */ - /** @{ */ - - void set_key_anchor(csubstr anchor) - { - _c4dbgpf("{}/{}: set_key_anchor", m_evt_curr, m_evt_size); - _enable_(c4::yml::KEYANCH); - if(m_evt_curr + 2 < m_evt_size) - { - m_evt[m_evt_curr] = evt::KEY_|evt::ANCH; - _add_scalar_(m_evt_curr, anchor); - } - m_evt_prev = m_evt_curr; - m_evt_curr += 3; - } - void set_val_anchor(csubstr anchor) - { - _c4dbgpf("{}/{}: set_val_anchor", m_evt_curr, m_evt_size); - _enable_(c4::yml::VALANCH); - if(m_evt_curr + 2 < m_evt_size) - { - m_evt[m_evt_curr] = evt::VAL_|evt::ANCH; - _add_scalar_(m_evt_curr, anchor); - } - m_evt_prev = m_evt_curr; - m_evt_curr += 3; - } - - void set_key_ref(csubstr ref) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); - _enable_(c4::yml::KEY|c4::yml::KEYREF); - _send_str_(ref.sub(1), evt::KEY_|evt::ALIA); // skip the leading * - } - void set_val_ref(csubstr ref) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); - _enable_(c4::yml::VAL|c4::yml::VALREF); - _send_str_(ref.sub(1), evt::VAL_|evt::ALIA); // skip the leading * - } - - /** @} */ - -public: - - /** @name YAML tag events */ - /** @{ */ - - void set_key_tag(csubstr tag) - { - _enable_(c4::yml::KEYTAG); - csubstr ttag = _transform_directive(tag, m_key_tag_buf); - _RYML_CB_ASSERT(m_stack.m_callbacks, !ttag.empty()); - if(ttag.begins_with('!') && !ttag.begins_with("!!")) - ttag = ttag.sub(1); - if(m_evt_curr + 2 < m_evt_size) - { - m_evt[m_evt_curr] = evt::KEY_|evt::TAG_; - _add_scalar_(m_evt_curr, ttag); - } - m_evt_prev = m_evt_curr; - m_evt_curr += 3; - } - void set_val_tag(csubstr tag) - { - _enable_(c4::yml::VALTAG); - csubstr ttag = _transform_directive(tag, m_val_tag_buf); - _RYML_CB_ASSERT(m_stack.m_callbacks, !ttag.empty()); - if(ttag.begins_with('!') && !ttag.begins_with("!!")) - ttag = ttag.sub(1); - if(m_evt_curr + 2 < m_evt_size) - { - m_evt[m_evt_curr] = evt::VAL_|evt::TAG_; - _add_scalar_(m_evt_curr, ttag); - } - m_evt_prev = m_evt_curr; - m_evt_curr += 3; - } - - /** @} */ - -public: - - /** @name YAML directive events */ - /** @{ */ - - void add_directive(csubstr directive) - { - _RYML_CB_ERR(m_stack.m_callbacks, "tag directives not supported"); - } - - /** @} */ - -public: - - /** @name YAML arena events */ - /** @{ */ - - substr alloc_arena(size_t len) - { - const size_t sz = m_arena.size(); - csubstr prev = to_csubstr(m_arena); - m_arena.resize(sz + len); - substr out = to_substr(m_arena).sub(sz); - substr curr = to_substr(m_arena); - if(curr.str != prev.str) - _stack_relocate_to_new_arena(prev, curr); - return out; - } - - substr alloc_arena(size_t len, substr *relocated) - { - csubstr prev = to_csubstr(m_arena); - if(!prev.is_super(*relocated)) - return alloc_arena(len); - substr out = alloc_arena(len); - substr curr = to_substr(m_arena); - if(curr.str != prev.str) - *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); - return out; - } - - /** @} */ - -public: - - /** push a new parent, add a child to the new parent, and set the - * child as the current node */ - void _push() - { - _stack_push(); - m_curr->evt_type = {}; - } - - /** end the current scope */ - void _pop() - { - _stack_pop(); - } - - template C4_ALWAYS_INLINE void _enable__() noexcept - { - m_curr->evt_type |= bits; - } - template C4_ALWAYS_INLINE void _disable__() noexcept - { - m_curr->evt_type &= ~bits; - } - template C4_ALWAYS_INLINE bool _has_any__() const noexcept - { - return (m_curr->evt_type & bits) != c4::yml::type_bits(0); - } - - void _mark_parent_with_children_() - { - if(m_parent) - m_parent->has_children = true; - } - - C4_ALWAYS_INLINE void _send_flag_only_(evt::DataType flags) - { - _c4dbgpf("{}/{}: flag only", m_evt_curr, m_evt_size); - if(m_evt_curr < m_evt_size) - m_evt[m_evt_curr] = flags; - m_curr->evt_id = m_evt_curr; - m_evt_prev = m_evt_curr; - ++m_evt_curr; - } - - C4_ALWAYS_INLINE void _send_key_scalar_(csubstr scalar, evt::DataType flags) - { - _c4dbgpf("{}/{}: key scalar", m_evt_curr, m_evt_size); - if(m_evt_curr + 2 < m_evt_size) - { - m_evt[m_evt_curr] = evt::SCLR|evt::KEY_|flags; - _add_scalar_(m_evt_curr, scalar); - } - m_curr->evt_id = m_evt_curr; - m_evt_prev = m_evt_curr; - m_evt_curr += 3; - } - - C4_ALWAYS_INLINE void _send_val_scalar_(csubstr scalar, evt::DataType flags) - { - _c4dbgpf("{}/{}: val scalar", m_evt_curr, m_evt_size); - if(m_evt_curr + 2 < m_evt_size) - { - m_evt[m_evt_curr] = evt::SCLR|evt::VAL_|flags; - _add_scalar_(m_evt_curr, scalar); - } - m_curr->evt_id = m_evt_curr; - m_evt_prev = m_evt_curr; - m_evt_curr += 3; - } - - C4_ALWAYS_INLINE void _send_str_(csubstr scalar, evt::DataType flags) - { - _c4dbgpf("{}/{}: send str", m_evt_curr, m_evt_size); - if(m_evt_curr + 2 < m_evt_size) - { - m_evt[m_evt_curr] = flags; - _add_scalar_(m_evt_curr, scalar); - } - m_curr->evt_id = m_evt_curr; - m_evt_prev = m_evt_curr; - m_evt_curr += 3; - } - - csubstr _transform_directive(csubstr tag, substr output) - { - if(tag.begins_with("!!")) - { - return tag; - } - else if(tag.begins_with('!')) - { - if(c4::yml::is_custom_tag(tag)) - { - _RYML_CB_ERR_(m_stack.m_callbacks, "tag not found", m_curr->pos); - } - } - csubstr result = c4::yml::normalize_tag_long(tag, output); - _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); - _RYML_CB_CHECK(m_stack.m_callbacks, result.str); - return result; - } -#undef _enable_ -#undef _disable_ -#undef _has_any_ - -}; - -} // namespace ys - -C4_SUPPRESS_WARNING_GCC_POP - -#endif /* _C4_YML_EVENT_HANDLER_EVT_HPP_ */ diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp index 63de41d49..8d1f49713 100644 --- a/rapidyaml/native/ysparse_test.cpp +++ b/rapidyaml/native/ysparse_test.cpp @@ -1,45 +1,14 @@ #include -#include +#include +#include +#include #include using c4::csubstr; using c4::substr; using ryml::Location; - -namespace c4 -{ -template<> -c4::EnumSymbols const esyms() -{ - static constexpr typename c4::EnumSymbols::Sym syms[] = { - {evt::KEY_, "KEY_"}, - {evt::VAL_, "VAL_"}, - {evt::SCLR, "SCLR"}, - {evt::BSEQ, "BSEQ"}, - {evt::ESEQ, "ESEQ"}, - {evt::BMAP, "BMAP"}, - {evt::EMAP, "EMAP"}, - {evt::ALIA, "ALIA"}, - {evt::ANCH, "ANCH"}, - {evt::TAG_, "TAG_"}, - {evt::PLAI, "PLAI"}, - {evt::SQUO, "SQUO"}, - {evt::DQUO, "DQUO"}, - {evt::LITL, "LITL"}, - {evt::FOLD, "FOLD"}, - {evt::FLOW, "FLOW"}, - {evt::BLCK, "BLCK"}, - {evt::BDOC, "BDOC"}, - {evt::EDOC, "EDOC"}, - {evt::BSTR, "BSTR"}, - {evt::ESTR, "ESTR"}, - {evt::EXPL, "EXPL"}, - }; - return c4::EnumSymbols(syms); -} -} - +namespace ievt = c4::yml::extra::ievt; //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -74,10 +43,10 @@ struct TestResult // data in a single structure struct EvtWithScalar { - evt::DataType flags, str_start, str_len; + int flags, str_start, str_len; csubstr scalar; bool needs_filter; - EvtWithScalar(evt::DataType t, evt::DataType start=0, evt::DataType len=0, csubstr sclr={}, bool needs_filter_=false) + EvtWithScalar(int t, int start=0, int len=0, csubstr sclr={}, bool needs_filter_=false) { flags = t; str_start = start; @@ -85,7 +54,7 @@ struct EvtWithScalar scalar = sclr; needs_filter = needs_filter_; } - size_t required_size() const { return (flags & evt::HAS_STR) ? 3u : 1u; } + size_t required_size() const { return (flags & ievt::WSTR) ? 3u : 1u; } }; size_t expected_size(std::vector const& evt) @@ -139,6 +108,7 @@ size_t expected_size(std::vector const& evt) struct TestCase { + int line; csubstr ys; std::vector evt; @@ -162,16 +132,24 @@ struct TestCase if(evt.empty()) return {}; TestResult tr = {}; std::string input_(ys.begin(), ys.end()); + std::string arena_; + arena_.resize((ys.size() * size_t(3)) / size_t(2)); substr input = c4::to_substr(input_); - std::vector output; + substr arena = c4::to_substr(arena_); + std::vector output; output.resize(2 * expected_size(evt)); - size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", - input.str, (size_type)input.len, - &output[0], (size_type)output.size()); - CHECK_MSG((size_t)reqsize == expected_size(evt), "%d vs %zu", reqsize, expected_size(evt)); - CHECK(reqsize != 0); - output.resize(reqsize); - CHECK(testeq(output, input)); + int estimated_size = c4::yml::extra::estimate_events_ints_size(input); + bool fits_buffers = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + arena.str, (size_type)arena.len, + &output[0], (size_type)output.size()); + CHECK(fits_buffers); + int reqsize_evt = ysparse_reqsize_evt(ryml2evt); + CHECK_MSG((size_t)reqsize_evt == expected_size(evt), "%d vs %zu", reqsize_evt, expected_size(evt)); + CHECK(reqsize_evt != 0); + CHECK(reqsize_evt <= estimated_size); + output.resize(reqsize_evt); + CHECK(testeq(output, input, arena)); return tr; } TestResult test_evt_large_enough() const @@ -185,23 +163,38 @@ struct TestCase { TestResult tr = {}; std::string input_(ys.begin(), ys.end()); + std::string arena_; + arena_.resize(0); substr input = c4::to_substr(input_); - std::vector output; - output.resize(expected_size(evt)); - size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); - CHECK(reqsize == expected_size(evt)); - CHECK(reqsize != 0); - output.resize(reqsize); - input_.assign(ys.begin(), ys.end()); // FIXME - input = c4::to_substr(input_); - size_type reqsize2 = ysparse_parse(ryml2evt, "ysfilename", + substr arena = c4::to_substr(arena_); + std::vector output; + int estimated_size = c4::yml::extra::estimate_events_ints_size(input); + bool fits_buffers = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, + arena.str, (size_type)arena.len, output.data(), (size_type)output.size()); - CHECK(reqsize2 == reqsize); - output.resize(reqsize2); - CHECK(testeq(output, input)); + CHECK(!fits_buffers); + int reqsize_evt = ysparse_reqsize_evt(ryml2evt); + int reqsize_arena = ysparse_reqsize_arena(ryml2evt); + CHECK(reqsize_evt == expected_size(evt)); + CHECK(reqsize_evt != 0); + CHECK(reqsize_evt <= estimated_size); + output.resize(reqsize_evt); + arena_.resize(reqsize_arena); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + arena = c4::to_substr(arena_); + bool fits_buffers2 = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + arena.str, (size_type)arena.len, + output.data(), (size_type)output.size()); + CHECK(fits_buffers2); + int reqsize_evt2 = ysparse_reqsize_evt(ryml2evt); + int reqsize_arena2 = ysparse_reqsize_arena(ryml2evt); + CHECK(reqsize_evt2 == reqsize_evt); + CHECK(reqsize_arena2 == reqsize_arena); + output.resize(reqsize_evt2); + CHECK(testeq(output, input, arena)); return tr; } TestResult test_evt_too_small() const @@ -216,21 +209,35 @@ struct TestCase TestResult tr = {}; std::string input_(ys.begin(), ys.end()); substr input = c4::to_substr(input_); - size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", - input.str, (size_type)input.len, - nullptr, 0); - CHECK(reqsize == expected_size(evt)); - CHECK(reqsize != 0); - std::vector output; - output.resize(reqsize); + int estimated_size = c4::yml::extra::estimate_events_ints_size(input); + bool fits_buffers = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0, + nullptr, 0); + CHECK(!fits_buffers); + int reqsize_evt = ysparse_reqsize_evt(ryml2evt); + int reqsize_arena = ysparse_reqsize_arena(ryml2evt); + CHECK(reqsize_evt == expected_size(evt)); + CHECK(reqsize_evt <= estimated_size); + CHECK(reqsize_evt != 0); + std::string arena_; + arena_.resize(reqsize_arena); + substr arena = c4::to_substr(arena_); + std::vector output; + output.resize(reqsize_evt); input_.assign(ys.begin(), ys.end()); // FIXME input = c4::to_substr(input_); - size_type reqsize2 = ysparse_parse(ryml2evt, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); - CHECK(reqsize2 == reqsize); - CHECK(reqsize2 == output.size()); - CHECK(testeq(output, input)); + bool fits_buffers2 = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + arena.str, (size_type)arena.len, + output.data(), (size_type)output.size()); + CHECK(fits_buffers2); + int reqsize_evt2 = ysparse_reqsize_evt(ryml2evt); + int reqsize_arena2 = ysparse_reqsize_arena(ryml2evt); + CHECK(reqsize_evt2 == reqsize_evt); + CHECK(reqsize_arena2 == reqsize_arena); + CHECK(reqsize_evt2 == output.size()); + CHECK(testeq(output, input, arena)); return tr; } TestResult test_evt_nullptr() const @@ -241,7 +248,7 @@ struct TestCase public: - bool testeq(std::vector const& actual, csubstr parsed_source) const + bool testeq(std::vector const& actual, csubstr parsed_source, csubstr arena) const { int status = true; size_t num_events_expected = evt.size(); @@ -259,37 +266,38 @@ struct TestCase actual.size()); same_size = false; } - for(size_t i = 0, ie = 0; ie < num_events_expected; ++ie) + for(size_t ia = 0, ie = 0; ie < num_events_expected; ++ie) { - if(i >= actual.size()) + if(ia >= actual.size()) { - printf("fail: bad actual size. i=%zu vs %zu=actual.size()=\n", i, actual.size()); + printf("fail: bad actual size. i=%zu vs %zu=actual.size()=\n", ia, actual.size()); status = false; break; } #define _testcmp(fmt, cmp, ...) \ - if(showcmp) { printf("status=%d cmp=%d evt=%zu i=%zu: " fmt "\n", status, (cmp), ie, i, ## __VA_ARGS__); } \ + if(showcmp) { printf("status=%d cmp=%d ie=%zu ia=%zu: " fmt "\n", status, (cmp), ie, ia, ## __VA_ARGS__); } \ status &= (cmp) - char actualbuf[100]; - char expectedbuf[100]; - size_t reqsize_actual = c4::bm2str(actual[i] & evt::MASK, actualbuf, sizeof(actualbuf)); - size_t reqsize_expected = c4::bm2str(evt[ie].flags & evt::MASK, expectedbuf, sizeof(expectedbuf)); - C4_CHECK(reqsize_actual < sizeof(actualbuf)); - C4_CHECK(reqsize_expected < sizeof(expectedbuf)); - _testcmp("exp=%d(%s) vs act=%d(%s)", evt[ie].flags == actual[i], evt[ie].flags, expectedbuf, actual[i], actualbuf); - status &= (evt[ie].flags == actual[i]); - if((evt[ie].flags & evt::HAS_STR) && (actual[i] & evt::HAS_STR)) + char actualbuf_[100]; + char expectedbuf_[100]; + csubstr actualbuf = c4::yml::extra::ievt::to_chars_sub(actualbuf_, actual[ia] & ievt::MASK); + csubstr expectedbuf = c4::yml::extra::ievt::to_chars_sub(expectedbuf_, evt[ie].flags & ievt::MASK); + _testcmp("exp=%d(%.*s) vs act=%d(%.*s)", evt[ie].flags == actual[ia], + evt[ie].flags, (int)expectedbuf.len, expectedbuf.str, + actual[ia], (int)actualbuf.len, actualbuf.str); + status &= (evt[ie].flags == actual[ia]); + if((evt[ie].flags & ievt::WSTR) && (actual[ia] & ievt::WSTR)) { - _testcmp(" exp=%d vs act=%d", evt[ie].str_start == actual[i + 1], evt[ie].str_start, actual[i + 1]); - _testcmp(" exp=%d vs act=%d", evt[ie].str_len == actual[i + 2], evt[ie].str_len, actual[i + 2]); - bool safeactual = (i + 2 < actual.size()) && (actual[i + 1] < (int)parsed_source.len && actual[i + 1] + actual[i + 2] <= (int)parsed_source.len); - bool safeexpected = (evt[ie].str_start < (int)parsed_source.len && evt[ie].str_start + evt[ie].str_len <= (int)parsed_source.len); + csubstr region = (actual[ia] & ievt::AREN) ? arena : parsed_source; + _testcmp(" exp=%d vs act=%d", evt[ie].str_start == actual[ia + 1], evt[ie].str_start, actual[ia + 1]); + _testcmp(" exp=%d vs act=%d", evt[ie].str_len == actual[ia + 2], evt[ie].str_len, actual[ia + 2]); + bool safeactual = (ia + 2 < actual.size()) && (actual[ia + 1] < (int)region.len && actual[ia + 1] + actual[ia + 2] <= (int)region.len); + bool safeexpected = (evt[ie].str_start < (int)region.len && evt[ie].str_start + evt[ie].str_len <= (int)region.len); _testcmp(" safeactual=%d", safeactual, safeactual); _testcmp(" safeactual=%d safeexpected=%d", safeactual == safeexpected, safeactual, safeexpected); if(safeactual && safeexpected) { - csubstr evtstr = parsed_source.sub((size_t)evt[ie].str_start, (size_t)evt[ie].str_len); - csubstr actualstr = parsed_source.sub((size_t)actual[i + 1], (size_t)actual[i + 2]); + csubstr evtstr = region.sub((size_t)evt[ie].str_start, (size_t)evt[ie].str_len); + csubstr actualstr = region.sub((size_t)actual[ia + 1], (size_t)actual[ia + 2]); _testcmp(" ref=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", evt[ie].scalar == actualstr, evt[ie].scalar.len, (int)evt[ie].scalar.len, evt[ie].scalar.str, @@ -303,7 +311,7 @@ struct TestCase } } } - i += (actual[i] & evt::HAS_STR) ? 3 : 1; + ia += (actual[ia] & ievt::WSTR) ? 3 : 1; } if(!status) printf("------\n" @@ -319,12 +327,13 @@ struct TestCase struct TestCaseErr { + int line; csubstr ys; bool is_parse_err; ryml::Location loc; - TestCaseErr(csubstr ys_) : ys(ys_), is_parse_err(false), loc() {} - TestCaseErr(csubstr ys_, ryml::Location loc_) : ys(ys_), is_parse_err(true), loc(loc_) {} + TestCaseErr(int line_, csubstr ys_) : line(line_), ys(ys_), is_parse_err(false), loc() {} + TestCaseErr(int line_, csubstr ys_, ryml::Location loc_) : line(line_), ys(ys_), is_parse_err(true), loc(loc_) {} TestResult test(ysparse *ryml2evt) const { @@ -344,6 +353,7 @@ struct TestCaseErr { size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", input.str, (size_type)input.len, + nullptr, 0, nullptr, 0); } catch(YsParseError const& exc) @@ -378,9 +388,9 @@ struct TestCaseErr namespace { // make the declarations shorter -#define tc(ys, ...) {ys, std::vector(__VA_ARGS__)} +#define tc(ys, ...) {__LINE__, ys, std::vector(__VA_ARGS__)} #define e(...) EvtWithScalar{__VA_ARGS__} -using namespace evt; +using namespace ievt; inline constexpr bool needs_filter = true; const TestCase test_cases[] = { // case ------------------------------------------------- @@ -388,13 +398,13 @@ const TestCase test_cases[] = { { e(BSTR), e(BDOC), - e(VAL_|TAG_, 1, 18, "yamlscript/v0/bare"), - e(VAL_|SCLR|PLAI, 0, 0, ""), - e(EDOC), + e(VAL_|TAG_, 0, 19, "!yamlscript/v0/bare"), + e(VAL_|SCLR|PLAI|PSTR, 0, 0, ""), + e(EDOC|PSTR), e(BDOC|EXPL), - e(VAL_|TAG_, 25, 4, "code"), - e(VAL_|SCLR|PLAI, 30, 2, "42"), - e(EDOC), + e(VAL_|TAG_, 24, 5, "!code"), + e(VAL_|SCLR|PLAI|PSTR, 30, 2, "42"), + e(EDOC|PSTR), e(ESTR), }), // case ------------------------------------------------- @@ -403,9 +413,9 @@ const TestCase test_cases[] = { e(BSTR), e(BDOC), e(VAL_|BMAP|BLCK), - e(KEY_|SCLR|PLAI, 0, 1, "a"), - e(VAL_|SCLR|PLAI, 3, 1, "1"), - e(EMAP), + e(KEY_|SCLR|PLAI, 0, 1, "a"), + e(VAL_|SCLR|PLAI|PSTR, 3, 1, "1"), + e(EMAP|PSTR), e(EDOC), e(ESTR), }), @@ -415,9 +425,9 @@ const TestCase test_cases[] = { e(BSTR), e(BDOC), e(VAL_|BMAP|BLCK), - e(KEY_|SCLR|PLAI, 0, 3, "say"), - e(VAL_|SCLR|PLAI, 5, 5, "2 + 2"), - e(EMAP), + e(KEY_|SCLR|PLAI, 0, 3, "say"), + e(VAL_|SCLR|PLAI|PSTR, 5, 5, "2 + 2"), + e(EMAP|PSTR), e(EDOC), e(ESTR), }), @@ -427,9 +437,9 @@ const TestCase test_cases[] = { e(BSTR), e(BDOC), e(VAL_|BMAP|BLCK), - e(KEY_|SCLR|PLAI, 0, 4, "𝄞"), - e(VAL_|SCLR|PLAI, 6, 3, "✅"), - e(EMAP), + e(KEY_|SCLR|PLAI, 0, 4, "𝄞"), + e(VAL_|SCLR|PLAI|PSTR, 6, 3, "✅"), + e(EMAP|PSTR), e(EDOC), e(ESTR), }), @@ -439,10 +449,10 @@ const TestCase test_cases[] = { e(BSTR), e(BDOC), e(VAL_|BSEQ|FLOW), - e(VAL_|SCLR|PLAI, 1, 1, "a"), - e(VAL_|SCLR|PLAI, 4, 1, "b"), - e(VAL_|SCLR|PLAI, 7, 1, "c"), - e(ESEQ), + e(VAL_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI|PSTR, 4, 1, "b"), + e(VAL_|SCLR|PLAI|PSTR, 7, 1, "c"), + e(ESEQ|PSTR), e(EDOC), e(ESTR), }), @@ -453,9 +463,9 @@ const TestCase test_cases[] = { e(BDOC), e(VAL_|BSEQ|FLOW), e(VAL_|BMAP|FLOW), - e(KEY_|SCLR|PLAI, 1, 1, "a"), - e(VAL_|SCLR|PLAI, 4, 1, "b"), - e(EMAP), + e(KEY_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI|PSTR, 4, 1, "b"), + e(EMAP|PSTR), e(ESEQ), e(EDOC), e(ESTR), @@ -480,42 +490,42 @@ another: doc { e(BSTR), e(BDOC|EXPL), - e(VAL_|TAG_, 5, 13, "yamlscript/v0"), - e(VAL_|BMAP|BLCK), + e(VAL_|TAG_, 4, 14, "!yamlscript/v0"), + e(VAL_|BMAP|BLCK|PSTR), e(KEY_|SCLR|PLAI, 19, 3, "foo"), - e(VAL_|TAG_, 25, 0, ""), - e(VAL_|BSEQ|BLCK), + e(VAL_|TAG_|PSTR, 24, 1, "!"), + e(VAL_|BSEQ|BLCK|PSTR), e(VAL_|BMAP|FLOW), e(KEY_|SCLR|PLAI, 29, 1, "x"), - e(VAL_|SCLR|PLAI, 32, 1, "y"), - e(EMAP), + e(VAL_|SCLR|PLAI|PSTR, 32, 1, "y"), + e(EMAP|PSTR), e(VAL_|BSEQ|FLOW), e(VAL_|SCLR|PLAI, 38, 1, "x"), - e(VAL_|SCLR|PLAI, 41, 1, "y"), - e(ESEQ), + e(VAL_|SCLR|PLAI|PSTR, 41, 1, "y"), + e(ESEQ|PSTR), e(VAL_|SCLR|PLAI, 46, 3, "foo"), - e(VAL_|SCLR|SQUO, 53, 3, "foo"), - e(VAL_|SCLR|DQUO, 61, 3, "foo"), - e(VAL_|SCLR|LITL, 70, 4, "foo\n", needs_filter), - e(VAL_|SCLR|FOLD, 80, 4, "foo\n", needs_filter), - e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|SQUO|PSTR, 53, 3, "foo"), + e(VAL_|SCLR|DQUO|PSTR, 61, 3, "foo"), + e(VAL_|SCLR|LITL|PSTR, 70, 4, "foo\n", needs_filter), + e(VAL_|SCLR|FOLD|PSTR, 80, 4, "foo\n", needs_filter), + e(VAL_|BSEQ|FLOW|PSTR), e(VAL_|SCLR|PLAI, 89, 1, "1"), - e(VAL_|SCLR|PLAI, 92, 1, "2"), - e(VAL_|SCLR|PLAI, 95, 4, "true"), - e(VAL_|SCLR|PLAI, 101, 5, "false"), - e(VAL_|SCLR|PLAI, 108, 4, "null"), - e(ESEQ), - e(VAL_|TAG_, 127, 5, "tag-1"), - e(VAL_|ANCH, 117, 8, "anchor-1"), - e(VAL_|SCLR|PLAI, 133, 6, "foobar"), - e(ESEQ), + e(VAL_|SCLR|PLAI|PSTR, 92, 1, "2"), + e(VAL_|SCLR|PLAI|PSTR, 95, 4, "true"), + e(VAL_|SCLR|PLAI|PSTR, 101, 5, "false"), + e(VAL_|SCLR|PLAI|PSTR, 108, 4, "null"), + e(ESEQ|PSTR), + e(VAL_|TAG_, 126, 6, "!tag-1"), + e(VAL_|ANCH|PSTR, 117, 8, "anchor-1"), + e(VAL_|SCLR|PLAI|PSTR, 133, 6, "foobar"), + e(ESEQ|PSTR), e(EMAP), e(EDOC), e(BDOC|EXPL), e(VAL_|BMAP|BLCK), e(KEY_|SCLR|PLAI, 144, 7, "another"), - e(VAL_|SCLR|PLAI, 153, 3, "doc"), - e(EMAP), + e(VAL_|SCLR|PLAI|PSTR, 153, 3, "doc"), + e(EMAP|PSTR), e(EDOC), e(ESTR), }), @@ -540,16 +550,16 @@ fold: > e(BDOC), e(VAL_|BMAP|BLCK), e(KEY_|SCLR|PLAI, 0, 5, "plain"), - e(VAL_|SCLR|PLAI, 7, 10, "well a b c"), - e(KEY_|SCLR|PLAI, 24, 4, "squo"), - e(VAL_|SCLR|SQUO, 31, 12, "single'quote", needs_filter), - e(KEY_|SCLR|PLAI, 46, 4, "dquo"), - e(VAL_|SCLR|DQUO, 53, 4, "x\t\ny", needs_filter), - e(KEY_|SCLR|PLAI, 61, 3, "lit"), - e(VAL_|SCLR|LITL, 68, 6, "X\nY\nZ\n", needs_filter), - e(KEY_|SCLR|PLAI, 89, 4, "fold"), - e(VAL_|SCLR|FOLD, 97, 6, "U V W\n", needs_filter), - e(EMAP), + e(VAL_|SCLR|PLAI|PSTR, 7, 10, "well a b c"), + e(KEY_|SCLR|PLAI|PSTR, 24, 4, "squo"), + e(VAL_|SCLR|SQUO|PSTR, 31, 12, "single'quote", needs_filter), + e(KEY_|SCLR|PLAI|PSTR, 46, 4, "dquo"), + e(VAL_|SCLR|DQUO|PSTR, 53, 4, "x\t\ny", needs_filter), + e(KEY_|SCLR|PLAI|PSTR, 61, 3, "lit"), + e(VAL_|SCLR|LITL|PSTR, 68, 6, "X\nY\nZ\n", needs_filter), + e(KEY_|SCLR|PLAI|PSTR, 89, 4, "fold"), + e(VAL_|SCLR|FOLD|PSTR, 97, 6, "U V W\n", needs_filter), + e(EMAP|PSTR), e(EDOC), e(ESTR), }), @@ -560,7 +570,7 @@ fold: > e(BDOC), e(VAL_|BSEQ|BLCK), e(VAL_|TAG_, 2, 5, "!!seq"), - e(VAL_|BSEQ|FLOW), + e(VAL_|BSEQ|FLOW|PSTR), e(ESEQ), e(ESEQ), e(EDOC), @@ -578,13 +588,13 @@ fold: > e(BSTR), e(BDOC), e(VAL_|BMAP|BLCK), - e(KEY_|SCLR|PLAI, 0, 28, "defn run(prompt session=nil)"), - e(VAL_|BMAP|BLCK), - e(KEY_|SCLR|PLAI, 32, 12, "when session"), - e(VAL_|BMAP|BLCK), - e(KEY_|SCLR|PLAI, 50, 28, "write session _ :append true"), - e(VAL_|SCLR|LITL, 83, 54, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n", needs_filter), - e(EMAP), + e(KEY_|SCLR|PLAI, 0, 28, "defn run(prompt session=nil)"), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 32, 12, "when session"), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 50, 28, "write session _ :append true"), + e(VAL_|SCLR|LITL|PSTR, 83, 54, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n", needs_filter), + e(EMAP|PSTR), e(EMAP), e(EMAP), e(EDOC), @@ -623,42 +633,42 @@ defn run(prompt session=nil): e(BDOC), e(VAL_|BMAP|BLCK), e(KEY_|SCLR|PLAI, 21, 28, "defn run(prompt session=nil)"), - e(VAL_|BMAP|BLCK), + e(VAL_|BMAP|BLCK|PSTR), e(KEY_|SCLR|PLAI, 53, 14, "session-text ="), - e(VAL_|BMAP|BLCK), + e(VAL_|BMAP|BLCK|PSTR), e(KEY_|SCLR|PLAI, 73, 28, "when session && session:fs-e"), - e(VAL_|SCLR|PLAI, 0, 0, ""), // note empty scalar pointing at the front - e(EMAP), + e(VAL_|SCLR|PLAI|PSTR, 0, 0, ""), // note empty scalar pointing at the front + e(EMAP|PSTR), e(KEY_|SCLR|PLAI, 106, 8, "answer ="), - e(VAL_|BMAP|BLCK), + e(VAL_|BMAP|BLCK|PSTR), e(KEY_|SCLR|PLAI, 120, 4, "cond"), - e(VAL_|BMAP|BLCK), + e(VAL_|BMAP|BLCK|PSTR), e(KEY_|SCLR|PLAI, 132, 22, "api-model =~ /^dall-e/"), - e(VAL_|SCLR|PLAI, 164, 31, "openai-image(prompt).data.0.url"), - e(KEY_|SCLR|PLAI, 202, 31, "api-model.in?(anthropic-models)"), - e(VAL_|SCLR|PLAI, 243, 42, "anthropic(prompt):anthropic-message:format"), - e(KEY_|SCLR|PLAI, 292, 26, "api-model.in?(groq-models)"), - e(VAL_|SCLR|PLAI, 328, 45, "groq(prompt).choices.0.message.content:format"), - e(KEY_|SCLR|PLAI, 380, 28, "api-model.in?(openai-models)"), - e(VAL_|SCLR|PLAI, 418, 52, "openai-chat(prompt).choices.0.message.content:format"), - e(KEY_|SCLR|PLAI, 477, 4, "else"), - e(VAL_|SCLR|PLAI, 483, 5, "die()"), - e(EMAP), + e(VAL_|SCLR|PLAI|PSTR, 164, 31, "openai-image(prompt).data.0.url"), + e(KEY_|SCLR|PLAI|PSTR, 202, 31, "api-model.in?(anthropic-models)"), + e(VAL_|SCLR|PLAI|PSTR, 243, 42, "anthropic(prompt):anthropic-message:format"), + e(KEY_|SCLR|PLAI|PSTR, 292, 26, "api-model.in?(groq-models)"), + e(VAL_|SCLR|PLAI|PSTR, 328, 45, "groq(prompt).choices.0.message.content:format"), + e(KEY_|SCLR|PLAI|PSTR, 380, 28, "api-model.in?(openai-models)"), + e(VAL_|SCLR|PLAI|PSTR, 418, 52, "openai-chat(prompt).choices.0.message.content:format"), + e(KEY_|SCLR|PLAI|PSTR, 477, 4, "else"), + e(VAL_|SCLR|PLAI|PSTR, 483, 5, "die()"), + e(EMAP|PSTR), e(EMAP), e(KEY_|SCLR|PLAI, 492, 3, "say"), - e(VAL_|SCLR|PLAI, 497, 6, "answer"), - e(KEY_|SCLR|PLAI, 507, 12, "when session"), - e(VAL_|BMAP|BLCK), + e(VAL_|SCLR|PLAI|PSTR, 497, 6, "answer"), + e(KEY_|SCLR|PLAI|PSTR, 507, 12, "when session"), + e(VAL_|BMAP|BLCK|PSTR), e(KEY_|SCLR|PLAI, 525, 28, "write session _ :append true"), - e(VAL_|SCLR|LITL, 558, 55, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n", needs_filter), - e(EMAP), + e(VAL_|SCLR|LITL|PSTR, 558, 55, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n", needs_filter), + e(EMAP|PSTR), e(EMAP), e(EMAP), e(EDOC), e(ESTR), }), }; -#define tcf(...) TestCaseErr(__VA_ARGS__) +#define tcf(...) TestCaseErr(__LINE__, __VA_ARGS__) const TestCaseErr test_cases_err[] = { tcf("- !!str, xxx\n", Location(13, 2, 1)), //FIXME tcf(": : : :", Location(2, 1, 3)), @@ -683,23 +693,31 @@ int main(int argc, const char *argv[]) for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) { printf("-----------------------------------------\n" - "case %zu/%zu ...\n" - "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); + "%s:%d: case %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", + __FILE__, test_cases[i].line, + i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); const TestResult tr = test_cases[i].test(ys2evt.ryml2evt); total.add(tr); failed_cases += (!tr); - printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + printf("%s:%d: case %zu/%zu: %s\n", + __FILE__, test_cases[i].line, + i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); } size_t num_cases_err = C4_COUNTOF(test_cases_err); for(size_t i = 0; i < C4_COUNTOF(test_cases_err); ++i) { printf("-----------------------------------------\n" - "errcase %zu/%zu ...\n" - "[%zu]~~~%.*s~~~\n", i, num_cases_err, test_cases_err[i].ys.len, (int)test_cases_err[i].ys.len, test_cases_err[i].ys.str); + "%s:%d: errcase %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", + __FILE__, test_cases[i].line, + i, num_cases_err, test_cases_err[i].ys.len, (int)test_cases_err[i].ys.len, test_cases_err[i].ys.str); const TestResult tr = test_cases_err[i].test(ys2evt.ryml2evt); total.add(tr); failed_cases += (!tr); - printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + printf("%s:%d: case %zu/%zu: %s\n", + __FILE__, test_cases[i].line, + i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); } printf("assertions: %u/%u pass %u/%u fail\n", total.num_assertions - total.num_failed_assertions, total.num_assertions, total.num_failed_assertions, total.num_assertions); printf("tests: %u/%u pass %u/%u fail\n", total.num_tests - total.num_failed_tests, total.num_tests, total.num_failed_tests, total.num_tests); From 0cdc1d49f0cbb607ac8008e81e0b80547cb42a4d Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 30 Sep 2025 19:11:24 +0100 Subject: [PATCH 27/29] int events from ryml WIP: update java code WIP --- .../src/main/java/org/rapidyaml/Evt.java | 46 +++++++---- .../main/java/org/rapidyaml/Rapidyaml.java | 77 ++++++++++--------- 2 files changed, 73 insertions(+), 50 deletions(-) diff --git a/rapidyaml/src/main/java/org/rapidyaml/Evt.java b/rapidyaml/src/main/java/org/rapidyaml/Evt.java index 5305610be..90b0ec43e 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Evt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Evt.java @@ -14,30 +14,48 @@ public class Evt { public static final int ALIA = 1 << 9; // =ALI // Style flags - public static final int PLAI = 1 << 16; // : (plain scalar) - public static final int SQUO = 1 << 17; // ' (single-quoted scalar) - public static final int DQUO = 1 << 18; // " (double-quoted scalar) - public static final int LITL = 1 << 19; // | (block literal scalar) - public static final int FOLD = 1 << 20; // > (block folded scalar) + public static final int PLAI = 1 << 10; // : (plain scalar) + public static final int SQUO = 1 << 11; // ' (single-quoted scalar) + public static final int DQUO = 1 << 12; // " (double-quoted scalar) + public static final int LITL = 1 << 13; // | (block literal scalar) + public static final int FOLD = 1 << 14; // > (block folded scalar) - public static final int FLOW = 1 << 21; // flow container: + public static final int FLOW = 1 << 15; // flow container: // [] for seqs or {} for maps - public static final int BLCK = 1 << 22; // block container + public static final int BLCK = 1 << 16; // block container // Modifiers - public static final int ANCH = 1 << 24; // anchor - public static final int TAG_ = 1 << 25; // tag + public static final int ANCH = 1 << 17; // anchor + public static final int TAG_ = 1 << 18; // tag // Structure flags - public static final int KEY_ = 1 << 26; // as key - public static final int VAL_ = 1 << 27; // as value - public static final int EXPL = 1 << 28; // --- (with BDOC) or + public static final int KEY_ = 1 << 19; // as key + public static final int VAL_ = 1 << 20; // as value + public static final int EXPL = 1 << 21; // --- (with BDOC) or // ... (with EDOC) // (may be fused with FLOW // if needed) - // Utility flags - public static final int LAST = EXPL; + // Buffer flags + ///< IMPORTANT. Marks events whose string was placed in the + ///< arena. Fhis happens when the filtered string is larger than the + ///< original string in the YAML code (eg from tags that resolve to + ///< a larger string, or from "\L" or "\P" in double quotes, which + ///< expand from two to three bytes). Because of this size + ///< expansion, the filtered string cannot be placed in the original + ///< source and needs to be placed in the arena. + public static final int AREN = 1 << 22; + ///< special flag to enable look back in the event array. it + ///< signifies that the previous event has a string, meaning that + ///< the jump back to that event is 3 positions. without this flag it + ///< would be impossible to jump to the previous event + public static final int PSTR = (1 << 23); + ///< special flag to mark a scalar as unfiltered (when the parser + ///< is set not to filter) + public static final int UNFILT = (1 << 24); + + // Utility flags/masks + public static final int LAST = UNFILT; public static final int MASK = ((LAST << 1) - 1); public static final int HAS_STR = SCLR|ALIA|ANCH|TAG_; diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 85502f140..27a3239ef 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -12,22 +12,23 @@ * Interface with the librapidyaml shared library */ public class Rapidyaml { - public static String RAPIDYAML_NAME = "rapidyaml.0.8.0"; - public static String RAPIDYAML_LIBNAME = "librapidyaml.0.8.0.so"; + public static String RAPIDYAML_NAME = "ysparse.0.9.0"; + public static String RAPIDYAML_LIBNAME = "ysparse.0.9.0.so"; + private final long ysparse; ///< pointer to the c++ object private native long ysparse_init(); private native void ysparse_destroy(long ysparse); private native void ysparse_timing_set(boolean yes); - private native int ysparse_parse( - long ysparse, String filename, - byte[] ys, int ys_length, - int[] evt, int evt_length); - private native int ysparse_parse_buf( - long ysparse, String filename, - ByteBuffer ys, int ys_length, - IntBuffer evt, int evt_length); - private final long ysparse; - + private native boolean ysparse_parse(long ysparse, String filename, + byte[] ys, int ys_length, + byte[] arena, int arena_length, + int[] evt, int evt_length); + private native boolean ysparse_parse_buf(long ysparse, String filename, + ByteBuffer ys, int ys_length, + ByteBuffer arena, int arena_length, + IntBuffer evt, int evt_length); + private native int ysparse_reqsize_evt(long ysparse); + private native int ysparse_reqsize_arena(long ysparse); // XXX this 'main' is for testing @@ -72,34 +73,32 @@ public Rapidyaml() throws Exception, IOException { // EVT //------------------------ - public int parseYsToEvt( - byte[] src, int[] evts - ) throws Exception { - return parseYsToEvt("yamlscript", src, evts); + public boolean parseYs(byte[] src, byte[] arena, int[] evts) throws Exception + { + return parseYs("yamlscript", src, arena, evts); } - public int parseYsToEvtBuf( - ByteBuffer src, IntBuffer evt - ) throws Exception { - return parseYsToEvtBuf("yamlscript", src, evt); + public boolean parseYsDirect(ByteBuffer src, ByteBuffer arena, IntBuffer evt) throws Exception + { + return parseYsDirect("yamlscript", src, arena, evt); } - public int parseYsToEvt( - String filename, byte[] src, int[] evts - ) throws Exception { + public boolean parseYs(String filename, byte[] src, byte[] arena, int[] evts) throws Exception + { long t = timingStart("ysparse"); - int required_size = ysparse_parse( - this.ysparse, filename, - src, src.length, evts, evts.length); + boolean fits_buffers = ysparse_parse(this.ysparse, filename, + src, src.length, + arena, arena.length, + evts, evts.length); timingStop("ysparse", t, src.length); - return required_size; + return fits_buffers; } - public int parseYsToEvtBuf( - String filename, ByteBuffer src, IntBuffer evt - ) throws Exception { + public boolean parseYsDirect(String filename, ByteBuffer src, ByteBuffer arena, IntBuffer evt) throws Exception { if (! src.isDirect()) throw new RuntimeException("src must be direct"); + if (! arena.isDirect()) + throw new RuntimeException("arena must be direct"); if (! evt.isDirect()) throw new RuntimeException("evt must be direct"); // the byte order for src does not matter @@ -108,15 +107,21 @@ public int parseYsToEvtBuf( throw new RuntimeException("evt byte order must be native"); long t = timingStart("ysparseBuf"); evt.position(evt.capacity()); - int reqsize = ysparse_parse_buf( - this.ysparse, filename, - src, src.position(), evt, evt.capacity()); - if (reqsize <= evt.capacity()) - evt.position(reqsize); + boolean fits_buffers = ysparse_parse_buf(this.ysparse, filename, + src, src.position(), + arena, arena.position(), + evt, evt.capacity()); + if (fits_buffers) + evt.position(ysparse_reqsize_evt(this.ysparse)); timingStop("ysparseBuf", t, src.position()); - return reqsize; + return fits_buffers; } + /** Get the required size for the event output buffer, from the last parse call */ + public int reqsizeEvt() { return ysparse_reqsize_evt(this.ysparse); } + /** Get the required size for the arena buffer, from the last parse call */ + public int reqsizeArena() { return ysparse_reqsize_arena(this.ysparse); } + public static IntBuffer mkIntBuffer(int numInts) { ByteBuffer bb = ByteBuffer.allocateDirect(/*numBytes*/4 * numInts); // !!! need to explicitly set the byte order to the native order From 4996c6266b1e278bc7d306597f00ab1be1e6add7 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 30 Sep 2025 22:10:43 +0100 Subject: [PATCH 28/29] fix maven --- common/vars.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/vars.mk b/common/vars.mk index e6eb3fd33..c241abb06 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -169,7 +169,7 @@ endif # Set MAVEN variables: #------------------------------------------------------------------------------ -MAVEN_VER := 3.9.6 +MAVEN_VER := 3.9.11 MAVEN_SRC := https://dlcdn.apache.org/maven/maven-3/$(MAVEN_VER)/binaries MAVEN_TAR := apache-maven-$(MAVEN_VER)-bin.tar.gz MAVEN_URL := $(MAVEN_SRC)/$(MAVEN_TAR) From 831e9e2c3435f50f4d407d3c4ca93c0aef523538 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 30 Sep 2025 23:33:17 +0100 Subject: [PATCH 29/29] java WIP --- .../src/main/java/org/rapidyaml/Evt.java | 21 +++- .../main/java/org/rapidyaml/Rapidyaml.java | 9 +- .../main/java/org/rapidyaml/cmp/CmpEvt.java | 104 ------------------ .../main/java/org/rapidyaml/cmp/manifest.mf | 1 - .../src/main/java/org/rapidyaml/cmp/run.sh | 22 ---- .../java/org/rapidyaml/RapidyamlTest.java | 79 +++++++------ 6 files changed, 67 insertions(+), 169 deletions(-) delete mode 100644 rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java delete mode 100644 rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf delete mode 100755 rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh diff --git a/rapidyaml/src/main/java/org/rapidyaml/Evt.java b/rapidyaml/src/main/java/org/rapidyaml/Evt.java index 90b0ec43e..0e156efe9 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Evt.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Evt.java @@ -36,6 +36,11 @@ public class Evt { // (may be fused with FLOW // if needed) + // Directives + public static final int YAML = 1 << 22; // `%YAML ` followed by version string + public static final int TAGD = 1 << 23; // tag directive name : `%TAG .......` followed by name string + public static final int TAGV = 1 << 24; // tag directive value: `%TAG ...... ` followed by value string + // Buffer flags ///< IMPORTANT. Marks events whose string was placed in the ///< arena. Fhis happens when the filtered string is larger than the @@ -44,19 +49,23 @@ public class Evt { ///< expand from two to three bytes). Because of this size ///< expansion, the filtered string cannot be placed in the original ///< source and needs to be placed in the arena. - public static final int AREN = 1 << 22; + public static final int AREN = 1 << 25; ///< special flag to enable look back in the event array. it ///< signifies that the previous event has a string, meaning that ///< the jump back to that event is 3 positions. without this flag it ///< would be impossible to jump to the previous event - public static final int PSTR = (1 << 23); + public static final int PSTR = 1 << 26; ///< special flag to mark a scalar as unfiltered (when the parser ///< is set not to filter) - public static final int UNFILT = (1 << 24); + public static final int UNFILT = 1 << 27; // Utility flags/masks - public static final int LAST = UNFILT; - public static final int MASK = ((LAST << 1) - 1); - public static final int HAS_STR = SCLR|ALIA|ANCH|TAG_; + public static final int LAST = UNFILT; ///< the last flag defined above + public static final int MASK = (LAST << 1) - 1; ///< a mask of all bits in this enumeration + /// with string: mask of all the events that encode a string + /// following the event. in the event has a string. the next two + /// integers will provide respectively the string's offset and + /// length. See also @ref PSTR. + public static final int WSTR = SCLR|ALIA|ANCH|TAG_|TAGD|TAGV|YAML; } diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index 27a3239ef..c9cd8048d 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -101,8 +101,8 @@ public boolean parseYsDirect(String filename, ByteBuffer src, ByteBuffer arena, throw new RuntimeException("arena must be direct"); if (! evt.isDirect()) throw new RuntimeException("evt must be direct"); - // the byte order for src does not matter - // but for evt it really does + // the byte order for src+arena does not matter + // but for evt it really does: if (evt.order() != ByteOrder.nativeOrder()) throw new RuntimeException("evt byte order must be native"); long t = timingStart("ysparseBuf"); @@ -118,10 +118,11 @@ public boolean parseYsDirect(String filename, ByteBuffer src, ByteBuffer arena, } /** Get the required size for the event output buffer, from the last parse call */ - public int reqsizeEvt() { return ysparse_reqsize_evt(this.ysparse); } + public int requiredSizeEvt() { return ysparse_reqsize_evt(this.ysparse); } /** Get the required size for the arena buffer, from the last parse call */ - public int reqsizeArena() { return ysparse_reqsize_arena(this.ysparse); } + public int requiredSizeArena() { return ysparse_reqsize_arena(this.ysparse); } + /** a helper to create a direct int buffer */ public static IntBuffer mkIntBuffer(int numInts) { ByteBuffer bb = ByteBuffer.allocateDirect(/*numBytes*/4 * numInts); // !!! need to explicitly set the byte order to the native order diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java b/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java deleted file mode 100644 index 19b6fab7c..000000000 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/CmpEvt.java +++ /dev/null @@ -1,104 +0,0 @@ -package cmp; - -import org.rapidyaml.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; - -// https://stackoverflow.com/questions/804466/how-do-i-create-executable-java-program -public class CmpEvt -{ - public static void main(String[] args) throws Exception - { - Rapidyaml rapidyaml = new Rapidyaml(); - rapidyaml.timingEnabled(true); - compareEvt(rapidyaml, "./yamllm.ys"); - compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/appveyor.yml"); - compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/compile_commands.json"); - compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner100.yml"); - compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner100.yml"); - compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000.yml"); - compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000.yml"); - compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_seqs_flow_outer1000_inner1000_json.json"); - compareEvt(rapidyaml, "/home/jpmag/proj/rapidyaml/bm/cases/style_maps_flow_outer1000_inner1000_json.yml"); - } - - public static void compareEvt(Rapidyaml rapidyaml, String path) throws Exception - { - String ys_ = java.nio.file.Files.readString(Paths.get(path), StandardCharsets.UTF_8); - byte[] ys = ys_.getBytes(StandardCharsets.UTF_8); - byte[] ysarr = new byte[ys.length]; - ByteBuffer ysbuf = ByteBuffer.allocateDirect(ys.length); - // - System.out.printf("-----\n"); - System.out.printf("%s\n", path); - System.out.printf(" ys.length=%d\n", ys.length); - // - long t = timingStart("evt"); - int[] evtarr = callEvt(rapidyaml, ys, ysarr); - timingStop("evt", t, ys.length); - // - t = timingStart("evtBuf"); - IntBuffer evtbuf = callEvtBuf(rapidyaml, ys, ysbuf); - timingStop("evtBuf", t, ys.length); - } - - static int[] callEvt(Rapidyaml rapidyaml, byte[] src, byte[] srcbuf) throws Exception - { - System.arraycopy(src, 0, srcbuf, 0, src.length); - int[] evt = new int[10000000]; - int reqsize = rapidyaml.parseYsToEvt(srcbuf, evt); - if(reqsize > evt.length) { - evt = new int[reqsize]; - System.arraycopy(src, 0, srcbuf, 0, src.length); - int reqsize2 = rapidyaml.parseYsToEvt(srcbuf, evt); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - return evt; - } - int[] ret = new int[reqsize]; - System.arraycopy(evt, 0, ret, 0, reqsize); - return ret; - } - - static IntBuffer callEvtBuf(Rapidyaml rapidyaml, byte[] src, ByteBuffer srcbuf) throws Exception - { - srcbuf.position(0); - srcbuf.put(src); - IntBuffer evt = Rapidyaml.mkIntBuffer(10000000); - int reqsize = rapidyaml.parseYsToEvtBuf(srcbuf, evt); - if(reqsize > evt.capacity()) { - evt = Rapidyaml.mkIntBuffer(reqsize); - srcbuf.position(0); - srcbuf.put(src); - int reqsize2 = rapidyaml.parseYsToEvtBuf(srcbuf, evt); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - } - evt.position(reqsize); - return evt; - } - - static private long timingStart(String name) - { - System.out.printf(" call:%s...\n", name); - return System.nanoTime(); - } - static private void timingStop(String name, long t) - { - t = System.nanoTime() - t; - System.out.printf(" call:%s: %.6fms\n", name, (float)t/1.e6f); - } - static private void timingStop(String name, long t, int numBytes) - { - t = System.nanoTime() - t; - float dt = (float)t; - float fb = (float)numBytes; - System.out.printf(" call:%s: %.6fms %.3fMB/s %dB\n", name, dt/1.e6f, 1.e3f*fb/dt, numBytes); - } -} diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf b/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf deleted file mode 100644 index aa44b6579..000000000 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/manifest.mf +++ /dev/null @@ -1 +0,0 @@ -Main-class: cmp.CmpEvt diff --git a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh b/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh deleted file mode 100755 index 4f655cfe2..000000000 --- a/rapidyaml/src/main/java/org/rapidyaml/cmp/run.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -xe - -thisdir=$(dirname $0) -nativedir=$(cd $thisdir/../../../../../../native ; pwd) -rymldir=$(cd $nativedir/.. ; pwd) - -make -C $nativedir build RAPIDYAML_TIMED=1 -make -C $rymldir test RAPIDYAML_TIMED=1 - - -cd $thisdir -if [ ! -f yamllm.ys ] ; then - wget https://raw.githubusercontent.com/yaml/yamllm/refs/heads/main/bin/yamllm.ys -fi -ls -lFhp -jd=${jd:-/usr/lib/jvm/java-23-openjdk/bin} -$jd/javac -d . ../*.java -$jd/javac -d . -cp . CmpEvt.java -$jd/jar -cmf manifest.mf CmpEvt.jar cmp org -$jd/java -jar -Djava.library.path=$nativedir CmpEvt.jar diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index 1da786510..d32bb7bcc 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -166,18 +166,18 @@ public void testTaggedSeq() }; testEvt_(ys, expected); } - + public void testDocTag() { String ys = "!yamlscript/v0/bare\n--- !code\n42\n"; ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC), - mkev(Evt.VAL_|Evt.TAG_, 1, 18, "yamlscript/v0/bare"), + mkev(Evt.VAL_|Evt.TAG_, 0, 19, "!yamlscript/v0/bare"), mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 0, 0, ""), mkev(Evt.EDOC), mkev(Evt.BDOC|Evt.EXPL), - mkev(Evt.VAL_|Evt.TAG_, 25, 4, "code"), + mkev(Evt.VAL_|Evt.TAG_, 24, 5, "!code"), mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 30, 2, "42"), mkev(Evt.EDOC), mkev(Evt.ESTR), @@ -207,7 +207,7 @@ public void testLargeCase() ExpectedEvent[] expected = { mkev(Evt.BSTR), mkev(Evt.BDOC|Evt.EXPL), - mkev(Evt.VAL_|Evt.TAG_, 5, 13, "yamlscript/v0"), + mkev(Evt.VAL_|Evt.TAG_, 4, 14, "!yamlscript/v0"), mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), mkev(Evt.VAL_|Evt.TAG_, 25, 0, ""), @@ -248,7 +248,7 @@ public void testLargeCase() }; testEvt_(ys, expected); } - + public void testFilterCase() { String ys = "" + @@ -286,16 +286,17 @@ public void testFilterCase() }; testEvt_(ys, expected); } - + public void testFailure() throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); String ys = "{a: b"; byte[] src = ys.getBytes(StandardCharsets.UTF_8); + byte[] arena = new byte[src.length]; byte[] srcbuf = new byte[src.length]; boolean gotit = false; try { - callEvt(src, srcbuf); + callParse(src, srcbuf, arena); } catch(YamlParseErrorException e) { gotit = true; @@ -313,17 +314,18 @@ public void testFailure() throws Exception } assertTrue(gotit); } - + public void testFailureBuf() throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); String ys = "{a: b"; byte[] src = ys.getBytes(StandardCharsets.UTF_8); + ByteBuffer arena = ByteBuffer.allocateDirect(src.length); ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); bbuf.put(src); boolean gotit = false; try { - callEvtBuf(src, bbuf); + callParseDirect(src, bbuf, arena); } catch(YamlParseErrorException e) { gotit = true; @@ -347,9 +349,10 @@ private void testEvt_(String ys, ExpectedEvent[] expected) { byte[] src = ys.getBytes(StandardCharsets.UTF_8); byte[] srcbuf = new byte[src.length]; + byte[] arenaarr = new byte[src.length]; int[] actual; try { - actual = callEvt(src, srcbuf); + actual = callParse(src, srcbuf, arenaarr); } catch (Exception e) { fail("parse error:\n" + e.getMessage()); @@ -364,11 +367,12 @@ private void testEvt_(String ys, ExpectedEvent[] expected) } //------ src = ys.getBytes(StandardCharsets.UTF_8); + ByteBuffer arenabuf = ByteBuffer.allocateDirect(src.length); ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); bbuf.put(src); IntBuffer buf; try { - buf = callEvtBuf(src, bbuf); + buf = callParseDirect(src, bbuf, arenabuf); actual = buf2arr(buf); } catch (Exception e) { @@ -470,43 +474,54 @@ public static int[] buf2arr(IntBuffer evt) return ret; } - static int[] callEvt(byte[] src, byte[] srcbuf) throws Exception + static int[] callParse(byte[] src, byte[] srcbuf, byte[] arena) throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); System.arraycopy(src, 0, srcbuf, 0, src.length); int[] evt = new int[10000]; - int reqsize = rapidyaml.parseYsToEvt(srcbuf, evt); - if(reqsize > evt.length) { - evt = new int[reqsize]; + boolean fitsBuffers = rapidyaml.parseYs(srcbuf, arena, evt); + int reqsizeInts = rapidyaml.requiredSizeEvt(); + int reqsizeArena = rapidyaml.requiredSizeArena(); + if(!fitsBuffers) { + if(reqsizeInts > evt.length) evt = new int[reqsizeInts]; + if(reqsizeArena > arena.length) throw new RuntimeException("resize arena"); System.arraycopy(src, 0, srcbuf, 0, src.length); - int reqsize2 = rapidyaml.parseYsToEvt(srcbuf, evt); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } - return evt; + boolean fitsBuffers2 = rapidyaml.parseYs(srcbuf, arena, evt); + int reqsizeInts2 = rapidyaml.requiredSizeEvt(); + int reqsizeArena2 = rapidyaml.requiredSizeArena(); + if(fitsBuffers2 != fitsBuffers) throw new RuntimeException("fitsBuffers"); + if(reqsizeInts2 != reqsizeInts) throw new RuntimeException("reqsizeInts"); + if(reqsizeArena2 != reqsizeArena) throw new RuntimeException("reqsizeArena"); } - int[] ret = new int[reqsize]; - System.arraycopy(evt, 0, ret, 0, reqsize); + int[] ret = new int[reqsizeInts]; + System.arraycopy(evt, 0, ret, 0, reqsizeInts); return ret; } - static IntBuffer callEvtBuf(byte[] src, ByteBuffer srcbuf) throws Exception + static IntBuffer callParseDirect(byte[] src, ByteBuffer srcbuf, ByteBuffer arena) throws Exception { Rapidyaml rapidyaml = new Rapidyaml(); + arena.position(0); srcbuf.position(0); srcbuf.put(src); IntBuffer evt = Rapidyaml.mkIntBuffer(10000); - int reqsize = rapidyaml.parseYsToEvtBuf(srcbuf, evt); - if(reqsize > evt.capacity()) { - evt = Rapidyaml.mkIntBuffer(reqsize); + boolean fitsBuffers = rapidyaml.parseYsDirect(srcbuf, arena, evt); + int reqsizeInts = rapidyaml.requiredSizeEvt(); + int reqsizeArena = rapidyaml.requiredSizeArena(); + if(!fitsBuffers) { + if(reqsizeInts > srcbuf.position()) evt = Rapidyaml.mkIntBuffer(reqsizeInts); + if(reqsizeArena > arena.capacity()) throw new RuntimeException("resize arena"); + arena.position(0); srcbuf.position(0); srcbuf.put(src); - int reqsize2 = rapidyaml.parseYsToEvtBuf(srcbuf, evt); - if(reqsize2 != reqsize) { - throw new RuntimeException("reqsize"); - } + boolean fitsBuffers2 = rapidyaml.parseYsDirect(srcbuf, arena, evt); + int reqsizeInts2 = rapidyaml.requiredSizeEvt(); + int reqsizeArena2 = rapidyaml.requiredSizeArena(); + if(fitsBuffers2 != fitsBuffers) throw new RuntimeException("fitsBuffers"); + if(reqsizeInts2 != reqsizeInts) throw new RuntimeException("reqsizeInts"); + if(reqsizeArena2 != reqsizeArena) throw new RuntimeException("reqsizeArena"); } - evt.position(reqsize); + evt.position(reqsizeInts); return evt; } @@ -545,7 +560,7 @@ class ExpectedEvent } int required_size() { - return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; + return ((flags & Evt.WSTR) != 0) ? 3 : 1; } public static int required_size_(ExpectedEvent[] evts)