diff --git a/.asf.yaml b/.asf.yaml index 92afea9e4f..3e9abc121d 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -40,6 +40,9 @@ github: strict: true contexts: - continuous-integration/jenkins/pr-merge + 3.5.x: + required_status_checks: + strict: true 3.4.x: required_status_checks: strict: true diff --git a/.elp.toml b/.elp.toml new file mode 100644 index 0000000000..e28ee26aed --- /dev/null +++ b/.elp.toml @@ -0,0 +1,38 @@ +[build_info] +apps = ["src/*"] +# 3rd party dependencies (not type-checked), defaults to [] (see rebar.config.script) +deps = [ + "src/erlfmt", + "src/rebar", + "src/rebar3", + "src/meck", + "src/cowlib", + "src/gun", + "src/recon", + "src/proper", + "src/fauxton", + "src/docs", + "src/meck", + "src/jiffy", + "src/ibrowse", + "src/mochiweb", + "src/snappy" +] +# List of OTP application names to exclude from indexing. This can help improve performance by not loading rarely used OTP apps. +[otp] +exclude_apps = [ + "megaco", + "common_test", + "edoc", + "eldap", + "erl_docgen", + "et", + "ftp", + "mnesia", + "odbc", + "observer", + "snmp", + "tftp", + "wx", + "xmerl" +] diff --git a/.gitignore b/.gitignore index 080a7dd6ff..d46777512b 100644 --- a/.gitignore +++ b/.gitignore @@ -53,10 +53,12 @@ src/couch/priv/couchspawnkillable src/couch/priv/couch_ejson_compare/couch_ejson_compare.d src/couch/priv/couch_js/**/*.d src/couch/priv/icu_driver/couch_icu_driver.d +src/cowlib/ src/mango/src/mango_cursor_text.nocompile src/excoveralls/ src/fauxton/ src/folsom/ +src/gun/ src/hackney/ src/hqueue/ src/ibrowse/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ba7276ba8..f229fc1069 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -302,6 +302,19 @@ hub checkout link-to-pull-request meaning that you will automatically check out the branch for the pull request, without needing any other steps like setting git upstreams! :sparkles: +## Artificial Intelligence and Large Language Models Contributions Policy + +The CouchDB project has a long-standing focus on license compatibility, and +appropriate attribution of source code. AI and LLMs, by their nature, are unable +to provide the necessary assurance, that the generated material is compatible +with the Apache 2 license, or that the material has been appropriately +attributed to the original authors. + +Thus, it is expressly forbidden to contribute material generated by AI, LLMs, +and similar technologies, to the CouchDB project. This includes, but is not +limited to, source code, documentation, commit messages, or any other areas of +the project. + ## Thanks Special thanks to [Hoodie][#gh_hoodie] for the great diff --git a/INSTALL.Unix.md b/INSTALL.Unix.md index a078897987..a27ff626a0 100644 --- a/INSTALL.Unix.md +++ b/INSTALL.Unix.md @@ -1,6 +1,6 @@ # Apache CouchDB INSTALL.Unix -A high-level guide to Unix-like systems, inc. Mac OS X and Ubuntu. +A high-level guide to Unix-like systems, inc. macOS and Ubuntu. Community installation guides are available on the wiki: @@ -84,9 +84,9 @@ You can install the dependencies by running: You can install Node.JS via [NodeSource](https://github.com/nodesource/distributions#rpminstall). -### Mac OS X +### macOS -To build CouchDB from source on Mac OS X, you will need to install +To build CouchDB from source on macOS, you will need to install the Command Line Tools: xcode-select --install @@ -96,6 +96,12 @@ You can then install the other dependencies by running: brew install autoconf autoconf-archive automake libtool \ erlang icu4c spidermonkey pkg-config +Note that newer versions of Homebrew install `icu4c` as “keg-only”. +That means the CouchDB build system can’t find it. Either follow +the instructions presented by Homebrew or run + + brew link icu4c + You can install Node.JS via the [official Macintosh installer](https://nodejs.org/en/download/). @@ -103,9 +109,9 @@ You will need Homebrew installed to use the `brew` command. Learn more about Homebrew at: - http://mxcl.github.com/homebrew/ + https://brew.sh -Some versions of Mac OS X ship a problematic OpenSSL library. If +Some versions of macOS ship a problematic OpenSSL library. If you're experiencing troubles with CouchDB crashing intermittently with a segmentation fault or a bus error, you will need to install your own version of OpenSSL. See the wiki, mentioned above, for more information. @@ -169,7 +175,7 @@ On many Unix-like systems you can run: --group --gecos \ "CouchDB Administrator" couchdb -On Mac OS X you can use the Workgroup Manager to create users up to version +On macOS you can use the Workgroup Manager to create users up to version 10.9, and dscl or sysadminctl after version 10.9. Search Apple's support site to find the documentation appropriate for your system. As of recent versions of OS X, this functionality is also included in Server.app, diff --git a/Makefile b/Makefile index 8b1df51e65..6847037df7 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,10 @@ EXUNIT_OPTS=$(subst $(comma),$(space),$(tests)) TEST_OPTS="-c 'startup_jitter=0' -c 'default_security=admin_local' -c 'iterations=9'" +ifneq ($(ERLANG_COOKIE),) +TEST_OPTS+=" --erlang-cookie=$(ERLANG_COOKIE)" +endif + ################################################################################ # Main commands ################################################################################ @@ -350,6 +354,12 @@ weatherreport-test: devclean escriptize @dev/run "$(TEST_OPTS)" -n 1 -a adm:pass --no-eval \ 'bin/weatherreport --etc dev/lib/node1/etc --level error' +.PHONY: quickjs-test262 +# target: quickjs-javascript-tests - Run QuickJS JS conformance tests +quickjs-test262: couch + make -C src/couch_quickjs/quickjs test2-bootstrap + make -C src/couch_quickjs/quickjs test2 + ################################################################################ # Developing ################################################################################ @@ -474,7 +484,12 @@ clean: @rm -rf .rebar/ @rm -f bin/couchjs @rm -f bin/weatherreport - @rm -rf src/*/ebin + @find src/*/ebin \ + -not -path 'src/cowlib/ebin/cowlib.app' \ + -not -path 'src/cowlib/ebin' \ + -not -path 'src/gun/ebin/gun.app' \ + -not -path 'src/gun/ebin' \ + -delete @rm -rf src/*/.rebar @rm -rf src/*/priv/*.so @rm -rf share/server/main.js share/server/main-ast-bypass.js share/server/main-coffee.js @@ -482,7 +497,7 @@ clean: @rm -rf src/mango/.venv @rm -f src/couch/priv/couch_js/config.h @rm -f dev/*.beam dev/devnode.* dev/pbkdf2.pyc log/crash.log - @rm -f src/couch_dist/certs/out + @rm -rf src/couch_dist/certs/out @rm -rf src/docs/build src/docs/.venv ifeq ($(with_nouveau), true) @cd nouveau && $(GRADLE) clean diff --git a/Makefile.win b/Makefile.win index 000874b70b..d510ee51df 100644 --- a/Makefile.win +++ b/Makefile.win @@ -93,6 +93,10 @@ EXUNIT_OPTS=$(subst $(comma),$(space),$(tests)) TEST_OPTS="-c startup_jitter=0 -c default_security=admin_local -c iterations=9" +ifneq ($(ERLANG_COOKIE),) +TEST_OPTS+=" --erlang-cookie=$(ERLANG_COOKIE)" +endif + ################################################################################ # Main commands ################################################################################ diff --git a/README-DEV.rst b/README-DEV.rst index 1f0711b4a1..d6dcc4a558 100644 --- a/README-DEV.rst +++ b/README-DEV.rst @@ -111,8 +111,8 @@ Centos 7 and RHEL 7 python-pygments gnupg nodejs npm -Mac OS X -~~~~~~~~ +macOS +~~~~~ Install `Homebrew `_, if you do not have it already. @@ -120,7 +120,7 @@ Unless you want to install the optional dependencies, skip to the next section. Install what else we can with Homebrew:: - brew install help2man gnupg md5sha1sum node python + brew install help2man gnupg md5sha1sum node python elixir If you don't already have pip installed, install it:: @@ -311,9 +311,9 @@ follows:: dev/run --with-clouseau -When a specific Erlang cookie string is set in -``rel/overlay/etc/vm.args``, the ``--erlang-cookie`` flag could be -used to configure Clouseau to work with that:: +When a specific Erlang cookie string is needed, the +``--erlang-cookie`` flag could be used to configure CouchDB and +Clouseau to work with that:: dev/run --with-clouseau --erlang-cookie=brumbrum diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile similarity index 65% rename from build-aux/Jenkinsfile.full rename to build-aux/Jenkinsfile index 479aec7ea2..dfba236c7b 100644 --- a/build-aux/Jenkinsfile.full +++ b/build-aux/Jenkinsfile @@ -13,12 +13,24 @@ // License for the specific language governing permissions and limitations under // the License. -// Erlang version embedded in binary packages -ERLANG_VERSION = '26.2.5.11' +// This is image used to build the tarball, check source formatting and build +// the docs +DOCKER_IMAGE_BASE = 'apache/couchdbci-debian:bookworm-erlang' + +// Erlang version embedded in binary packages. Also the version most builds +// will run. +ERLANG_VERSION = '26.2.5.15' // Erlang version used for rebar in release process. CouchDB will not build from // the release tarball on Erlang versions older than this -MINIMUM_ERLANG_VERSION = '25.3.2.20' +MINIMUM_ERLANG_VERSION = '26.2.5.15' + +// Highest support Erlang version. +MAXIMUM_ERLANG_VERSION = '28.0.4' + +// Use these to detect if just documents changed +docs_changed = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q '^src/docs/'" +other_changes = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q -v '^src/docs/'" // We create parallel build / test / package stages for each OS using the metadata // in this map. Adding a new OS should ideally only involve adding a new entry here. @@ -28,6 +40,7 @@ meta = [ spidermonkey_vsn: '60', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}" ], @@ -36,22 +49,16 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-centos:9-erlang-${ERLANG_VERSION}" ], - 'focal': [ - name: 'Ubuntu 20.04', - spidermonkey_vsn: '68', - with_nouveau: true, - with_clouseau: true, - image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}" - ], - 'jammy': [ name: 'Ubuntu 22.04', spidermonkey_vsn: '91', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}" ], @@ -60,48 +67,106 @@ meta = [ spidermonkey_vsn: '115', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-ubuntu:noble-erlang-${ERLANG_VERSION}" ], - 'bookworm-ppc64': [ - name: 'Debian POWER', + 'bullseye': [ + name: 'Debian x86_64', spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, - image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}", - node_label: 'ppc64le' + quickjs_test262: true, + image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" ], - 'bookworm-s390x': [ - name: 'Debian s390x', + // Sometimes we "pick up" ppc64le workers from the asf jenkins intance That + // seems like a good thing, however, those workers allow running multiple + // agents at a time and often time out even with retries. The build times + // take close to an hour, which is at least x2 as long as it takes to run + // other arch CI jobs and they still time out often and fail. Disable for + // now. This is a low demand arch distro, maybe remove support altogether? + // + // 'base-ppc64': [ + // name: 'Debian POWER', + // spidermonkey_vsn: '78', + // with_nouveau: true, + // with_clouseau: true, + // quickjs_test262: true, + // image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}", + // node_label: 'ppc64le' + // ], + + // Just like in the ppc64le case we sometimes "pick up" built-in s390x workers added to + // our jenkins, but since those are managed by us, our .mix/.venv/.hex packaging + // cache hack in /home/jenkins doesn't work, and so elixir tests fail. Skip this arch build + // until we figure out caching (probably need to use a proper caching plugin). + // + // 'base-s390x': [ + // name: 'Debian s390x', + // spidermonkey_vsn: '78', + // with_nouveau: true, + // // QuickJS test262 shows a discrepancy typedarray-arg-set-values-same-buffer-other-type.js + // // Test262Error: 51539607552,42,0,4,5,6,7,8 + // quickjs_test262: false, + // image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}", + // node_label: 's390x' + // ], + + 'base': [ + name: 'Debian x86_64', spidermonkey_vsn: '78', with_nouveau: true, - image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}", - node_label: 's390x' + with_clouseau: true, + // Test this in in the bookworm-quickjs variant + quickjs_test262: false, + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" ], - 'bullseye': [ + 'base-max-erlang': [ name: 'Debian x86_64', spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, - image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" + quickjs_test262: false, + image: "${DOCKER_IMAGE_BASE}-${MAXIMUM_ERLANG_VERSION}" ], - 'bookworm': [ - name: 'Debian x86_64', + 'base-quickjs': [ + name: 'Debian 12 with QuickJS', + disable_spidermonkey: true, + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" + ], + + // This runs on a docker ARM64 host. Normally we should be able to run all + // ubuntu and centos containers on top any docker host, however spidermonkey + // 60 from CentOS cannot build on ARM64 so we're forced to separate it as a + // separate build usable for ubuntu/debian only and isolated it from other + // builds. At some point when we remove CentOS 8 or switch to QuickJS only, + // remove the docker-arm64 label on ubuntu-nc-arm64-12 node in Jenkins and + // remove this flavor + // + 'base-arm64': [ + name: 'Debian ARM64', spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, - image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" + // Test this in in the bookworm-quickjs variant + quickjs_test262: false, + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}", + node_label: 'docker-arm64' ], - 'bookworm-quickjs': [ - name: 'Debian 12 with QuickJS', - disable_spidermonkey: true, + 'trixie': [ + name: 'Debian x86_64', + spidermonkey_vsn: '128', with_nouveau: true, with_clouseau: true, - image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" + quickjs_test262: true, + image: "apache/couchdbci-debian:trixie-erlang-${ERLANG_VERSION}" ], 'freebsd-x86_64': [ @@ -109,6 +174,7 @@ meta = [ spidermonkey_vsn: '91', with_clouseau: true, clouseau_java_home: '/usr/local/openjdk8-jre', + quickjs_test262: false, gnu_make: 'gmake' ], @@ -119,21 +185,27 @@ meta = [ disable_spidermonkey: true, with_clouseau: true, clouseau_java_home: '/usr/local/openjdk8-jre', + quickjs_test262: false, gnu_make: 'gmake' ], - 'macos': [ - name: 'macOS', - spidermonkey_vsn: '128', - with_nouveau: false, - with_clouseau: true, - clouseau_java_home: '/opt/java/openjdk8/zulu-8.jre/Contents/Home', - gnu_make: 'make' - ], + // Disable temporarily. Forks / shell execs seem to fail there currently + // + // + // 'macos': [ + // name: 'macOS', + // spidermonkey_vsn: '128', + // with_nouveau: false, + // with_clouseau: true, + // clouseau_java_home: '/opt/java/openjdk8/zulu-8.jre/Contents/Home', + // gnu_make: 'make' + // ], 'win2022': [ name: 'Windows 2022', spidermonkey_vsn: '128', + with_clouseau: true, + quickjs_test262: false, node_label: 'win' ] ] @@ -170,7 +242,7 @@ def generateNativeStage(platform) { return { stage(platform) { node(platform) { - timeout(time: 90, unit: "MINUTES") { + timeout(time: 180, unit: "MINUTES") { // Steps to configure and build CouchDB on *nix platforms if (isUnix()) { try { @@ -190,12 +262,13 @@ def generateNativeStage(platform) { dir( "${platform}/build" ) { sh "${configure(meta[platform])}" sh '$MAKE' - sh '$MAKE eunit' - sh '$MAKE elixir' - sh '$MAKE elixir-search' - sh '$MAKE mango-test' - sh '$MAKE weatherreport-test' - sh '$MAKE nouveau-test' + retry (3) {sh '$MAKE eunit'} + if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} + retry (3) {sh '$MAKE elixir'} + retry (3) {sh '$MAKE elixir-search'} + retry (3) {sh '$MAKE mango-test'} + retry (3) {sh '$MAKE weatherreport-test'} + retry (3) {sh '$MAKE nouveau-test'} } } } @@ -229,9 +302,11 @@ def generateNativeStage(platform) { powershell( script: "New-Item -ItemType Directory -Path '${platform}/build' -Force", label: 'Create build directories' ) powershell( script: "tar -xf (Get-Item apache-couchdb-*.tar.gz) -C '${platform}/build' --strip-components=1", label: 'Unpack release' ) dir( "${platform}/build" ) { + withClouseau = meta[platform].with_clouseau ? '-WithClouseau' : '' + powershell( script: """ .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 - .\\configure.ps1 -SkipDeps -WithNouveau -SpiderMonkeyVersion ${meta[platform].spidermonkey_vsn} + .\\configure.ps1 -SkipDeps -WithNouveau ${withClouseau} -SpiderMonkeyVersion ${meta[platform].spidermonkey_vsn} Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' make -f Makefile.win release """, label: 'Configure and Build') @@ -239,13 +314,17 @@ def generateNativeStage(platform) { //powershell( script: ".\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win eunit", label: 'EUnit tests') //powershell( script: ".\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win elixir", label: 'Elixir tests') - powershell( script: '& .\\..\\..\\couchdb-glazier\\bin\\shell.ps1; Write-Host "NOT AVAILABLE: make -f Makefile.win elixir-search"', label: 'N/A Clouseau tests') - powershell( script: """ .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' - make -f Makefile.win mango-test - """, label: 'Mango tests') + make -f Makefile.win elixir-search ERLANG_COOKIE=crumbles + """, label: 'Clouseau tests') + + powershell( script: """ + .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 + Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + make -f Makefile.win mango-test ERLANG_COOKIE=crumbles + """, label: 'Mango tests') powershell( script: '.\\..\\..\\couchdb-glazier\\bin\\shell.ps1; Write-Host "NOT AVAILABLE: make -f Makefile.win weatherreport-test"', label: 'N/A Weatherreport tests') @@ -295,17 +374,19 @@ def generateContainerStage(platform) { node(meta[platform].get('node_label', 'docker')) { docker.withRegistry('https://docker.io/', 'dockerhub_creds') { docker.image(meta[platform].image).inside("${DOCKER_ARGS}") { - timeout(time: 90, unit: "MINUTES") { + timeout(time: 180, unit: "MINUTES") { stage("${meta[platform].name} - build & test") { try { sh( script: "rm -rf ${platform} apache-couchdb-*", label: 'Clean workspace' ) unstash 'tarball' sh( script: "mkdir -p ${platform}/build", label: 'Create build directories' ) sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) + quickjs_tests262 = meta[platform].quickjs_test262 dir( "${platform}/build" ) { sh "${configure(meta[platform])}" sh 'make' retry(3) {sh 'make eunit'} + if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} retry(3) {sh 'make elixir'} retry(3) {sh 'make elixir-search'} retry(3) {sh 'make mango-test'} @@ -383,23 +464,161 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) preserveStashes(buildCount: 10) - timeout(time: 3, unit: 'HOURS') + timeout(time: 4, unit: 'HOURS') timestamps() } stages { + + stage('Setup Env') { + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 10, unit: 'MINUTES') + } + steps { + script { + env.DOCS_CHANGED = '0' + env.ONLY_DOCS_CHANGED = '0' + if ( sh(returnStatus: true, script: docs_changed) == 0 ) { + env.DOCS_CHANGED = '1' + if (sh(returnStatus: true, script: other_changes) == 1) { + env.ONLY_DOCS_CHANGED = '1' + } + } + } + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage 'Setup Environment' + + stage('Docs Check') { + // Run docs `make check` stage if any docs changed + when { + beforeOptions true + expression { DOCS_CHANGED == '1' } + } + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 15, unit: 'MINUTES') + } + steps { + sh ''' + make python-black + ''' + sh ''' + (cd src/docs && make check) + ''' + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage Docs Check + + stage('Build Docs') { + // Build docs separately if only docs changed. If there are other changes, docs are + // already built as part of `make dist` + when { + beforeOptions true + expression { ONLY_DOCS_CHANGED == '1' } + } + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 30, unit: 'MINUTES') + } + steps { + sh ''' + (cd src/docs && ./setup.sh ; make html) + ''' + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage Build Docs + + stage('Source Format Checks') { + when { + beforeOptions true + expression { ONLY_DOCS_CHANGED == '0' } + } + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 15, unit: "MINUTES") + } + steps { + sh ''' + rm -rf apache-couchdb-* + ./configure --skip-deps --spidermonkey-version 78 + make erlfmt-check + make elixir-source-checks + make python-black + ''' + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage Erlfmt + stage('Build Release Tarball') { + when { + beforeOptions true + expression { ONLY_DOCS_CHANGED == '0' } + } agent { docker { label 'docker' - image "apache/couchdbci-debian:bookworm-erlang-${MINIMUM_ERLANG_VERSION}" + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" args "${DOCKER_ARGS}" registryUrl 'https://docker.io/' registryCredentialsId 'dockerhub_creds' } } steps { - timeout(time: 15, unit: "MINUTES") { + timeout(time: 30, unit: "MINUTES") { sh (script: 'rm -rf apache-couchdb-*', label: 'Clean workspace of any previous release artifacts' ) sh "./configure --spidermonkey-version 78 --with-nouveau" sh 'make dist' @@ -421,10 +640,14 @@ pipeline { } // stage Build Release Tarball stage('Test and Package') { + when { + beforeOptions true + expression { ONLY_DOCS_CHANGED == '0' } + } steps { script { // Including failFast: true in map fails the build immediately if any parallel step fails - parallelStagesMap = meta.collectEntries( [failFast: false] ) { key, values -> + parallelStagesMap = meta.collectEntries( [failFast: true] ) { key, values -> if (values.image) { ["${key}": generateContainerStage(key)] } @@ -436,64 +659,6 @@ pipeline { } } } - - stage('Publish') { - - when { - expression { return env.BRANCH_NAME ==~ /main|2.*.x|3.*.x|4.*.x|jenkins-.*/ } - } - - agent { - docker { - image "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - skipDefaultCheckout() - timeout(time: 90, unit: "MINUTES") - } - - steps { - sh 'rm -rf ${WORKSPACE}/*' - unstash 'tarball' - unarchive mapping: ['pkgs/' : '.'] - - sh( label: 'Setup repo dirs', script: ''' - mkdir -p $BRANCH_NAME/debian $BRANCH_NAME/el8 $BRANCH_NAME/el9 $BRANCH_NAME/source - git clone https://github.com/apache/couchdb-pkg - ''' ) - - sh( label: 'Build Debian repo', script: ''' - for plat in bullseye bookworm focal - do - reprepro -b couchdb-pkg/repo includedeb $plat pkgs/$plat/*.deb - done - ''' ) - - sh( label: 'Build CentOS 8', script: ''' - (cd pkgs/centos8 && createrepo_c --database .) - ''' ) - - sh( label: 'Build CentOS 9', script: ''' - (cd pkgs/centos9 && createrepo_c --database .) - ''' ) - - sh( label: 'Build unified repo', script: ''' - mv couchdb-pkg/repo/pool $BRANCH_NAME/debian - mv couchdb-pkg/repo/dists $BRANCH_NAME/debian - mv pkgs/centos8/* $BRANCH_NAME/el8 - mv pkgs/centos9/* $BRANCH_NAME/el9 - mv apache-couchdb-*.tar.gz $BRANCH_NAME/source - cd $BRANCH_NAME/source - ls -1tr | head -n -10 | xargs -d '\n' rm -f -- - cd ../.. - ''' ) - } // steps - } // stage } // stages post { diff --git a/build-aux/Jenkinsfile.pr b/build-aux/Jenkinsfile.pr deleted file mode 100644 index 09180b17a7..0000000000 --- a/build-aux/Jenkinsfile.pr +++ /dev/null @@ -1,293 +0,0 @@ -#!groovy -// -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -build = ''' -mkdir -p ${ERLANG_VERSION} -cd ${ERLANG_VERSION} -rm -rf build -mkdir build -cd build -tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz -cd apache-couchdb-* -./configure --with-nouveau --with-clouseau --js-engine=${JS_ENGINE} -''' - -docs_changed = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q '^src/docs/'" -other_changes = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q -v '^src/docs/'" - -pipeline { - - // no top-level agent; agents must be declared for each stage - agent none - - environment { - recipient = 'notifications@couchdb.apache.org' - // Following fix an issue with git <= 2.6.5 where no committer - // name or email are present for reflog, required for git clone - GIT_COMMITTER_NAME = 'Jenkins User' - GIT_COMMITTER_EMAIL = 'couchdb@apache.org' - // Parameters for the matrix build - DOCKER_IMAGE_BASE = 'apache/couchdbci-debian:bookworm-erlang' - // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64 - // We need the jenkins user mapped inside of the image - // npm config cache below deals with /home/jenkins not mapping correctly - // inside the image - DOCKER_ARGS = '-e npm_config_cache=/home/jenkins/.npm -e HOME=. -e MIX_HOME=/home/jenkins/.mix -e HEX_HOME=/home/jenkins/.hex -e PIP_CACHE_DIR=/home/jenkins/.cache/pip -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group -v /home/jenkins/.gradle:/home/jenkins/.gradle:rw,z -v /home/jenkins/.hex:/home/jenkins/.hex:rw,z -v /home/jenkins/.npm:/home/jenkins/.npm:rw,z -v /home/jenkins/.cache/pip:/home/jenkins/.cache/pip:rw,z -v /home/jenkins/.mix:/home/jenkins/.mix:rw,z' - - // *** BE SURE TO ALSO CHANGE THE ERLANG VERSIONS FARTHER DOWN *** - // Search for ERLANG_VERSION - // see https://issues.jenkins.io/browse/JENKINS-61047 for why this cannot - // be done parametrically - LOW_ERLANG_VER = '25.3.2.20' - } - - options { - buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) - // This fails the build immediately if any parallel step fails - parallelsAlwaysFailFast() - preserveStashes(buildCount: 10) - timeout(time: 3, unit: 'HOURS') - timestamps() - } - - stages { - - stage('Setup Env') { - agent { - docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - timeout(time: 10, unit: 'MINUTES') - } - steps { - script { - env.DOCS_CHANGED = '0' - env.ONLY_DOCS_CHANGED = '0' - if ( sh(returnStatus: true, script: docs_changed) == 0 ) { - env.DOCS_CHANGED = '1' - if (sh(returnStatus: true, script: other_changes) == 1) { - env.ONLY_DOCS_CHANGED = '1' - } - } - } - } - post { - cleanup { - // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 - sh 'rm -rf ${WORKSPACE}/*' - } - } - } // stage 'Setup Environment' - - stage('Docs Check') { - // Run docs `make check` stage if any docs changed - when { - beforeOptions true - expression { DOCS_CHANGED == '1' } - } - agent { - docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - timeout(time: 15, unit: 'MINUTES') - } - steps { - sh ''' - make python-black - ''' - sh ''' - (cd src/docs && make check) - ''' - } - post { - cleanup { - // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 - sh 'rm -rf ${WORKSPACE}/*' - } - } - } // stage Docs Check - - stage('Build Docs') { - // Build docs separately if only docs changed. If there are other changes, docs are - // already built as part of `make dist` - when { - beforeOptions true - expression { ONLY_DOCS_CHANGED == '1' } - } - agent { - docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - timeout(time: 30, unit: 'MINUTES') - } - steps { - sh ''' - (cd src/docs && ./setup.sh ; make html) - ''' - } - post { - cleanup { - // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 - sh 'rm -rf ${WORKSPACE}/*' - } - } - } // stage Build Docs - - stage('Source Format Checks') { - when { - beforeOptions true - expression { ONLY_DOCS_CHANGED == '0' } - } - agent { - docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - timeout(time: 15, unit: "MINUTES") - } - steps { - sh ''' - rm -rf apache-couchdb-* - ./configure --skip-deps --spidermonkey-version 78 - make erlfmt-check - make elixir-source-checks - make python-black - ''' - } - post { - cleanup { - // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 - sh 'rm -rf ${WORKSPACE}/*' - } - } - } // stage Erlfmt - - - stage('Make Dist') { - when { - beforeOptions true - expression { ONLY_DOCS_CHANGED == '0' } - } - agent { - docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - timeout(time: 15, unit: "MINUTES") - } - steps { - sh ''' - rm -rf apache-couchdb-* - ./configure --spidermonkey-version 78 --with-nouveau --with-clouseau - make dist - chmod -R a+w * . - ''' - } - post { - success { - stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball' - } - cleanup { - // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 - sh 'rm -rf ${WORKSPACE}/*' - } - } - } // stage Make Dist - - // TODO Rework once Improved Docker Pipeline Engine is released - // https://issues.jenkins-ci.org/browse/JENKINS-47962 - // https://issues.jenkins-ci.org/browse/JENKINS-48050 - - stage('Make Check') { - when { - beforeOptions true - expression { ONLY_DOCS_CHANGED == '0' } - } - matrix { - axes { - axis { - name 'ERLANG_VERSION' - values '25.3.2.20', '26.2.5.11', '27.3.3' - } - axis { - name 'SM_VSN' - values '78' - } - axis { - name 'JS_ENGINE' - values 'quickjs', 'spidermonkey' - } - } - - stages { - stage('Build and Test') { - agent { - docker { - image "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" - label 'docker' - args "${DOCKER_ARGS}" - } - } - options { - skipDefaultCheckout() - timeout(time: 90, unit: "MINUTES") - } - steps { - unstash 'tarball' - sh( script: build ) - retry(3) {sh 'cd ${ERLANG_VERSION}/build/apache-couchdb-* && make check || (make build-report && false)'} - } - post { - always { - junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' - } - cleanup { - sh 'rm -rf ${WORKSPACE}/*' - } - } - } // stage - } // stages - } // matrix - } // stage "Make Check" - } // stages -} // pipeline diff --git a/build-aux/xref-helper.sh b/build-aux/xref-helper.sh index 5e1c3ed1ab..2b83c74f31 100755 --- a/build-aux/xref-helper.sh +++ b/build-aux/xref-helper.sh @@ -9,7 +9,6 @@ mkdir -p ./tmp $REBAR --keep-going --recursive xref $DIALYZE_OPTS | \ grep -v '==>' | \ grep -v 'WARN' | \ - grep -v hastings | \ sort > ./tmp/xref-output.txt # compare result against known allowed output diff --git a/configure b/configure index 70350a07b5..30ad40c24a 100755 --- a/configure +++ b/configure @@ -21,8 +21,8 @@ basename=`basename $0` PACKAGE_AUTHOR_NAME="The Apache Software Foundation" -REBAR3_BRANCH="3.23.0" -ERLFMT_VERSION="v1.3.0" +REBAR3_BRANCH="3.25.1" +ERLFMT_VERSION="v1.7.0" # TEST=0 WITH_PROPER="true" @@ -43,9 +43,10 @@ JS_ENGINE=${JS_ENGINE:-"spidermonkey"} SM_VSN=${SM_VSN:-"91"} CLOUSEAU_MTH=${CLOUSEAU_MTH:-"dist"} CLOUSEAU_URI=${CLOUSEAU_URI:-"https://github.com/cloudant-labs/clouseau/releases/download/%s/clouseau-%s-dist.zip"} -CLOUSEAU_VSN=${CLOUSEAU_VSN:-"2.23.1"} +CLOUSEAU_VSN=${CLOUSEAU_VSN:-"2.25.0"} CLOUSEAU_DIR="$(pwd)"/clouseau ARCH="$(uname -m)" +MULTIARCH_NAME="$(command -v dpkg-architecture > /dev/null && dpkg-architecture -q DEB_HOST_MULTIARCH || true)" ERLANG_VER="$(run_erlang 'io:put_chars(erlang:system_info(otp_release)).')" ERLANG_OS="$(run_erlang 'case os:type() of {OS, _} -> io:format("~s~n", [OS]) end.')" @@ -126,6 +127,7 @@ parse_opts() { --dev) WITH_DOCS="false" WITH_FAUXTON="false" + WITH_SPIDERMONKEY="false" shift continue ;; @@ -134,6 +136,7 @@ parse_opts() { WITH_DOCS="false" WITH_FAUXTON="false" WITH_NOUVEAU="true" + WITH_SPIDERMONKEY="false" shift continue ;; @@ -327,6 +330,7 @@ if [ "${WITH_SPIDERMONKEY}" = "true" ] && [ "${ERLANG_OS}" = "unix" ]; then # This list is taken from src/couch/rebar.config.script, please keep them in sync. if [ ! -d "/usr/include/${SM_HEADERS}" ] && \ + [ -n "${MULTIARCH_NAME}" -a ! -d "/usr/include/${MULTIARCH_NAME}/${SM_HEADERS}" ] && \ [ ! -d "/usr/local/include/${SM_HEADERS}" ] && \ [ ! -d "/opt/homebrew/include/${SM_HEADERS}" ]; then echo "ERROR: SpiderMonkey ${SM_VSN} is not found. Please specify with --spidermonkey-version." diff --git a/configure.ps1 b/configure.ps1 index faf3669ece..c7c730c70d 100644 --- a/configure.ps1 +++ b/configure.ps1 @@ -14,9 +14,9 @@ -CouchDBUser USER set the username to run as (defaults to current user) -SpiderMonkeyVersion VSN select the version of SpiderMonkey to use (default 91) -JSEngine ENGINE select JS engine to use (spidermonkey or quickjs) (default spidermonkey) - -ClouseauVersion VSN select the version of Clouseau to use (default 2.23.1) + -ClouseauVersion VSN select the version of Clouseau to use (default 2.25.0) -ClouseauMethod MTH method for Clouseau to deploy: git or dist (default dist) - -ClouseauUri URI location for retrieving Clouseau (default https://github.com/cloudant-labs/clouseau/releases/download/2.23.1/clouseau-2.23.1-dist.zip) + -ClouseauUri URI location for retrieving Clouseau (default https://github.com/cloudant-labs/clouseau/releases/download/2.25.0/clouseau-2.25.0-dist.zip) Installation directories: -Prefix PREFIX install architecture-independent files in PREFIX @@ -66,9 +66,9 @@ Param( [ValidateNotNullOrEmpty()] [string]$ClouseauMethod = "dist", # method for Clouseau to deploy: git or dist (default dist) [ValidateNotNullOrEmpty()] - [string]$ClouseauVersion = "2.23.1", # select the version of Clouseau to use (default 2.23.1) + [string]$ClouseauVersion = "2.25.0", # select the version of Clouseau to use (default 2.25.0) [ValidateNotNullOrEmpty()] - [string]$ClouseauUri = "https://github.com/cloudant-labs/clouseau/releases/download/{0}/clouseau-{0}-dist.zip", # location for retrieving Clouseau (default https://github.com/cloudant-labs/clouseau/releases/download/2.23.1/clouseau-2.23.1-dist.zip) + [string]$ClouseauUri = "https://github.com/cloudant-labs/clouseau/releases/download/{0}/clouseau-{0}-dist.zip", # location for retrieving Clouseau (default https://github.com/cloudant-labs/clouseau/releases/download/2.25.0/clouseau-2.25.0-dist.zip) [ValidateNotNullOrEmpty()] [string]$Prefix = "C:\Program Files\Apache\CouchDB", # install architecture-independent file location (default C:\Program Files\Apache\CouchDB) [ValidateNotNullOrEmpty()] diff --git a/dev/run b/dev/run index 85ea9622d5..478fb2ae9a 100755 --- a/dev/run +++ b/dev/run @@ -515,7 +515,7 @@ def write_config(ctx, node, env): content = apply_config_overrides(ctx, content) elif base == "local.ini": content = hack_local_ini(ctx, content) - elif ctx["enable_tls"] and base == "vm.args": + elif base == "vm.args": content = hack_vm_args(ctx, node, content) with open(tgt, "w") as handle: @@ -839,16 +839,23 @@ def hack_local_ini(ctx, contents): def hack_vm_args(ctx, node, contents): - contents += f""" + if ctx["enable_tls"]: + contents += f""" -proto_dist couch -couch_dist no_tls '"clouseau{node[-1]}@127.0.0.1"' -ssl_dist_optfile {ctx["rootdir"]}/src/couch_dist/certs/out/couch_dist.conf - """ - if ctx["no_tls"]: - no_tls_nodes = ctx["no_tls"].split(",") - for node_name in no_tls_nodes: - node_name = node_name if "@" in node_name else f"{node_name}@127.0.0.1" - contents += f"""\n-couch_dist no_tls '"{node_name}"'""" +""" + if ctx["no_tls"]: + no_tls_nodes = ctx["no_tls"].split(",") + for node_name in no_tls_nodes: + node_name = node_name if "@" in node_name else f"{node_name}@127.0.0.1" + contents += f""" +-couch_dist no_tls '"{node_name}"' +""" + if ctx["erlang_cookie"]: + contents += f""" +-setcookie {ctx["erlang_cookie"]} +""" return contents diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Ok.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Ok.java new file mode 100644 index 0000000000..b393e1978a --- /dev/null +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Ok.java @@ -0,0 +1,26 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.api; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Ok { + + public static final Ok INSTANCE = new Ok(); + + @JsonProperty + public boolean ok() { + return true; + } +} diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java index 85230d3223..c4f59f7a15 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java @@ -33,7 +33,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Stream; import org.apache.couchdb.nouveau.api.IndexDefinition; import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory; import org.apache.couchdb.nouveau.lucene9.Lucene9Index; @@ -250,35 +249,49 @@ public boolean exists(final String name) { } public void deleteAll(final String path, final List exclusions) throws IOException { - LOGGER.info("deleting indexes below {} (excluding {})", path, exclusions == null ? "nothing" : exclusions); + LOGGER.info( + "deleting indexes matching {} (excluding {})", + path, + exclusions == null || exclusions.isEmpty() ? "nothing" : exclusions); + var parts = path.split("/"); + deleteAll(rootDir, parts, 0, exclusions); + } - final Path indexRootPath = indexRootPath(path); - if (!indexRootPath.toFile().exists()) { + private void deleteAll(final Path path, final String[] parts, final int index, final List exclusions) + throws IOException { + // End of the path + if (index == parts.length - 1) { + try (var stream = Files.newDirectoryStream(path, parts[index])) { + stream.forEach(p -> { + if (exclusions != null && exclusions.indexOf(p.getFileName().toString()) != -1) { + return; + } + final String relativeName = rootDir.relativize(p).toString(); + try { + deleteIndex(relativeName); + } catch (final IOException | InterruptedException e) { + LOGGER.error("Exception deleting {}", p, e); + } + // Clean any newly empty directories. + do { + final File f = p.toFile(); + if (f.isDirectory() && f.list().length == 0) { + f.delete(); + } + } while ((p = p.getParent()) != null && !rootDir.equals(p)); + }); + } return; } - Stream stream = Files.find(indexRootPath, 100, (p, attr) -> attr.isDirectory() && isIndex(p)); - try { - stream.forEach((p) -> { - final String relativeToExclusions = indexRootPath.relativize(p).toString(); - if (exclusions != null && exclusions.indexOf(relativeToExclusions) != -1) { - return; - } - final String relativeName = rootDir.relativize(p).toString(); + // Recurse + try (var stream = Files.newDirectoryStream(path, parts[index])) { + stream.forEach(p -> { try { - deleteIndex(relativeName); - } catch (final IOException | InterruptedException e) { - LOGGER.error("Exception deleting {}", p, e); + deleteAll(p, parts, index + 1, exclusions); + } catch (IOException e) { + LOGGER.warn("Exception during delete of " + rootDir.relativize(p), e); } - // Clean any newly empty directories. - do { - final File f = p.toFile(); - if (f.isDirectory() && f.list().length == 0) { - f.delete(); - } - } while ((p = p.getParent()) != null && !rootDir.equals(p)); }); - } finally { - stream.close(); } } diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java index a6ca2c47b6..a52e00da91 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java @@ -35,6 +35,7 @@ import org.apache.couchdb.nouveau.api.IndexDefinition; import org.apache.couchdb.nouveau.api.IndexInfo; import org.apache.couchdb.nouveau.api.IndexInfoRequest; +import org.apache.couchdb.nouveau.api.Ok; import org.apache.couchdb.nouveau.api.SearchRequest; import org.apache.couchdb.nouveau.api.SearchResults; import org.apache.couchdb.nouveau.core.IndexManager; @@ -54,27 +55,29 @@ public IndexResource(final IndexManager indexManager) { } @PUT - public void createIndex(@PathParam("name") String name, @NotNull @Valid IndexDefinition indexDefinition) + public Ok createIndex(@PathParam("name") String name, @NotNull @Valid IndexDefinition indexDefinition) throws IOException { indexManager.create(name, indexDefinition); + return Ok.INSTANCE; } @DELETE @Path("/doc/{docId}") - public void deleteDoc( + public Ok deleteDoc( @PathParam("name") String name, @PathParam("docId") String docId, @NotNull @Valid DocumentDeleteRequest request) throws Exception { - indexManager.with(name, (index) -> { + return indexManager.with(name, (index) -> { index.delete(docId, request); - return null; + return Ok.INSTANCE; }); } @DELETE - public void deletePath(@PathParam("name") String path, @Valid final List exclusions) throws IOException { + public Ok deletePath(@PathParam("name") String path, @Valid final List exclusions) throws IOException { indexManager.deleteAll(path, exclusions); + return Ok.INSTANCE; } @GET @@ -85,9 +88,8 @@ public IndexInfo getIndexInfo(@PathParam("name") String name) throws Exception { } @POST - public void setIndexInfo(@PathParam("name") String name, @NotNull @Valid IndexInfoRequest request) - throws Exception { - indexManager.with(name, (index) -> { + public Ok setIndexInfo(@PathParam("name") String name, @NotNull @Valid IndexInfoRequest request) throws Exception { + return indexManager.with(name, (index) -> { if (request.getMatchUpdateSeq().isPresent() && request.getUpdateSeq().isPresent()) { index.setUpdateSeq( @@ -99,7 +101,7 @@ public void setIndexInfo(@PathParam("name") String name, @NotNull @Valid IndexIn request.getMatchPurgeSeq().getAsLong(), request.getPurgeSeq().getAsLong()); } - return null; + return Ok.INSTANCE; }); } @@ -114,14 +116,14 @@ public SearchResults searchIndex(@PathParam("name") String name, @NotNull @Valid @PUT @Path("/doc/{docId}") - public void updateDoc( + public Ok updateDoc( @PathParam("name") String name, @PathParam("docId") String docId, @NotNull @Valid DocumentUpdateRequest request) throws Exception { - indexManager.with(name, (index) -> { + return indexManager.with(name, (index) -> { index.update(docId, request); - return null; + return Ok.INSTANCE; }); } } diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java index a994cdd0c2..bb8f0b6470 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java @@ -16,7 +16,10 @@ import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ScheduledExecutorService; @@ -24,19 +27,21 @@ import org.apache.couchdb.nouveau.api.IndexDefinition; import org.apache.couchdb.nouveau.api.SearchRequest; import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; public class IndexManagerTest { - private static IndexManager manager; - private static ScheduledExecutorService executorService; + private Path rootDir; + private IndexManager manager; + private ScheduledExecutorService executorService; - @BeforeAll - public static void setupManager(@TempDir Path path) throws Exception { + @BeforeEach + public void setupManager(@TempDir Path path) throws Exception { executorService = Executors.newScheduledThreadPool(2); + rootDir = path; manager = new IndexManager(); manager.setRootDir(path); @@ -47,8 +52,8 @@ public static void setupManager(@TempDir Path path) throws Exception { manager.start(); } - @AfterAll - public static void cleanup() throws Exception { + @AfterEach + public void cleanup() throws Exception { executorService.shutdownNow(); executorService.awaitTermination(5, TimeUnit.SECONDS); manager.stop(); @@ -82,4 +87,60 @@ public void managerReopensAClosedIndex() throws Exception { }); assertThat(isOpen); } + + @Test + public void deleteAllRemovesIndexByName() throws Exception { + final IndexDefinition indexDefinition = new IndexDefinition(); + indexDefinition.setDefaultAnalyzer("standard"); + + assertThat(countIndexes()).isEqualTo(0); + manager.create("bar", indexDefinition); + assertThat(countIndexes()).isEqualTo(1); + manager.deleteAll("bar", null); + assertThat(countIndexes()).isEqualTo(0); + } + + @Test + public void deleteAllRemovesIndexByPath() throws Exception { + final IndexDefinition indexDefinition = new IndexDefinition(); + indexDefinition.setDefaultAnalyzer("standard"); + + assertThat(countIndexes()).isEqualTo(0); + manager.create("foo/bar", indexDefinition); + assertThat(countIndexes()).isEqualTo(1); + manager.deleteAll("foo/bar", null); + assertThat(countIndexes()).isEqualTo(0); + } + + @Test + public void deleteAllRemovesIndexByGlob() throws Exception { + final IndexDefinition indexDefinition = new IndexDefinition(); + indexDefinition.setDefaultAnalyzer("standard"); + + assertThat(countIndexes()).isEqualTo(0); + manager.create("foo/bar", indexDefinition); + assertThat(countIndexes()).isEqualTo(1); + manager.deleteAll("foo/*", null); + assertThat(countIndexes()).isEqualTo(0); + } + + @Test + public void deleteAllRemovesIndexByGlobExceptExclusions() throws Exception { + final IndexDefinition indexDefinition = new IndexDefinition(); + indexDefinition.setDefaultAnalyzer("standard"); + + assertThat(countIndexes()).isEqualTo(0); + manager.create("foo/bar", indexDefinition); + manager.create("foo/baz", indexDefinition); + assertThat(countIndexes()).isEqualTo(2); + manager.deleteAll("foo/*", List.of("bar")); + assertThat(countIndexes()).isEqualTo(1); + } + + private long countIndexes() throws IOException { + try (var stream = + Files.find(rootDir, 10, (p, attr) -> p.getFileName().toString().equals("index_definition.json"))) { + return stream.count(); + } + } } diff --git a/rebar.config.script b/rebar.config.script index 57fa8fae6b..efc03a35e6 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -150,17 +150,20 @@ SubDirs = [ "rel" ]. +% Try to keep deps in sync with .elp.toml deps + DepDescs = [ %% Independent Apps {snappy, "snappy", {tag, "CouchDB-1.0.9"}}, %% %% Non-Erlang deps {fauxton, {url, "https://github.com/apache/couchdb-fauxton"}, - {tag, "v1.3.4"}, [raw]}, + {tag, "v1.3.5"}, [raw]}, {ibrowse, "ibrowse", {tag, "CouchDB-4.4.2-6"}}, +{gun, "gun", {tag, "2.2.0-couchdb"}}, {jiffy, "jiffy", {tag, "1.1.2"}}, -{mochiweb, "mochiweb", {tag, "v3.2.2"}}, -{meck, "meck", {tag, "1.0.0"}}, +{mochiweb, "mochiweb", {tag, "v3.3.0"}}, +{meck, "meck", {tag, "v1.1.0"}}, {recon, "recon", {tag, "2.5.6"}} ]. @@ -168,7 +171,7 @@ WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}. OptionalDeps = case WithProper of true -> - [{proper, {url, "https://github.com/proper-testing/proper"}, "f2a44ee11a238c84403e72ee8ec68e6f65fd7e42"}]; + [{proper, {url, "https://github.com/proper-testing/proper"}, "v1.5.0"}]; false -> [] end. @@ -191,7 +194,7 @@ end. AddConfig = [ {cover_enabled, true}, {cover_print_enabled, true}, - {require_otp_vsn, "25|26|27|28"}, + {require_otp_vsn, "26|27|28"}, {deps_dir, "src"}, {deps, lists:map(MakeDep, DepDescs ++ OptionalDeps)}, {sub_dirs, SubDirs}, diff --git a/rel/nouveau.yaml b/rel/nouveau.yaml index 40837f12cb..0f33d5f25c 100644 --- a/rel/nouveau.yaml +++ b/rel/nouveau.yaml @@ -8,12 +8,12 @@ logging: server: applicationConnectors: - - type: http + - type: h2c bindHost: 127.0.0.1 port: {{nouveau_port}} useDateHeader: false adminConnectors: - - type: http + - type: h2c bindHost: 127.0.0.1 port: {{nouveau_admin_port}} useDateHeader: false diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index dfefa62dc0..00d92bafe3 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -71,11 +71,18 @@ view_index_dir = {{view_index_dir}} ;default_engine = couch ; Enable this to only "soft-delete" databases when DELETE /{db} requests are -; made. This will place a .recovery directory in your data directory and -; move deleted databases/shards there instead. You can then manually delete -; these files later, as desired. +; made. This will add `.deleted.couch` extension to the database shard files +; in the data directory. You can then manually delete these files later, as +; desired. ;enable_database_recovery = false +; Applies only when `enable_database_recovery = false`. +; When DELETE /{db} requests are made: +; - If `true`, delete the database shard files from the `data` directory. +; - If `false`, rename the database shard files and move them to the `.delete` +; directory. +;delete_after_rename = true + ; Set the maximum size allowed for a partition. This helps users avoid ; inadvertently abusing partitions resulting in hot shards. The default ; is 10GiB. A value of 0 or less will disable partition size checks. @@ -116,16 +123,43 @@ view_index_dir = {{view_index_dir}} ; When enabled, use cfile parallel reads for all the requests. By default the ; setting is "false", so only requests which are configured to bypass the IOQ ; would use the cfile parallel reads. If there is enough RAM available for a -; large file cache and the disks have enough IO bandwith, consider enabling +; large file cache and the disks have enough IO bandwidth, consider enabling ; this setting. ;cfile_skip_ioq = false -[purge] -; Allowed maximum number of documents in one purge request -;max_document_id_number = 100 +; CouchDB will not normally allow a database downgrade to a previous version if +; the new version added new fields to the database file header structure. You +; can disable this protection by setting this to false. We recommend +; re-enabling the protection after you have performed the downgrade but it is +; not mandatory to do so. +; Note that some future versions of CouchDB might not support downgrade at all, +; whatever value this is set to. In those cases CouchDB will refuse to +; downgrade or even open the databases in question. +;prohibit_downgrade = true + +[bt_engine_cache] +; Memory used for btree engine cache. This is a cache for top levels of +; database btrees (id tree, seq tree) and a few terms from the db header. Value +; is in bytes. +;max_size = 67108864 +; +; Items not accessed in a while are eventually evicted. However, if the memory +; used is below this percentage, then even the unused items are left in the +; cache. The trade-off here is when a new workload starts, it may find the +; cache with some stale items during the first few seconds and not be able to +; insert its entries in. +;leave_percent = 30 +; +; Cache database btree nodes up to this depth only. Depth starts at 1 at root, +; then at 2 the next level down, and so on. Only intermediate (pointer) nodes +; will be cached, those are the nodes which point to other nodes, as opposed to +; leaf key-value nodes, which hold revision trees. To disable db btree node +; caching set the value to 0 +;db_btree_cache_depth = 3 +[purge] ; Allowed maximum number of accumulated revisions in one purge request -;max_revisions_number = 1000 +;max_revisions_number = infinity ; Allowed durations when index is not updated for local purge checkpoint ; document. Default is 24 hours. @@ -217,7 +251,7 @@ bind_address = 127.0.0.1 ; Set to false to revert to a previous _bulk_get implementation using single ; doc fetches internally. Using batches should be faster, however there may be -; bugs in the new new implemention, so expose this option to allow reverting to +; bugs in the new implementation, so expose this option to allow reverting to ; the old behavior. ;bulk_get_use_batches = true @@ -401,7 +435,8 @@ hash_algorithms = sha256, sha ;uuid_prefix_len = 7 ;request_timeout = 60000 ;all_docs_timeout = 10000 -;attachments_timeout = 60000 +;all_docs_view_permsg_timeout = 5000 +;attachments_timeout = 600000 ;view_timeout = infinity ;view_permsg_timeout = 3600000 ;partition_view_timeout = infinity @@ -428,6 +463,16 @@ hash_algorithms = sha256, sha ;update_db = true ;[view_updater] +; Configure the queue capacity used during indexing. These settings apply to +; both the queue between the changes feed and the JS mapper, and between the +; JS mapper and the disk writer. +; Whichever limit happens to be hit first is the one that takes effect. + +; The maximum queue memory size +;queue_memory_cap = 100000 +; The maximum queue length +;queue_item_cap = 500 + ;min_writer_items = 100 ;min_writer_size = 16777216 @@ -500,12 +545,19 @@ authentication_db = _users ; Erlang Query Server ;enable_erlang_query_server = false -; Changing reduce_limit to false will disable reduce_limit. -; If you think you're hitting reduce_limit with a "good" reduce function, -; please let us know on the mailing list so we can fine tune the heuristic. + [query_server_config] ;commit_freq = 5 +; Changing reduce_limit to false will disable reduce_limit. Setting the reduce +; limit to log will only log a warning instead of crashing the view. If you +; think you're hitting reduce_limit with a "good" reduce function, please let +; us know on the mailing list so we can fine tune the heuristic. ;reduce_limit = true +; Don't log/crash a reduce if the result is less than this number of bytes; +;reduce_limit_threshold = 5000 +; Don't log/crash a reduce if the result is less than the ratio multiplied +; by input size +;reduce_limit_ratio = 2.0 ;os_process_limit = 100 ;os_process_idle_limit = 300 ;os_process_soft_limit = 100 @@ -659,13 +711,17 @@ partitioned||* = true ; Maximum peer certificate depth (must be set even if certificate validation is off). ;ssl_certificate_max_depth = 3 +; How often to reload operating system CA certificates (in hours). The default +; is 24 hours. +;cacert_reload_interval_hours = 24 + ; Maximum document ID length for replication. ;max_document_id_length = infinity ; How much time to wait before retrying after a missing doc exception. This ; exception happens if the document was seen in the changes feed, but internal ; replication hasn't caught up yet, and fetching document's revisions -; fails. This a common scenario when source is updated while continuous +; fails. This is a common scenario when source is updated while continuous ; replication is running. The retry period would depend on how quickly internal ; replication is expected to catch up. In general this is an optimisation to ; avoid crashing the whole replication job, which would consume more resources @@ -684,7 +740,7 @@ partitioned||* = true ; couch_replicator_auth_session - use _session cookie authentication ; couch_replicator_auth_noop - use basic authentication (previous default) ; Currently, the new _session cookie authentication is tried first, before -; falling back to the old basic authenticaion default: +; falling back to the old basic authentication default: ;auth_plugins = couch_replicator_auth_session,couch_replicator_auth_noop ; To restore the old behaviour, use the following value: @@ -712,7 +768,7 @@ partitioned||* = true ; priority 0, and would render this algorithm useless. The default value of ; 0.98 is picked such that if a job ran for one scheduler cycle, then didn't ; get to run for 7 hours, it would still have priority > 0. 7 hours was picked -; as it was close enought to 8 hours which is the default maximum error backoff +; as it was close enough to 8 hours which is the default maximum error backoff ; interval. ;priority_coeff = 0.98 @@ -925,12 +981,11 @@ port = {{prometheus_port}} [custodian] ; When set to `true`, force using `[cluster] n` values as the expected n value -; of of shard copies. In cases where the application prevents creating -; non-default n databases, this could help detect case where the shard map was -; altered by hand, or via an external tools, such that it doesn't have the -; necessary number of copies for some ranges. By default, when the setting is -; `false`, the expected n value is based on the number of available copies in -; the shard map. +; of shard copies. In cases where the application prevents creating non-default +; n databases, this could help detect case where the shard map was altered by +; hand, or via an external tools, such that it doesn't have the necessary number +; of copies for some ranges. By default, when the setting is `false`, the +; expected n value is based on the number of available copies in the shard map. ;use_cluster_n_as_expected_n = false [nouveau] @@ -1001,6 +1056,17 @@ url = {{nouveau_url}} ; is shared across all running plugins. ;doc_rate_limit = 1000 +; Limit the rate per second at which plugins write/update documents. The rate +; is shared across all running plugins. Unlike other rate limit which are +; applied automatically by the plugin backend this rate assume the plugins will +; explicitly use the couch_scanner_rate_limiter API when performing writes. +;doc_write_rate_limit = 500 + +; Batch size to use when fetching design documents. For lots of small design +; documents this value could be increased to 500 or 1000. If design documents +; are large (100KB+) it could make sense to decrease it a bit to 25 or 10. +;ddoc_batch_size = 100 + [couch_scanner_plugins] ;couch_scanner_plugin_ddoc_features = false ;couch_scanner_plugin_find = false diff --git a/rel/overlay/etc/vm.args b/rel/overlay/etc/vm.args index d762c33bf2..b4167be7b5 100644 --- a/rel/overlay/etc/vm.args +++ b/rel/overlay/etc/vm.args @@ -57,6 +57,17 @@ # -kernel prevent_overlapping_partitions false +# Set Erlang process limit. If not set in Erlang 27 and greater versions it +# defaults to 1048576. In Erlang versions less than 27 it defaulted to 262144. +# When a cluster reaches this limit it will emit "Too many processes" error in +# the log. That could be a time to bump it up. The actual value set will be a +# larger power of 2 number. To check the actual limit call +# `erlang:system_info(process_limit).` in remsh. +# +# For additional info see +# https://www.erlang.org/doc/apps/erts/erl_cmd.html#emulator-flags ++P 1048576 + # Increase the pool of dirty IO schedulers from 10 to 16 # Dirty IO schedulers are used for file IO. +SDio 16 diff --git a/rel/reltool.config b/rel/reltool.config index c1f0ea0706..b85bd49b62 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -48,6 +48,8 @@ ets_lru, fabric, global_changes, + gun, + cowlib, ibrowse, ioq, jiffy, @@ -111,6 +113,8 @@ {app, ets_lru, [{incl_cond, include}]}, {app, fabric, [{incl_cond, include}]}, {app, global_changes, [{incl_cond, include}]}, + {app, gun, [{incl_cond, include}]}, + {app, cowlib, [{incl_cond, include}]}, {app, ibrowse, [{incl_cond, include}]}, {app, ioq, [{incl_cond, include}]}, {app, jiffy, [{incl_cond, include}]}, diff --git a/share/server/views.js b/share/server/views.js index b59c9912e8..623b5c4b5b 100644 --- a/share/server/views.js +++ b/share/server/views.js @@ -36,9 +36,9 @@ var Views = (function() { var reduce_line = JSON.stringify(reductions); var reduce_length = reduce_line.length; var input_length = State.line_length - code_size - // TODO make reduce_limit config into a number if (State.query_config && State.query_config.reduce_limit && - reduce_length > 4096 && ((reduce_length * 2) > input_length)) { + reduce_length > State.query_config.reduce_limit_threshold && + ((reduce_length * State.query_config.reduce_limit_ratio) > input_length)) { var log_message = [ "Reduce output must shrink more rapidly:", "input size:", input_length, diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index a43baeae48..d0f5e01110 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -454,8 +454,7 @@ delete_db_req(#httpd{} = Req, DbName) -> end. do_db_req(#httpd{path_parts = [DbName | _], user_ctx = Ctx} = Req, Fun) -> - Shard = hd(mem3:shards(DbName)), - Props = couch_util:get_value(props, Shard#shard.opts, []), + Props = mem3:props(DbName), Opts = case Ctx of undefined -> @@ -695,22 +694,17 @@ db_req(#httpd{method = 'POST', path_parts = [_, <<"_purge">>]} = Req, Db) -> Options = [{user_ctx, Req#httpd.user_ctx}, {w, W}], {IdsRevs} = chttpd:json_body_obj(Req), IdsRevs2 = [{Id, couch_doc:parse_revs(Revs)} || {Id, Revs} <- IdsRevs], - MaxIds = config:get_integer("purge", "max_document_id_number", 100), - case length(IdsRevs2) =< MaxIds of - false -> throw({bad_request, "Exceeded maximum number of documents."}); - true -> ok - end, - RevsLen = lists:foldl( - fun({_Id, Revs}, Acc) -> - length(Revs) + Acc - end, - 0, - IdsRevs2 - ), - MaxRevs = config:get_integer("purge", "max_revisions_number", 1000), - case RevsLen =< MaxRevs of - false -> throw({bad_request, "Exceeded maximum number of revisions."}); - true -> ok + case config:get("purge", "max_revisions_number", "infinity") of + "infinity" -> + ok; + Val -> + MaxRevs = list_to_integer(Val), + SumFun = fun({_Id, Revs}, Acc) -> length(Revs) + Acc end, + RevsLen = lists:foldl(SumFun, 0, IdsRevs2), + case RevsLen =< MaxRevs of + false -> throw({bad_request, "Exceeded maximum number of revisions."}); + true -> ok + end end, couch_stats:increment_counter([couchdb, document_purges, total], length(IdsRevs2)), Results2 = @@ -1419,7 +1413,7 @@ receive_request_data(Req, Len) when Len == chunked -> GetChunk }; receive_request_data(Req, LenLeft) when LenLeft > 0 -> - Len = erlang:min(4096, LenLeft), + Len = min(4096, LenLeft), Data = chttpd:recv(Req, Len), {Data, fun() -> receive_request_data(Req, LenLeft - iolist_size(Data)) end}; receive_request_data(_Req, _) -> @@ -1460,13 +1454,13 @@ purge_results_to_json([{DocId, _Revs} | RIn], [{accepted, PRevs} | ROut]) -> {Code, Results} = purge_results_to_json(RIn, ROut), couch_stats:increment_counter([couchdb, document_purges, success]), NewResults = [{DocId, couch_doc:revs_to_strs(PRevs)} | Results], - {erlang:max(Code, 202), NewResults}; + {max(Code, 202), NewResults}; purge_results_to_json([{DocId, _Revs} | RIn], [Error | ROut]) -> {Code, Results} = purge_results_to_json(RIn, ROut), {NewCode, ErrorStr, Reason} = chttpd:error_info(Error), couch_stats:increment_counter([couchdb, document_purges, failure]), NewResults = [{DocId, {[{error, ErrorStr}, {reason, Reason}]}} | Results], - {erlang:max(NewCode, Code), NewResults}. + {max(NewCode, Code), NewResults}. send_updated_doc(Req, Db, DocId, Json) -> send_updated_doc(Req, Db, DocId, Json, []). @@ -1526,9 +1520,9 @@ update_doc(Db, DocId, #doc{deleted = Deleted, body = DocBody} = Doc, Options) -> {'DOWN', Ref, _, _, {exit_throw, Reason}} -> throw(Reason); {'DOWN', Ref, _, _, {exit_error, Reason}} -> - erlang:error(Reason); + error(Reason); {'DOWN', Ref, _, _, {exit_exit, Reason}} -> - erlang:exit(Reason) + exit(Reason) end, case Result of diff --git a/src/chttpd/src/chttpd_external.erl b/src/chttpd/src/chttpd_external.erl index 4cd1d996fe..fe63de3b22 100644 --- a/src/chttpd/src/chttpd_external.erl +++ b/src/chttpd/src/chttpd_external.erl @@ -139,7 +139,7 @@ to_json_terms(Data) -> to_json_terms([], Acc) -> {lists:reverse(Acc)}; to_json_terms([{Key, Value} | Rest], Acc) when is_atom(Key) -> - to_json_terms(Rest, [{list_to_binary(atom_to_list(Key)), list_to_binary(Value)} | Acc]); + to_json_terms(Rest, [{atom_to_binary(Key), list_to_binary(Value)} | Acc]); to_json_terms([{Key, Value} | Rest], Acc) -> to_json_terms(Rest, [{list_to_binary(Key), list_to_binary(Value)} | Acc]). diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl index d0bf363f31..c22a27b982 100644 --- a/src/chttpd/src/chttpd_misc.erl +++ b/src/chttpd/src/chttpd_misc.erl @@ -252,7 +252,7 @@ replicate({Props} = PostBody, Ctx) -> ]), case rpc:call(Node, couch_replicator, replicate, [PostBody, Ctx]) of {badrpc, Reason} -> - erlang:error(Reason); + error(Reason); Res -> Res end @@ -281,7 +281,7 @@ cancel_replication(PostBody, Ctx) -> choose_node(Key) when is_binary(Key) -> Checksum = erlang:crc32(Key), - Nodes = lists:sort([node() | erlang:nodes()]), + Nodes = lists:sort([node() | nodes()]), lists:nth(1 + Checksum rem length(Nodes), Nodes); choose_node(Key) -> choose_node(?term_to_bin(Key)). diff --git a/src/chttpd/src/chttpd_node.erl b/src/chttpd/src/chttpd_node.erl index 2a67da9e36..b0ea3686cb 100644 --- a/src/chttpd/src/chttpd_node.erl +++ b/src/chttpd/src/chttpd_node.erl @@ -191,7 +191,7 @@ handle_node_req(#httpd{ "max_http_request_size", 4294967296 ), NewOpts = [{body, MochiReq0:recv_body(MaxSize)} | MochiReq0:get(opts)], - Ref = erlang:make_ref(), + Ref = make_ref(), MochiReq = mochiweb_request:new( {remote, self(), Ref}, NewOpts, @@ -286,6 +286,7 @@ get_stats() -> {run_queue, SQ}, {run_queue_dirty_cpu, DCQ}, {ets_table_count, length(ets:all())}, + {bt_engine_cache, couch_bt_engine_cache:info()}, {context_switches, element(1, statistics(context_switches))}, {reductions, element(1, statistics(reductions))}, {garbage_collection_count, NumberOfGCs}, diff --git a/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl b/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl index 3085079ad9..554da524ea 100644 --- a/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl +++ b/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl @@ -64,7 +64,7 @@ base_url() -> "http://" ++ Addr ++ ":" ++ Port. make_auth_session_string(HashAlgorithm, User, Secret, TimeStamp) -> - SessionData = User ++ ":" ++ erlang:integer_to_list(TimeStamp, 16), + SessionData = User ++ ":" ++ integer_to_list(TimeStamp, 16), Hash = couch_util:hmac(HashAlgorithm, Secret, SessionData), "AuthSession=" ++ couch_util:encodeBase64Url(SessionData ++ ":" ++ ?b2l(Hash)). diff --git a/src/chttpd/test/eunit/chttpd_purge_tests.erl b/src/chttpd/test/eunit/chttpd_purge_tests.erl index ef8af69717..3b91571eee 100644 --- a/src/chttpd/test/eunit/chttpd_purge_tests.erl +++ b/src/chttpd/test/eunit/chttpd_purge_tests.erl @@ -49,7 +49,6 @@ purge_test_() -> ?TDEF_FE(t_purge_only_post_allowed), ?TDEF_FE(t_empty_purge_request), ?TDEF_FE(t_ok_purge_request), - ?TDEF_FE(t_ok_purge_with_max_document_id_number), ?TDEF_FE(t_accepted_purge_request), ?TDEF_FE(t_partial_purge_request), ?TDEF_FE(t_mixed_purge_request), @@ -85,23 +84,6 @@ t_ok_purge_request(DbUrl) -> ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := IdsRevs}, Response2), ?assert(Status =:= 201 orelse Status =:= 202). -t_ok_purge_with_max_document_id_number(DbUrl) -> - PurgedDocsNum = 101, - {201, Response1} = create_docs(DbUrl, docs(PurgedDocsNum)), - IdsRevs = ids_revs(Response1), - - {400, #{<<"reason">> := Error}} = req(post, url(DbUrl, "_purge"), IdsRevs), - ?assertEqual(<<"Exceeded maximum number of documents.">>, Error), - - ok = config:set("purge", "max_document_id_number", "101", _Persist = false), - try - {Status, Response2} = req(post, url(DbUrl, "_purge"), IdsRevs), - ?assertMatch(#{<<"purge_seq">> := null, <<"purged">> := IdsRevs}, Response2), - ?assert(Status =:= 201 orelse Status =:= 202) - after - ok = config:delete("purge", "max_document_id_number", _Persist) - end. - t_accepted_purge_request(DbUrl) -> try meck:new(fabric, [passthrough]), @@ -173,24 +155,14 @@ t_over_many_ids_or_revs_purge_request(DbUrl) -> <<"doc3">> => [Rev3] }, - % Ids larger than expected - config:set("purge", "max_document_id_number", "1", _Persist = false), - try - {Status1, #{<<"reason">> := Error1}} = req(post, url(DbUrl, "_purge"), IdsRevs), - ?assertEqual(<<"Exceeded maximum number of documents.">>, Error1), - ?assertEqual(400, Status1) - after - config:delete("purge", "max_document_id_number", _Persist) - end, - % Revs larger than expected - config:set("purge", "max_revisions_number", "1", _Persist), + config:set("purge", "max_revisions_number", "1", false), try {Status2, #{<<"reason">> := Error2}} = req(post, url(DbUrl, "_purge"), IdsRevs), ?assertEqual(<<"Exceeded maximum number of revisions.">>, Error2), ?assertEqual(400, Status2) after - config:delete("purge", "max_revisions_number", _Persist) + config:delete("purge", "max_revisions_number", false) end. t_purged_infos_limit_only_get_put_allowed(DbUrl) -> @@ -316,7 +288,7 @@ create_docs(Url, Docs) -> docs(Counter) -> lists:foldl( fun(I, Acc) -> - Id = ?l2b(integer_to_list(I)), + Id = integer_to_binary(I), Doc = #{<<"_id">> => Id, <<"val">> => I}, [Doc | Acc] end, diff --git a/src/config/src/config.erl b/src/config/src/config.erl index 695bc40b9d..70b8883f93 100644 --- a/src/config/src/config.erl +++ b/src/config/src/config.erl @@ -100,7 +100,7 @@ to_integer(Int) when is_integer(Int) -> to_integer(List) when is_list(List) -> list_to_integer(List); to_integer(Bin) when is_binary(Bin) -> - list_to_integer(binary_to_list(Bin)). + binary_to_integer(Bin). get_float(Section, Key, Default) when is_float(Default) -> try @@ -125,7 +125,7 @@ to_float(List) when is_list(List) -> to_float(Int) when is_integer(Int) -> list_to_float(integer_to_list(Int) ++ ".0"); to_float(Bin) when is_binary(Bin) -> - list_to_float(binary_to_list(Bin)). + binary_to_float(Bin). get_boolean(Section, Key, Default) when is_boolean(Default) -> try diff --git a/src/config/src/config_listener_mon.erl b/src/config/src/config_listener_mon.erl index b74a613062..d794e37ce1 100644 --- a/src/config/src/config_listener_mon.erl +++ b/src/config/src/config_listener_mon.erl @@ -40,7 +40,7 @@ subscribe(Module, InitSt) -> end. init({Pid, Mod, InitSt}) -> - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), case config_listener:start(Mod, {Mod, Pid}, {Pid, InitSt}) of ok -> proc_lib:init_ack({ok, self()}), diff --git a/src/config/test/config_tests.erl b/src/config/test/config_tests.erl index 1775e4f396..4e3bf60f71 100644 --- a/src/config/test/config_tests.erl +++ b/src/config/test/config_tests.erl @@ -700,15 +700,15 @@ should_remove_handler_when_pid_exits({_Apps, Pid}) -> % Monitor the config_listener_mon process {monitored_by, [Mon]} = process_info(Pid, monitored_by), - MonRef = erlang:monitor(process, Mon), + MonRef = monitor(process, Mon), % Kill the process synchronously - PidRef = erlang:monitor(process, Pid), + PidRef = monitor(process, Pid), exit(Pid, kill), receive {'DOWN', PidRef, _, _, _} -> ok after ?TIMEOUT -> - erlang:error({timeout, config_listener_death}) + error({timeout, config_listener_death}) end, % Wait for the config_listener_mon process to @@ -716,7 +716,7 @@ should_remove_handler_when_pid_exits({_Apps, Pid}) -> receive {'DOWN', MonRef, _, _, normal} -> ok after ?TIMEOUT -> - erlang:error({timeout, config_listener_mon_death}) + error({timeout, config_listener_mon_death}) end, ?assertEqual(0, n_handlers()) @@ -728,7 +728,7 @@ should_stop_monitor_on_error({_Apps, Pid}) -> % Monitor the config_listener_mon process {monitored_by, [Mon]} = process_info(Pid, monitored_by), - MonRef = erlang:monitor(process, Mon), + MonRef = monitor(process, Mon), % Have the process throw an error ?assertEqual(ok, config:set("throw_error", "foo", "bar", false)), @@ -742,7 +742,7 @@ should_stop_monitor_on_error({_Apps, Pid}) -> receive {'DOWN', MonRef, _, _, shutdown} -> ok after ?TIMEOUT -> - erlang:error({timeout, config_listener_mon_shutdown}) + error({timeout, config_listener_mon_shutdown}) end, ?assertEqual(0, n_handlers()) @@ -784,7 +784,7 @@ should_unsubscribe_when_subscriber_gone(_Subscription, {_Apps, Pid}) -> ?assert(is_process_alive(Pid)), % Monitor subscriber process - MonRef = erlang:monitor(process, Pid), + MonRef = monitor(process, Pid), exit(Pid, kill), @@ -792,7 +792,7 @@ should_unsubscribe_when_subscriber_gone(_Subscription, {_Apps, Pid}) -> receive {'DOWN', MonRef, _, _, _} -> ok after ?TIMEOUT -> - erlang:error({timeout, config_notifier_shutdown}) + error({timeout, config_notifier_shutdown}) end, ?assertNot(is_process_alive(Pid)), @@ -1015,7 +1015,7 @@ wait_config_get(Sec, Key, Val) -> spawn_config_listener() -> Self = self(), - Pid = erlang:spawn(fun() -> + Pid = spawn(fun() -> ok = config:listen_for_changes(?MODULE, {self(), undefined}), Self ! registered, loop(undefined) @@ -1023,13 +1023,13 @@ spawn_config_listener() -> receive registered -> ok after ?TIMEOUT -> - erlang:error({timeout, config_handler_register}) + error({timeout, config_handler_register}) end, Pid. spawn_config_notifier(Subscription) -> Self = self(), - Pid = erlang:spawn(fun() -> + Pid = spawn(fun() -> ok = config:subscribe_for_changes(Subscription), Self ! registered, loop(undefined) @@ -1037,7 +1037,7 @@ spawn_config_notifier(Subscription) -> receive registered -> ok after ?TIMEOUT -> - erlang:error({timeout, config_handler_register}) + error({timeout, config_handler_register}) end, Pid. @@ -1050,7 +1050,7 @@ loop(undefined) -> {get_msg, _, _} = Msg -> loop(Msg); Msg -> - erlang:error({invalid_message, Msg}) + error({invalid_message, Msg}) end; loop({get_msg, From, Ref}) -> receive @@ -1059,7 +1059,7 @@ loop({get_msg, From, Ref}) -> {config_change, _, _, _, _} = Msg -> From ! {Ref, Msg}; Msg -> - erlang:error({invalid_message, Msg}) + error({invalid_message, Msg}) end, loop(undefined); loop({config_msg, _} = Msg) -> @@ -1067,17 +1067,17 @@ loop({config_msg, _} = Msg) -> {get_msg, From, Ref} -> From ! {Ref, Msg}; Msg -> - erlang:error({invalid_message, Msg}) + error({invalid_message, Msg}) end, loop(undefined). getmsg(Pid) -> - Ref = erlang:make_ref(), + Ref = make_ref(), Pid ! {get_msg, self(), Ref}, receive {Ref, {config_msg, Msg}} -> Msg after ?TIMEOUT -> - erlang:error({timeout, config_msg}) + error({timeout, config_msg}) end. n_handlers() -> @@ -1112,7 +1112,7 @@ wait_process_restart(Name, Timeout, Delay, Started, _Prev) -> end. stop_sync(Pid, Timeout) when is_pid(Pid) -> - MRef = erlang:monitor(process, Pid), + MRef = monitor(process, Pid), try begin catch unlink(Pid), @@ -1125,7 +1125,7 @@ stop_sync(Pid, Timeout) when is_pid(Pid) -> end end after - erlang:demonitor(MRef, [flush]) + demonitor(MRef, [flush]) end; stop_sync(_, _) -> error(badarg). diff --git a/src/couch/include/couch_db.hrl b/src/couch/include/couch_db.hrl index 9c1df21b69..138c202d57 100644 --- a/src/couch/include/couch_db.hrl +++ b/src/couch/include/couch_db.hrl @@ -182,16 +182,6 @@ db_open_options = [] }). --record(btree, { - fd, - root, - extract_kv, - assemble_kv, - less, - reduce = nil, - compression = ?DEFAULT_COMPRESSION -}). - -record(proc, { pid, lang, diff --git a/src/couch/include/couch_eunit.hrl b/src/couch/include/couch_eunit.hrl index 3ffa0506fe..2c10e257de 100644 --- a/src/couch/include/couch_eunit.hrl +++ b/src/couch/include/couch_eunit.hrl @@ -68,7 +68,7 @@ ((fun (__X) -> case (Expr) of __V when __V == __X -> ok; - __Y -> erlang:error({assertEquiv_failed, + __Y -> error({assertEquiv_failed, [{module, ?MODULE}, {line, ?LINE}, {expression, (??Expr)}, diff --git a/src/couch/include/couch_eunit_proper.hrl b/src/couch/include/couch_eunit_proper.hrl index dcf07701aa..75172bfb8e 100644 --- a/src/couch/include/couch_eunit_proper.hrl +++ b/src/couch/include/couch_eunit_proper.hrl @@ -18,12 +18,12 @@ { atom_to_list(F), {timeout, QuickcheckTimeout, - ?_assert(proper:quickcheck(?MODULE:F(), [ + ?_test(?assertMatch(true, proper:quickcheck(?MODULE:F(), [ {to_file, user}, {start_size, 2}, {numtests, NumTests}, long_result - ]))} + ]), "Failed property test '" ++ atom_to_list(F) ++ "'"))} } || {F, 0} <- ?MODULE:module_info(exports), F > 'prop_', F < 'prop`' ]). diff --git a/src/couch/priv/couch_cfile/couch_cfile.c b/src/couch/priv/couch_cfile/couch_cfile.c index 06328b0697..c7e6f16f02 100644 --- a/src/couch/priv/couch_cfile/couch_cfile.c +++ b/src/couch/priv/couch_cfile/couch_cfile.c @@ -11,7 +11,15 @@ // the License. -#ifndef _WIN32 +// Not supported on Windows or 32 bit architectures. When not supported the NIF +// will still build, but the API functions will return {error, einval}, and +// we'd fallback to the regular file API then +// +#if !defined(_WIN32) && defined(__LP64__) +#define COUCH_CFILE_SUPPORTED 1 +#endif + +#ifdef COUCH_CFILE_SUPPORTED #include #include @@ -225,7 +233,7 @@ int efile_datasync(int fd, posix_errno_t* res_errno) { // static ERL_NIF_TERM dup_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED int fd, newfd; handle_t* hdl; ErlNifRWLock *lock; @@ -293,7 +301,7 @@ static ERL_NIF_TERM dup_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED handle_t* hdl; if (argc != 1 || !get_handle(env, argv[0], &hdl)) { @@ -326,7 +334,7 @@ static ERL_NIF_TERM close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[ // static ERL_NIF_TERM close_fd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED int fd; if (argc != 1 || !enif_is_number(env, argv[0])) { @@ -351,7 +359,7 @@ static ERL_NIF_TERM close_fd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar // static ERL_NIF_TERM pread_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED handle_t* hdl; long offset, block_size, bytes_read; SysIOVec io_vec[1]; @@ -413,7 +421,7 @@ static ERL_NIF_TERM pread_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ // Follows implementation from prim_file_nif.c // static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED handle_t* hdl; ErlNifIOVec vec, *input = &vec; posix_errno_t res_errno = 0; @@ -452,7 +460,7 @@ static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ } static ERL_NIF_TERM seek_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED handle_t* hdl; long result, offset; int whence; @@ -502,7 +510,7 @@ static ERL_NIF_TERM seek_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] } static ERL_NIF_TERM datasync_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED handle_t* hdl; posix_errno_t res_errno = 0; @@ -530,7 +538,7 @@ static ERL_NIF_TERM datasync_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar } static ERL_NIF_TERM truncate_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED handle_t* hdl; off_t offset; @@ -568,7 +576,7 @@ static ERL_NIF_TERM truncate_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar // static ERL_NIF_TERM info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED handle_t* hdl; int fd, old_fd; @@ -601,7 +609,7 @@ static ERL_NIF_TERM info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] // static ERL_NIF_TERM eof_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { -#ifndef _WIN32 +#ifdef COUCH_CFILE_SUPPORTED handle_t* hdl; struct stat data; diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg index 6a7120f87e..18f32a47a8 100644 --- a/src/couch/priv/stats_descriptions.cfg +++ b/src/couch/priv/stats_descriptions.cfg @@ -474,3 +474,15 @@ {type, counter}, {desc, <<"number of mango selector evaluations">>} ]}. +{[couchdb, bt_engine_cache, hits], [ + {type, counter}, + {desc, <<"number of bt_engine cache hits">>} +]}. +{[couchdb, bt_engine_cache, misses], [ + {type, counter}, + {desc, <<"number of bt_engine cache misses">>} +]}. +{[couchdb, bt_engine_cache, full], [ + {type, counter}, + {desc, <<"number of times bt_engine cache was full">>} +]}. diff --git a/src/couch/rebar.config.script b/src/couch/rebar.config.script index c4e0b7a7ee..1c9809902c 100644 --- a/src/couch/rebar.config.script +++ b/src/couch/rebar.config.script @@ -36,16 +36,16 @@ end. CouchJSPath = filename:join(["priv", CouchJSName]). Version = case os:getenv("COUCHDB_VERSION") of false -> - string:strip(os:cmd("git describe --always"), right, $\n); + string:trim(os:cmd("git describe --always")); Version0 -> - string:strip(Version0, right) + string:trim(Version0) end. GitSha = case os:getenv("COUCHDB_GIT_SHA") of false -> ""; % release builds won\'t get a fallback GitSha0 -> - string:strip(GitSha0, right) + string:trim(GitSha0) end. CouchConfig = case filelib:is_file(os:getenv("COUCHDB_CONFIG")) of @@ -151,6 +151,25 @@ ProperConfig = case code:lib_dir(proper) of _ -> [{d, 'WITH_PROPER'}] end. +DpkgArchitectureCmd = "dpkg-architecture -q DEB_HOST_MULTIARCH". +GenericMozJSIncludePaths = "-I/usr/include/mozjs-" ++ SMVsn ++ " -I/usr/local/include/mozjs-" ++ SMVsn. +GenericMozJSLibPaths = "-L/usr/local/lib". + +WithDpkgArchitecture = os:find_executable("dpkg-architecture") /= false. +WithHomebrew = os:find_executable("brew") /= false. + +MozJSIncludePath = case {WithDpkgArchitecture, WithHomebrew} of + {false, false} -> GenericMozJSIncludePaths; + {true, false} -> GenericMozJSIncludePaths ++ " -I/usr/include/" ++ string:trim(os:cmd(DpkgArchitectureCmd)) ++ "/mozjs-" ++ SMVsn; + {false, true} -> GenericMozJSIncludePaths ++ " -I/opt/homebrew/include/mozjs-" ++ SMVsn +end. + +MozJSLibPath = case {WithDpkgArchitecture, WithHomebrew} of + {false, false} -> GenericMozJSLibPaths; + {true, false} -> GenericMozJSLibPaths ++ " -L/usr/lib/" ++ string:trim(os:cmd(DpkgArchitectureCmd)); + {false, true} -> GenericMozJSLibPaths ++ " -L/opt/homebrew/lib" +end. + % The include directories (parameters for the `-I` C compiler flag) are % considered in the `configure` script as a pre-check for their existence. % Please keep them in sync. @@ -195,45 +214,20 @@ end. "-DXP_UNIX -I/usr/include/mozjs-86 -I/usr/local/include/mozjs-86 -I/opt/homebrew/include/mozjs-86/ -std=c++17 -Wno-invalid-offsetof", "-L/usr/local/lib -L /opt/homebrew/lib/ -std=c++17 -lmozjs-86 -lm" }; - {unix, _} when SMVsn == "91" -> - { - "$CFLAGS -DXP_UNIX -I/usr/include/mozjs-91 -I/usr/local/include/mozjs-91 -I/opt/homebrew/include/mozjs-91/ -std=c++17 -Wno-invalid-offsetof", - "$LDFLAGS -L/usr/local/lib -L /opt/homebrew/lib/ -std=c++17 -lmozjs-91 -lm" - }; - {unix, _} when SMVsn == "102" -> + {unix, _} when SMVsn == "91"; SMVsn == "102"; SMVsn == "115"; SMVsn == "128" -> { - "$CFLAGS -DXP_UNIX -I/usr/include/mozjs-102 -I/usr/local/include/mozjs-102 -I/opt/homebrew/include/mozjs-102/ -std=c++17 -Wno-invalid-offsetof", - "$LDFLAGS -L/usr/local/lib -L /opt/homebrew/lib/ -std=c++17 -lmozjs-102 -lm" - }; - {unix, _} when SMVsn == "115" -> - { - "$CFLAGS -DXP_UNIX -I/usr/include/mozjs-115 -I/usr/local/include/mozjs-115 -I/opt/homebrew/include/mozjs-115/ -std=c++17 -Wno-invalid-offsetof", - "$LDFLAGS -L/usr/local/lib -L /opt/homebrew/lib/ -std=c++17 -lmozjs-115 -lm" - }; - {unix, _} when SMVsn == "128" -> - { - "$CFLAGS -DXP_UNIX -I/usr/include/mozjs-128 -I/usr/local/include/mozjs-128 -I/opt/homebrew/include/mozjs-128/ -std=c++17 -Wno-invalid-offsetof", - "$LDFLAGS -L/usr/local/lib -L /opt/homebrew/lib/ -std=c++17 -lmozjs-128 -lm" + "$CFLAGS -DXP_UNIX " ++ MozJSIncludePath ++ " -std=c++17 -Wno-invalid-offsetof", + "$LDFLAGS " ++ MozJSLibPath ++ " -std=c++17 -lm -lmozjs-" ++ SMVsn }; {win32, _} when SMVsn == "91" -> { "/std:c++17 /DXP_WIN", "$LDFLAGS mozjs-91.lib" }; - {win32, _} when SMVsn == "102" -> - { - "/std:c++17 /DXP_WIN /Zc:preprocessor /utf-8", - "$LDFLAGS mozjs-102.lib" - }; - {win32, _} when SMVsn == "115" -> - { - "/std:c++17 /DXP_WIN /Zc:preprocessor /utf-8", - "$LDFLAGS mozjs-115.lib" - }; - {win32, _} when SMVsn == "128" -> + {win32, _} when SMVsn == "102"; SMVsn == "115"; SMVsn == "128" -> { "/std:c++17 /DXP_WIN /Zc:preprocessor /utf-8", - "$LDFLAGS mozjs-128.lib" + "$LDFLAGS mozjs-" ++ SMVsn ++ ".lib" } end. @@ -273,12 +267,12 @@ end. IcuIncludePath = case WithBrew of false -> GenericIcuIncludePaths; - true -> "-I" ++ string:strip(os:cmd(BrewIcuPrefixCmd), right, $\n) ++ "/include" + true -> "-I" ++ string:trim(os:cmd(BrewIcuPrefixCmd)) ++ "/include" end. IcuLibPath = case WithBrew of false -> GenericIcuLibPaths; - true -> "-L" ++ string:strip(os:cmd(BrewIcuPrefixCmd), right, $\n) ++ "/lib" + true -> "-L" ++ string:trim(os:cmd(BrewIcuPrefixCmd)) ++ "/lib" end. IcuEnv = [{"DRV_CFLAGS", "$DRV_CFLAGS -DPIC -O2 -fno-common"}, diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl index 5ca3927e70..c7e051a285 100644 --- a/src/couch/src/couch_att.erl +++ b/src/couch/src/couch_att.erl @@ -491,7 +491,7 @@ encoded_lengths_from_json(Props) -> EncodedLen = Len; EncodingValue -> EncodedLen = couch_util:get_value(<<"encoded_length">>, Props, Len), - Encoding = list_to_existing_atom(binary_to_list(EncodingValue)) + Encoding = binary_to_existing_atom(EncodingValue) end, {Len, EncodedLen, Encoding}. @@ -585,7 +585,7 @@ flush_data(Db, Fun, Att) when is_function(Fun) -> end) end; flush_data(Db, {follows, Parser, Ref}, Att) -> - ParserRef = erlang:monitor(process, Parser), + ParserRef = monitor(process, Parser), Fun = fun() -> Parser ! {get_bytes, Ref, self()}, receive @@ -600,7 +600,7 @@ flush_data(Db, {follows, Parser, Ref}, Att) -> try flush_data(Db, Fun, store(data, Fun, Att)) after - erlang:demonitor(ParserRef, [flush]) + demonitor(ParserRef, [flush]) end; flush_data(Db, {stream, StreamEngine}, Att) -> case couch_db:is_active_stream(Db, StreamEngine) of @@ -645,7 +645,7 @@ foldl(DataFun, Att, Fun, Acc) when is_function(DataFun) -> Len = fetch(att_len, Att), fold_streamed_data(DataFun, Len, Fun, Acc); foldl({follows, Parser, Ref}, Att, Fun, Acc) -> - ParserRef = erlang:monitor(process, Parser), + ParserRef = monitor(process, Parser), DataFun = fun() -> Parser ! {get_bytes, Ref, self()}, receive @@ -660,7 +660,7 @@ foldl({follows, Parser, Ref}, Att, Fun, Acc) -> try foldl(DataFun, store(data, DataFun, Att), Fun, Acc) after - erlang:demonitor(ParserRef, [flush]) + demonitor(ParserRef, [flush]) end. range_foldl(Att, From, To, Fun, Acc) -> @@ -781,7 +781,7 @@ open_stream(StreamSrc, Data) -> true -> StreamSrc(Data); false -> - erlang:error({invalid_stream_source, StreamSrc}) + error({invalid_stream_source, StreamSrc}) end end. diff --git a/src/couch/src/couch_base32.erl b/src/couch/src/couch_base32.erl index 776fe773dd..03267ced07 100644 --- a/src/couch/src/couch_base32.erl +++ b/src/couch/src/couch_base32.erl @@ -150,7 +150,7 @@ decode(Encoded, ByteOffset, Acc) -> find_in_set(Char) -> case binary:match(?SET, Char) of nomatch -> - erlang:error(not_base32); + error(not_base32); {Offset, _} -> Offset end. diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index ad84e0db85..8f99926ca4 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -114,6 +114,19 @@ -include_lib("couch/include/couch_db.hrl"). -include("couch_bt_engine.hrl"). +% Commonly used header fields (used more than once in this module) +-define(UPDATE_SEQ, update_seq). +-define(SECURITY_PTR, security_ptr). +-define(PROPS_PTR, props_ptr). +-define(REVS_LIMIT, revs_limit). +-define(PURGE_INFOS_LIMIT, purge_infos_limit). +-define(COMPACTED_SEQ, compacted_seq). + +-define(DEFAULT_BTREE_CACHE_DEPTH, 3). +% Priority is about how long the entry will survive in the cache initially. A +% period is about 2 seconds and each period the value is halved. +-define(HEADER_CACHE_PRIORITY, 16). + exists(FilePath) -> case is_file(FilePath) of true -> @@ -196,14 +209,14 @@ handle_db_updater_info({'DOWN', Ref, _, _, _}, #st{fd_monitor = Ref} = St) -> {stop, normal, St#st{fd = undefined, fd_monitor = closed}}. incref(St) -> - {ok, St#st{fd_monitor = erlang:monitor(process, St#st.fd)}}. + {ok, St#st{fd_monitor = monitor(process, St#st.fd)}}. decref(St) -> - true = erlang:demonitor(St#st.fd_monitor, [flush]), + true = demonitor(St#st.fd_monitor, [flush]), ok. monitored_by(St) -> - case erlang:process_info(St#st.fd, monitored_by) of + case process_info(St#st.fd, monitored_by) of {monitored_by, Pids} -> lists:filter(fun is_pid/1, Pids); _ -> @@ -211,7 +224,7 @@ monitored_by(St) -> end. get_compacted_seq(#st{header = Header}) -> - couch_bt_engine_header:get(Header, compacted_seq). + couch_bt_engine_header:get(Header, ?COMPACTED_SEQ). get_del_doc_count(#st{} = St) -> {ok, Reds} = couch_btree:full_reduce(St#st.id_tree), @@ -242,10 +255,10 @@ get_oldest_purge_seq(#st{purge_seq_tree = PurgeSeqTree}) -> PurgeSeq. get_purge_infos_limit(#st{header = Header}) -> - couch_bt_engine_header:get(Header, purge_infos_limit). + couch_bt_engine_header:get(Header, ?PURGE_INFOS_LIMIT). get_revs_limit(#st{header = Header}) -> - couch_bt_engine_header:get(Header, revs_limit). + couch_bt_engine_header:get(Header, ?REVS_LIMIT). get_size_info(#st{} = St) -> {ok, FileSize} = couch_file:bytes(St#st.fd), @@ -305,26 +318,14 @@ get_partition_info(#st{} = St, Partition) -> ]} ]. -get_security(#st{header = Header} = St) -> - case couch_bt_engine_header:get(Header, security_ptr) of - undefined -> - []; - Pointer -> - {ok, SecProps} = couch_file:pread_term(St#st.fd, Pointer), - SecProps - end. +get_security(#st{} = St) -> + get_header_term(St, ?SECURITY_PTR, []). -get_props(#st{header = Header} = St) -> - case couch_bt_engine_header:get(Header, props_ptr) of - undefined -> - []; - Pointer -> - {ok, Props} = couch_file:pread_term(St#st.fd, Pointer), - Props - end. +get_props(#st{} = St) -> + get_header_term(St, ?PROPS_PTR, []). get_update_seq(#st{header = Header}) -> - couch_bt_engine_header:get(Header, update_seq). + couch_bt_engine_header:get(Header, ?UPDATE_SEQ). get_uuid(#st{header = Header}) -> couch_bt_engine_header:get(Header, uuid). @@ -332,7 +333,7 @@ get_uuid(#st{header = Header}) -> set_revs_limit(#st{header = Header} = St, RevsLimit) -> NewSt = St#st{ header = couch_bt_engine_header:set(Header, [ - {revs_limit, RevsLimit} + {?REVS_LIMIT, RevsLimit} ]), needs_commit = true }, @@ -341,33 +342,17 @@ set_revs_limit(#st{header = Header} = St, RevsLimit) -> set_purge_infos_limit(#st{header = Header} = St, PurgeInfosLimit) -> NewSt = St#st{ header = couch_bt_engine_header:set(Header, [ - {purge_infos_limit, PurgeInfosLimit} + {?PURGE_INFOS_LIMIT, PurgeInfosLimit} ]), needs_commit = true }, {ok, increment_update_seq(NewSt)}. -set_security(#st{header = Header} = St, NewSecurity) -> - Options = [{compression, St#st.compression}], - {ok, Ptr, _} = couch_file:append_term(St#st.fd, NewSecurity, Options), - NewSt = St#st{ - header = couch_bt_engine_header:set(Header, [ - {security_ptr, Ptr} - ]), - needs_commit = true - }, - {ok, increment_update_seq(NewSt)}. +set_security(#st{} = St, NewSecurity) -> + {ok, increment_update_seq(set_header_term(St, ?SECURITY_PTR, NewSecurity))}. -set_props(#st{header = Header} = St, Props) -> - Options = [{compression, St#st.compression}], - {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options), - NewSt = St#st{ - header = couch_bt_engine_header:set(Header, [ - {props_ptr, Ptr} - ]), - needs_commit = true - }, - {ok, increment_update_seq(NewSt)}. +set_props(#st{} = St, Props) -> + {ok, increment_update_seq(set_header_term(St, ?PROPS_PTR, Props))}. open_docs(#st{} = St, DocIds) -> Results = couch_btree:lookup(St#st.id_tree, DocIds), @@ -480,14 +465,14 @@ write_doc_infos(#st{} = St, Pairs, LocalDocs) -> NewUpdateSeq = lists:foldl( fun(#full_doc_info{update_seq = Seq}, Acc) -> - erlang:max(Seq, Acc) + max(Seq, Acc) end, get_update_seq(St), Add ), NewHeader = couch_bt_engine_header:set(St#st.header, [ - {update_seq, NewUpdateSeq} + {?UPDATE_SEQ, NewUpdateSeq} ]), {ok, St#st{ @@ -509,7 +494,7 @@ purge_docs(#st{} = St, Pairs, PurgeInfos) -> RemDocIds = [Old#full_doc_info.id || {Old, not_found} <- Pairs], RemSeqs = [Old#full_doc_info.update_seq || {Old, _} <- Pairs], DocsToAdd = [New || {_, New} <- Pairs, New /= not_found], - CurrSeq = couch_bt_engine_header:get(St#st.header, update_seq), + CurrSeq = couch_bt_engine_header:get(St#st.header, ?UPDATE_SEQ), Seqs = [FDI#full_doc_info.update_seq || FDI <- DocsToAdd], NewSeq = lists:max([CurrSeq | Seqs]), @@ -522,7 +507,7 @@ purge_docs(#st{} = St, Pairs, PurgeInfos) -> false -> NewSeq end, Header = couch_bt_engine_header:set(St#st.header, [ - {update_seq, UpdateSeq} + {?UPDATE_SEQ, UpdateSeq} ]), {ok, IdTree2} = couch_btree:add_remove(IdTree, DocsToAdd, RemDocIds), @@ -562,9 +547,7 @@ commit_data(St) -> case NewHeader /= OldHeader orelse NeedsCommit of true -> - couch_file:sync(Fd), - ok = couch_file:write_header(Fd, NewHeader), - couch_file:sync(Fd), + ok = couch_file:write_header(Fd, NewHeader, [sync]), couch_stats:increment_counter([couchdb, commits]), {ok, St#st{ header = NewHeader, @@ -615,7 +598,7 @@ fold_purge_infos(St, StartSeq0, UserFun, UserAcc, Options) -> % automatically. if MinSeq =< StartSeq -> ok; - true -> erlang:error({invalid_start_purge_seq, StartSeq0, MinSeq}) + true -> error({invalid_start_purge_seq, StartSeq0, MinSeq}) end, Wrapper = fun(Info, _Reds, UAcc) -> UserFun(Info, UAcc) @@ -804,30 +787,16 @@ purge_tree_reduce(rereduce, Reds) -> set_update_seq(#st{header = Header} = St, UpdateSeq) -> {ok, St#st{ header = couch_bt_engine_header:set(Header, [ - {update_seq, UpdateSeq} + {?UPDATE_SEQ, UpdateSeq} ]), needs_commit = true }}. -copy_security(#st{header = Header} = St, SecProps) -> - Options = [{compression, St#st.compression}], - {ok, Ptr, _} = couch_file:append_term(St#st.fd, SecProps, Options), - {ok, St#st{ - header = couch_bt_engine_header:set(Header, [ - {security_ptr, Ptr} - ]), - needs_commit = true - }}. +copy_security(#st{} = St, SecProps) -> + {ok, set_header_term(St, ?SECURITY_PTR, SecProps)}. -copy_props(#st{header = Header} = St, Props) -> - Options = [{compression, St#st.compression}], - {ok, Ptr, _} = couch_file:append_term(St#st.fd, Props, Options), - {ok, St#st{ - header = couch_bt_engine_header:set(Header, [ - {props_ptr, Ptr} - ]), - needs_commit = true - }}. +copy_props(#st{} = St, Props) -> + {ok, set_header_term(St, ?PROPS_PTR, Props)}. open_db_file(FilePath, Options) -> case couch_file:open(FilePath, Options) of @@ -864,7 +833,8 @@ init_state(FilePath, Fd, Header0, Options) -> {split, fun ?MODULE:id_tree_split/1}, {join, fun ?MODULE:id_tree_join/2}, {reduce, fun ?MODULE:id_tree_reduce/2}, - {compression, Compression} + {compression, Compression}, + {cache_depth, btree_cache_depth()} ]), SeqTreeState = couch_bt_engine_header:seq_tree_state(Header), @@ -872,28 +842,32 @@ init_state(FilePath, Fd, Header0, Options) -> {split, fun ?MODULE:seq_tree_split/1}, {join, fun ?MODULE:seq_tree_join/2}, {reduce, fun ?MODULE:seq_tree_reduce/2}, - {compression, Compression} + {compression, Compression}, + {cache_depth, btree_cache_depth()} ]), LocalTreeState = couch_bt_engine_header:local_tree_state(Header), {ok, LocalTree} = couch_btree:open(LocalTreeState, Fd, [ {split, fun ?MODULE:local_tree_split/1}, {join, fun ?MODULE:local_tree_join/2}, - {compression, Compression} + {compression, Compression}, + {cache_depth, btree_cache_depth()} ]), PurgeTreeState = couch_bt_engine_header:purge_tree_state(Header), {ok, PurgeTree} = couch_btree:open(PurgeTreeState, Fd, [ {split, fun ?MODULE:purge_tree_split/1}, {join, fun ?MODULE:purge_tree_join/2}, - {reduce, fun ?MODULE:purge_tree_reduce/2} + {reduce, fun ?MODULE:purge_tree_reduce/2}, + {cache_depth, btree_cache_depth()} ]), PurgeSeqTreeState = couch_bt_engine_header:purge_seq_tree_state(Header), {ok, PurgeSeqTree} = couch_btree:open(PurgeSeqTreeState, Fd, [ {split, fun ?MODULE:purge_seq_tree_split/1}, {join, fun ?MODULE:purge_seq_tree_join/2}, - {reduce, fun ?MODULE:purge_tree_reduce/2} + {reduce, fun ?MODULE:purge_tree_reduce/2}, + {cache_depth, btree_cache_depth()} ]), ok = couch_file:set_db_pid(Fd, self()), @@ -901,7 +875,7 @@ init_state(FilePath, Fd, Header0, Options) -> St = #st{ filepath = FilePath, fd = Fd, - fd_monitor = erlang:monitor(process, Fd), + fd_monitor = monitor(process, Fd), header = Header, needs_commit = false, id_tree = IdTree, @@ -933,22 +907,20 @@ update_header(St, Header) -> ]). increment_update_seq(#st{header = Header} = St) -> - UpdateSeq = couch_bt_engine_header:get(Header, update_seq), + UpdateSeq = couch_bt_engine_header:get(Header, ?UPDATE_SEQ), St#st{ header = couch_bt_engine_header:set(Header, [ - {update_seq, UpdateSeq + 1} + {?UPDATE_SEQ, UpdateSeq + 1} ]) }. set_default_security_object(Fd, Header, Compression, Options) -> - case couch_bt_engine_header:get(Header, security_ptr) of + case couch_bt_engine_header:get(Header, ?SECURITY_PTR) of Pointer when is_integer(Pointer) -> Header; _ -> Default = couch_util:get_value(default_security_object, Options), - AppendOpts = [{compression, Compression}], - {ok, Ptr, _} = couch_file:append_term(Fd, Default, AppendOpts), - couch_bt_engine_header:set(Header, security_ptr, Ptr) + set_header_term(Fd, Header, ?SECURITY_PTR, Default, Compression) end. % This function is here, and not in couch_bt_engine_header @@ -1015,9 +987,7 @@ init_set_props(Fd, Header, Options) -> Header; InitialProps -> Compression = couch_compress:get_compression_method(), - AppendOpts = [{compression, Compression}], - {ok, Ptr, _} = couch_file:append_term(Fd, InitialProps, AppendOpts), - couch_bt_engine_header:set(Header, props_ptr, Ptr) + set_header_term(Fd, Header, ?PROPS_PTR, InitialProps, Compression) end. delete_compaction_files(FilePath) -> @@ -1190,9 +1160,9 @@ finish_compaction_int(#st{} = OldSt, #st{} = NewSt1) -> {ok, NewSt2} = commit_data(NewSt1#st{ header = couch_bt_engine_header:set(Header, [ - {compacted_seq, get_update_seq(OldSt)}, - {revs_limit, get_revs_limit(OldSt)}, - {purge_infos_limit, get_purge_infos_limit(OldSt)} + {?COMPACTED_SEQ, get_update_seq(OldSt)}, + {?REVS_LIMIT, get_revs_limit(OldSt)}, + {?PURGE_INFOS_LIMIT, get_purge_infos_limit(OldSt)} ]), local_tree = NewLocal2 }), @@ -1229,3 +1199,44 @@ is_file(Path) -> {ok, #file_info{type = directory}} -> true; _ -> false end. + +get_header_term(#st{header = Header} = St, Key, Default) when is_atom(Key) -> + case couch_bt_engine_header:get(Header, Key) of + undefined -> + Default; + Pointer when is_integer(Pointer) -> + case couch_bt_engine_cache:lookup({St#st.fd, Pointer}) of + undefined -> + {ok, Term} = couch_file:pread_term(St#st.fd, Pointer), + couch_bt_engine_cache:insert({St#st.fd, Pointer}, Term, ?HEADER_CACHE_PRIORITY), + Term; + Term -> + Term + end + end. + +set_header_term(#st{} = St, Key, Term) when is_atom(Key) -> + #st{fd = Fd, header = Header, compression = Compression} = St, + St#st{ + header = set_header_term(Fd, Header, Key, Term, Compression), + needs_commit = true + }. + +set_header_term(Fd, Header, Key, Term, Compression) when is_atom(Key) -> + case couch_bt_engine_header:get(Header, Key) of + Pointer when is_integer(Pointer) -> + % Reset old one to 0 usage. Some old snapshot may still + % see it and use. But it will only survive only one more + % interval at most otherwise + couch_bt_engine_cache:reset({Fd, Pointer}); + _ -> + ok + end, + TermOpts = [{compression, Compression}], + {ok, Ptr, _} = couch_file:append_term(Fd, Term, TermOpts), + Result = couch_bt_engine_header:set(Header, Key, Ptr), + couch_bt_engine_cache:insert({Fd, Ptr}, Term, ?HEADER_CACHE_PRIORITY), + Result. + +btree_cache_depth() -> + config:get_integer("bt_engine_cache", "db_btree_cache_depth", ?DEFAULT_BTREE_CACHE_DEPTH). diff --git a/src/couch/src/couch_bt_engine_cache.erl b/src/couch/src/couch_bt_engine_cache.erl new file mode 100644 index 0000000000..a395649577 --- /dev/null +++ b/src/couch/src/couch_bt_engine_cache.erl @@ -0,0 +1,292 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_bt_engine_cache). + +-include_lib("stdlib/include/ms_transform.hrl"). + +% Main API +% +-export([ + insert/2, + insert/3, + reset/1, + lookup/1, + info/0, + tables/0 +]). + +% Supervision and start API +% +-export([ + create_tables/0, + sup_children/0, + start_link/1, + init/1 +]). + +-define(DEFAULT_SIZE, 67108864). +-define(DEFAULT_LEAVE_PERCENT, 30). +-define(INTERVAL_MSEC, 3000). +% How often cleaners check if the ets size increases. This is used in cases +% when initially, at the start the cleaning interval, ets table size is below +% "leave percent". Then we skip cleaning all the 0 usage entries. However, if a +% new set of requests come in soon after, but before the next clean-up interval +% they'll fail since the cache would be full. To be able to react quicker and +% make room, cleaners poll table sizes a bit more often. +-define(CLEANUP_INTERVAL_MSEC, 100). +% 1 bsl 58, power of 2 that's still an immediate integer +-define(MAX_PRIORITY, 288230376151711744). +-define(PTERM_KEY, {?MODULE, caches}). +% Metrics +-define(HITS, hits). +-define(MISSES, misses). +-define(FULL, full). + +-record(cache, {tid, max_size}). + +% Main API + +insert(Key, Term) -> + insert(Key, Term, 1). + +insert(Key, Term, Priority) when is_integer(Priority) -> + Priority1 = min(?MAX_PRIORITY, max(0, Priority)), + case get_cache(Key) of + #cache{tid = Tid, max_size = Max} -> + case ets:info(Tid, memory) < Max of + true -> + case ets:insert_new(Tid, {Key, Priority1, Term}) of + true -> + true; + false -> + bump_usage(Tid, Key), + false + end; + false -> + bump_metric(?FULL), + false + end; + undefined -> + false + end. + +reset(Key) -> + case get_cache(Key) of + #cache{tid = Tid} -> reset_usage(Tid, Key); + undefined -> true + end. + +lookup(Key) -> + case get_cache(Key) of + #cache{tid = Tid} -> + case ets:lookup_element(Tid, Key, 3, undefined) of + undefined -> + bump_metric(?MISSES), + undefined; + Term -> + bump_usage(Tid, Key), + bump_metric(?HITS), + Term + end; + undefined -> + undefined + end. + +info() -> + case persistent_term:get(?PTERM_KEY, undefined) of + Caches when is_tuple(Caches) -> + SizeMem = [info(C) || C <- tuple_to_list(Caches)], + MaxMem = [Max || #cache{max_size = Max} <- tuple_to_list(Caches)], + {Sizes, Mem} = lists:unzip(SizeMem), + #{ + size => lists:sum(Sizes), + memory => lists:sum(Mem), + max_memory => lists:sum(MaxMem) * wordsize(), + full => sample_metric(?FULL), + hits => sample_metric(?HITS), + misses => sample_metric(?MISSES), + shard_count => shard_count() + }; + undefined -> + #{} + end. + +tables() -> + case persistent_term:get(?PTERM_KEY, undefined) of + Caches when is_tuple(Caches) -> + [Tid || #cache{tid = Tid} <- tuple_to_list(Caches)]; + undefined -> + [] + end. + +% Supervisor helper functions + +create_tables() -> + BtCaches = [new() || _ <- lists:seq(1, shard_count())], + persistent_term:put(?PTERM_KEY, list_to_tuple(BtCaches)). + +sup_children() -> + [sup_child(I) || I <- lists:seq(1, shard_count())]. + +sup_child(N) -> + Name = list_to_atom("couch_bt_engine_cache_" ++ integer_to_list(N)), + #{id => Name, start => {?MODULE, start_link, [N]}, shutdown => brutal_kill}. + +% Process start and main loop + +start_link(N) when is_integer(N) -> + {ok, proc_lib:spawn_link(?MODULE, init, [N])}. + +init(N) -> + Caches = persistent_term:get(?PTERM_KEY), + Cache = #cache{tid = Tid} = element(N, Caches), + ets:delete_all_objects(Tid), + loop(Cache). + +loop(#cache{tid = Tid} = Cache) -> + decay(Tid), + Next = now_msec() + wait_interval(?INTERVAL_MSEC), + clean(Cache, Next - ?CLEANUP_INTERVAL_MSEC), + remove_dead(Cache), + WaitLeft = max(10, Next - now_msec()), + timer:sleep(WaitLeft), + loop(Cache). + +% Clean unused procs. If we haven't cleaned any keep polling at a higher +% rate so we react quicker if a new set of entries are added. +% +clean(#cache{tid = Tid} = Cache, Until) -> + case now_msec() < Until of + true -> + case should_clean(Cache) of + true -> + ets:match_delete(Tid, {'_', 0, '_'}); + false -> + timer:sleep(wait_interval(?CLEANUP_INTERVAL_MSEC)), + clean(Cache, Until) + end; + false -> + false + end. + +should_clean(#cache{tid = Tid, max_size = Max}) -> + ets:info(Tid, memory) >= Max * leave_percent() / 100. + +remove_dead(#cache{tid = Tid}) -> + All = pids(Tid), + Alive = sets:filter(fun is_process_alive/1, All), + Dead = sets:subtract(All, Alive), + % In OTP 27+ use sets:map/2 + Fun = fun(Pid, _) -> ets:match_delete(Tid, {{Pid, '_'}, '_', '_'}) end, + sets:fold(Fun, true, Dead). + +pids(Tid) -> + Acc = couch_util:new_set(), + try + ets:foldl(fun pids_fold/2, Acc, Tid) + catch + error:badarg -> Acc + end. + +pids_fold({{Pid, _}, _, _}, Acc) when is_pid(Pid) -> + sets:add_element(Pid, Acc); +pids_fold({_, _, _}, Acc) -> + Acc. + +new() -> + Opts = [public, {write_concurrency, true}, {read_concurrency, true}], + Max0 = round(max_size() / wordsize() / shard_count()), + % Some per-table overhead for the table metadata + Max = Max0 + round(250 * 1024 / wordsize()), + #cache{tid = ets:new(?MODULE, Opts), max_size = Max}. + +get_cache(Term) -> + case persistent_term:get(?PTERM_KEY, undefined) of + Caches when is_tuple(Caches) -> + Index = erlang:phash2(Term, tuple_size(Caches)), + #cache{} = element(1 + Index, Caches); + undefined -> + undefined + end. + +bump_usage(Tid, Key) -> + % We're updating the second field incrementing it by 1 and clamping it + % at ?MAX_PRIORITY. We don't set the default for the update_counter + % specifically to avoid creating bogus entries just from updating the + % counter, so expect the error:badarg here sometimes. + UpdateOp = {2, 1, ?MAX_PRIORITY, ?MAX_PRIORITY}, + try + ets:update_counter(Tid, Key, UpdateOp) + catch + error:badarg -> ok + end. + +reset_usage(Tid, Key) -> + % Reset the value of the usage to 0. Since max value is ?MAX_PRIORITY, + % subtract that and clamp it at 0. Do not provide a default since if an + % entry is missing we don't want to create a bogus one from this operation. + UpdateOp = {2, -?MAX_PRIORITY, 0, 0}, + try + ets:update_counter(Tid, Key, UpdateOp) + catch + error:badarg -> ok + end. + +info(#cache{tid = Tid}) -> + Memory = ets:info(Tid, memory) * wordsize(), + Size = ets:info(Tid, size), + {Size, Memory}. + +decay(Tid) -> + MatchSpec = ets:fun2ms( + fun({Key, Usage, Term}) when Usage > 0 -> + {Key, Usage bsr 1, Term} + end + ), + ets:select_replace(Tid, MatchSpec). + +shard_count() -> + % Use a minimum size of 16 even for there are less than 16 schedulers + % to keep the total tables size a bit smaller + max(16, erlang:system_info(schedulers)). + +wait_interval(Interval) -> + Jitter = rand:uniform(max(1, Interval bsr 1)), + Interval + Jitter. + +max_size() -> + config:get_integer("bt_engine_cache", "max_size", ?DEFAULT_SIZE). + +leave_percent() -> + Val = config:get_integer("bt_engine_cache", "leave_percent", ?DEFAULT_LEAVE_PERCENT), + max(0, min(90, Val)). + +now_msec() -> + erlang:monotonic_time(millisecond). + +bump_metric(Metric) when is_atom(Metric) -> + couch_stats:increment_counter([couchdb, bt_engine_cache, Metric]). + +sample_metric(Metric) when is_atom(Metric) -> + try + couch_stats:sample([couchdb, bt_engine_cache, Metric]) + catch + throw:unknown_metric -> + 0 + end. + +% ETS sizes are expressed in "words". To get the byte size need to multiply +% memory sizes by the wordsize. On 64 bit systems this should be 8 +% +wordsize() -> + erlang:system_info(wordsize). diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl index 8ed55b5c39..37295cb16f 100644 --- a/src/couch/src/couch_bt_engine_compactor.erl +++ b/src/couch/src/couch_bt_engine_compactor.erl @@ -147,7 +147,7 @@ open_compaction_files(DbName, OldSt, Options) -> } end, unlink(DataFd), - erlang:monitor(process, MetaFd), + monitor(process, MetaFd), {ok, CompSt}. copy_purge_info(#comp_st{} = CompSt) -> @@ -381,7 +381,7 @@ copy_compact(#comp_st{} = CompSt) -> % Copy general properties over Props = couch_bt_engine:get_props(St), - {ok, NewSt5} = couch_bt_engine:set_props(NewSt4, Props), + {ok, NewSt5} = couch_bt_engine:copy_props(NewSt4, Props), FinalUpdateSeq = couch_bt_engine:get_update_seq(St), {ok, NewSt6} = couch_bt_engine:set_update_seq(NewSt5, FinalUpdateSeq), @@ -693,7 +693,7 @@ merge_lookups([#doc_info{} = DI | RestInfos], [{ok, FDI} | RestLookups]) -> % Assert we've matched our lookups if DI#doc_info.id == FDI#full_doc_info.id -> ok; - true -> erlang:error({mismatched_doc_infos, DI#doc_info.id}) + true -> error({mismatched_doc_infos, DI#doc_info.id}) end, [FDI | merge_lookups(RestInfos, RestLookups)]; merge_lookups([FDI | RestInfos], Lookups) -> diff --git a/src/couch/src/couch_bt_engine_header.erl b/src/couch/src/couch_bt_engine_header.erl index 3581b1e398..e6e211de3c 100644 --- a/src/couch/src/couch_bt_engine_header.erl +++ b/src/couch/src/couch_bt_engine_header.erl @@ -204,20 +204,19 @@ upgrade_tuple(Old) when is_record(Old, db_header) -> Old; upgrade_tuple(Old) when is_tuple(Old) -> NewSize = record_info(size, db_header), - if - tuple_size(Old) < NewSize -> ok; - true -> erlang:error({invalid_header_size, Old}) - end, - {_, New} = lists:foldl( - fun(Val, {Idx, Hdr}) -> - {Idx + 1, setelement(Idx, Hdr, Val)} + Upgrade = tuple_size(Old) < NewSize, + ProhibitDowngrade = config:get_boolean("couchdb", "prohibit_downgrade", true), + OldKVs = + case {Upgrade, ProhibitDowngrade} of + {true, AnyBool} when is_boolean(AnyBool) -> tuple_to_list(Old); + {false, true} -> error({invalid_header_size, Old}); + {false, false} -> lists:sublist(tuple_to_list(Old), NewSize) end, - {1, #db_header{}}, - tuple_to_list(Old) - ), + FoldFun = fun(Val, {Idx, Hdr}) -> {Idx + 1, setelement(Idx, Hdr, Val)} end, + {_, New} = lists:foldl(FoldFun, {1, #db_header{}}, OldKVs), if is_record(New, db_header) -> ok; - true -> erlang:error({invalid_header_extension, {Old, New}}) + true -> error({invalid_header_extension, {Old, New}}) end, New. @@ -338,7 +337,7 @@ latest(_Else) -> undefined. -ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_eunit.hrl"). mk_header(Vsn) -> { @@ -478,4 +477,61 @@ get_epochs_from_old_header_test() -> Vsn5Header = mk_header(5), ?assertEqual(undefined, epochs(Vsn5Header)). +tuple_uprade_test_() -> + { + foreach, + fun() -> + Ctx = test_util:start_couch(), + config:set("couchdb", "prohibit_downgrade", "true", false), + Ctx + end, + fun(Ctx) -> + config:delete("couchdb", "prohibit_downgrade", false), + test_util:stop_couch(Ctx) + end, + [ + ?TDEF_FE(t_upgrade_tuple_same_size), + ?TDEF_FE(t_upgrade_tuple), + ?TDEF_FE(t_downgrade_default), + ?TDEF_FE(t_downgrade_allowed) + ] + }. + +t_upgrade_tuple_same_size(_) -> + Hdr = #db_header{disk_version = ?LATEST_DISK_VERSION}, + Hdr1 = upgrade_tuple(Hdr), + ?assertEqual(Hdr, Hdr1). + +t_upgrade_tuple(_) -> + Hdr = {db_header, ?LATEST_DISK_VERSION, 101}, + Hdr1 = upgrade_tuple(Hdr), + ?assertMatch( + #db_header{ + disk_version = ?LATEST_DISK_VERSION, + update_seq = 101, + purge_infos_limit = 1000 + }, + Hdr1 + ). + +t_downgrade_default(_) -> + Junk = lists:duplicate(50, x), + Hdr = list_to_tuple([db_header, ?LATEST_DISK_VERSION] ++ Junk), + % Not allowed by default + ?assertError({invalid_header_size, _}, upgrade_tuple(Hdr)). + +t_downgrade_allowed(_) -> + Junk = lists:duplicate(50, x), + Hdr = list_to_tuple([db_header, ?LATEST_DISK_VERSION, 42] ++ Junk), + config:set("couchdb", "prohibit_downgrade", "false", false), + Hdr1 = upgrade_tuple(Hdr), + ?assert(is_record(Hdr1, db_header)), + ?assertMatch( + #db_header{ + disk_version = ?LATEST_DISK_VERSION, + update_seq = 42 + }, + Hdr1 + ). + -endif. diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl index b974a22eec..1519b1fbcb 100644 --- a/src/couch/src/couch_btree.erl +++ b/src/couch/src/couch_btree.erl @@ -14,11 +14,30 @@ -export([open/2, open/3, query_modify/4, add/2, add_remove/3]). -export([fold/4, full_reduce/1, final_reduce/2, size/1, foldl/3, foldl/4]). --export([fold_reduce/4, lookup/2, get_state/1, set_options/2]). +-export([fold_reduce/4, lookup/2, set_options/2]). +-export([is_btree/1, get_state/1, get_fd/1, get_reduce_fun/1]). -export([extract/2, assemble/3, less/3]). -include_lib("couch/include/couch_db.hrl"). +-define(DEFAULT_CHUNK_SIZE, 1279). + +% For the btree cache, the priority of the root node will be +% this value. The priority is roughly how many cleanup interval +% (second) they'll survive without any updates in the cache +-define(ROOT_NODE_CACHE_PRIORITY, 8). + +-record(btree, { + fd, + root, + extract_kv, + assemble_kv, + less, + reduce = nil, + compression = ?DEFAULT_COMPRESSION, + cache_depth = 0 +}). + -define(FILL_RATIO, 0.5). extract(#btree{extract_kv = undefined}, Value) -> @@ -31,11 +50,18 @@ assemble(#btree{assemble_kv = undefined}, Key, Value) -> assemble(#btree{assemble_kv = Assemble}, Key, Value) -> Assemble(Key, Value). +-compile({inline, [less/3]}). less(#btree{less = undefined}, A, B) -> A < B; less(#btree{less = Less}, A, B) -> Less(A, B). +-compile({inline, [less_eq/3]}). +less_eq(#btree{less = undefined}, A, B) -> + A =< B; +less_eq(#btree{less = Less}, A, B) -> + Less(A, B) orelse not Less(B, A). + % pass in 'nil' for State if a new Btree. open(State, Fd) -> {ok, #btree{root = State, fd = Fd}}. @@ -51,14 +77,27 @@ set_options(Bt, [{less, Less} | Rest]) -> set_options(Bt, [{reduce, Reduce} | Rest]) -> set_options(Bt#btree{reduce = Reduce}, Rest); set_options(Bt, [{compression, Comp} | Rest]) -> - set_options(Bt#btree{compression = Comp}, Rest). + set_options(Bt#btree{compression = Comp}, Rest); +set_options(Bt, [{cache_depth, Depth} | Rest]) when is_integer(Depth) -> + set_options(Bt#btree{cache_depth = Depth}, Rest). open(State, Fd, Options) -> {ok, set_options(#btree{root = State, fd = Fd}, Options)}. +is_btree(#btree{}) -> + true; +is_btree(_) -> + false. + get_state(#btree{root = Root}) -> Root. +get_fd(#btree{fd = Fd}) -> + Fd. + +get_reduce_fun(#btree{reduce = Reduce}) -> + Reduce. + final_reduce(#btree{reduce = Reduce}, Val) -> final_reduce(Reduce, Val); final_reduce(Reduce, {[], []}) -> @@ -89,7 +128,8 @@ fold_reduce(#btree{root = Root} = Bt, Fun, Acc, Options) -> [], KeyGroupFun, Fun, - Acc + Acc, + 0 ), if GroupedKey2 == undefined -> @@ -236,7 +276,8 @@ fold(#btree{root = Root} = Bt, Fun, Acc, Options) -> InRange, Dir, convert_fun_arity(Fun), - Acc + Acc, + 0 ); StartKey -> stream_node( @@ -247,7 +288,8 @@ fold(#btree{root = Root} = Bt, Fun, Acc, Options) -> InRange, Dir, convert_fun_arity(Fun), - Acc + Acc, + 0 ) end, case Result of @@ -276,6 +318,11 @@ query_modify(Bt, LookupKeys, InsertValues, RemoveKeys) -> ), RemoveActions = [{remove, Key, nil} || Key <- RemoveKeys], FetchActions = [{fetch, Key, nil} || Key <- LookupKeys], + + UniqueFun = fun({Op, A, _}, {Op, B, _}) -> less_eq(Bt, A, B) end, + InsertActions1 = lists:usort(UniqueFun, InsertActions), + RemoveActions1 = lists:usort(UniqueFun, RemoveActions), + SortFun = fun({OpA, A, _}, {OpB, B, _}) -> case A == B of @@ -284,8 +331,8 @@ query_modify(Bt, LookupKeys, InsertValues, RemoveKeys) -> false -> less(Bt, A, B) end end, - Actions = lists:sort(SortFun, lists:append([InsertActions, RemoveActions, FetchActions])), - {ok, KeyPointers, QueryResults} = modify_node(Bt, Root, Actions, []), + Actions = lists:sort(SortFun, lists:append([InsertActions1, RemoveActions1, FetchActions])), + {ok, KeyPointers, QueryResults} = modify_node(Bt, Root, Actions, [], 0), {ok, NewRoot} = complete_root(Bt, KeyPointers), {ok, QueryResults, Bt#btree{root = NewRoot}}. @@ -301,64 +348,87 @@ lookup(#btree{root = Root, less = Less} = Bt, Keys) -> undefined -> lists:sort(Keys); _ -> lists:sort(Less, Keys) end, - {ok, SortedResults} = lookup(Bt, Root, SortedKeys), + {ok, SortedResults} = lookup(Bt, Root, SortedKeys, 0), % We want to return the results in the same order as the keys were input % but we may have changed the order when we sorted. So we need to put the % order back into the results. couch_util:reorder_results(Keys, SortedResults). -lookup(_Bt, nil, Keys) -> +lookup(_Bt, nil, Keys, _Depth) -> {ok, [{Key, not_found} || Key <- Keys]}; -lookup(Bt, Node, Keys) -> +lookup(Bt, Node, Keys, Depth0) -> + Depth = Depth0 + 1, Pointer = element(1, Node), - {NodeType, NodeList} = get_node(Bt, Pointer), + {NodeType, NodeList} = get_node(Bt, Pointer, Depth), case NodeType of kp_node -> - lookup_kpnode(Bt, list_to_tuple(NodeList), 1, Keys, []); + lookup_kpnode(Bt, list_to_tuple(NodeList), 1, Keys, [], Depth); kv_node -> - lookup_kvnode(Bt, list_to_tuple(NodeList), 1, Keys, []) + lookup_kvnode(Bt, list_to_tuple(NodeList), 1, Keys, [], Depth) end. -lookup_kpnode(_Bt, _NodeTuple, _LowerBound, [], Output) -> +lookup_kpnode(_Bt, _NodeTuple, _LowerBound, [], Output, _Depth) -> {ok, lists:reverse(Output)}; -lookup_kpnode(_Bt, NodeTuple, LowerBound, Keys, Output) when tuple_size(NodeTuple) < LowerBound -> +lookup_kpnode(_Bt, NodeTuple, LowerBound, Keys, Output, _Depth) when + tuple_size(NodeTuple) < LowerBound +-> {ok, lists:reverse(Output, [{Key, not_found} || Key <- Keys])}; -lookup_kpnode(Bt, NodeTuple, LowerBound, [FirstLookupKey | _] = LookupKeys, Output) -> +lookup_kpnode(Bt, NodeTuple, LowerBound, [FirstLookupKey | _] = LookupKeys, Output, Depth) -> N = find_first_gteq(Bt, NodeTuple, LowerBound, tuple_size(NodeTuple), FirstLookupKey), {Key, PointerInfo} = element(N, NodeTuple), SplitFun = fun(LookupKey) -> not less(Bt, Key, LookupKey) end, case lists:splitwith(SplitFun, LookupKeys) of {[], GreaterQueries} -> - lookup_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, Output); + lookup_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, Output, Depth); {LessEqQueries, GreaterQueries} -> - {ok, Results} = lookup(Bt, PointerInfo, LessEqQueries), - lookup_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, lists:reverse(Results, Output)) + {ok, Results} = lookup(Bt, PointerInfo, LessEqQueries, Depth), + lookup_kpnode( + Bt, NodeTuple, N + 1, GreaterQueries, lists:reverse(Results, Output), Depth + ) end. -lookup_kvnode(_Bt, _NodeTuple, _LowerBound, [], Output) -> +lookup_kvnode(_Bt, _NodeTuple, _LowerBound, [], Output, _Depth) -> {ok, lists:reverse(Output)}; -lookup_kvnode(_Bt, NodeTuple, LowerBound, Keys, Output) when tuple_size(NodeTuple) < LowerBound -> +lookup_kvnode(_Bt, NodeTuple, LowerBound, Keys, Output, _Depth) when + tuple_size(NodeTuple) < LowerBound +-> % keys not found {ok, lists:reverse(Output, [{Key, not_found} || Key <- Keys])}; -lookup_kvnode(Bt, NodeTuple, LowerBound, [LookupKey | RestLookupKeys], Output) -> +lookup_kvnode(Bt, NodeTuple, LowerBound, [LookupKey | RestLookupKeys], Output, Depth) -> N = find_first_gteq(Bt, NodeTuple, LowerBound, tuple_size(NodeTuple), LookupKey), {Key, Value} = element(N, NodeTuple), case less(Bt, LookupKey, Key) of true -> % LookupKey is less than Key - lookup_kvnode(Bt, NodeTuple, N, RestLookupKeys, [{LookupKey, not_found} | Output]); + lookup_kvnode( + Bt, NodeTuple, N, RestLookupKeys, [{LookupKey, not_found} | Output], Depth + ); false -> case less(Bt, Key, LookupKey) of true -> % LookupKey is greater than Key - lookup_kvnode(Bt, NodeTuple, N + 1, RestLookupKeys, [ - {LookupKey, not_found} | Output - ]); + lookup_kvnode( + Bt, + NodeTuple, + N + 1, + RestLookupKeys, + [ + {LookupKey, not_found} | Output + ], + Depth + ); false -> % LookupKey is equal to Key - lookup_kvnode(Bt, NodeTuple, N, RestLookupKeys, [ - {LookupKey, {ok, assemble(Bt, LookupKey, Value)}} | Output - ]) + lookup_kvnode( + Bt, + NodeTuple, + N, + RestLookupKeys, + [ + {LookupKey, {ok, assemble(Bt, LookupKey, Value)}} | Output + ], + Depth + ) end end. @@ -407,32 +477,29 @@ chunkify([InElement | RestInList], ChunkThreshold, OutList, OutListSize, OutputC -compile({inline, [get_chunk_size/0]}). get_chunk_size() -> - try - list_to_integer(config:get("couchdb", "btree_chunk_size", "1279")) - catch - error:badarg -> - 1279 - end. + config:get_integer("couchdb", "btree_chunk_size", ?DEFAULT_CHUNK_SIZE). -modify_node(Bt, RootPointerInfo, Actions, QueryOutput) -> +modify_node(Bt, RootPointerInfo, Actions, QueryOutput, Depth0) -> + Depth = Depth0 + 1, {NodeType, NodeList} = case RootPointerInfo of nil -> {kv_node, []}; _Tuple -> Pointer = element(1, RootPointerInfo), - get_node(Bt, Pointer) + get_node(Bt, Pointer, Depth) end, NodeTuple = list_to_tuple(NodeList), {ok, NewNodeList, QueryOutput2} = case NodeType of - kp_node -> modify_kpnode(Bt, NodeTuple, 1, Actions, [], QueryOutput); - kv_node -> modify_kvnode(Bt, NodeTuple, 1, Actions, [], QueryOutput) + kp_node -> modify_kpnode(Bt, NodeTuple, 1, Actions, [], QueryOutput, Depth); + kv_node -> modify_kvnode(Bt, NodeTuple, 1, Actions, [], QueryOutput, Depth) end, case NewNodeList of % no nodes remain [] -> + reset_cache_usage(Bt, RootPointerInfo, Depth), {ok, [], QueryOutput2}; % nothing changed NodeList -> @@ -444,6 +511,7 @@ modify_node(Bt, RootPointerInfo, Actions, QueryOutput) -> nil -> write_node(Bt, NodeType, NewNodeList); _ -> + reset_cache_usage(Bt, RootPointerInfo, Depth), {LastKey, _LastValue} = element(tuple_size(NodeTuple), NodeTuple), OldNode = {LastKey, RootPointerInfo}, write_node(Bt, OldNode, NodeType, NodeList, NewNodeList) @@ -470,7 +538,30 @@ reduce_tree_size(kp_node, _NodeSize, [{_K, {_P, _Red, nil}} | _]) -> reduce_tree_size(kp_node, NodeSize, [{_K, {_P, _Red, Sz}} | NodeList]) -> reduce_tree_size(kp_node, NodeSize + Sz, NodeList). -get_node(#btree{fd = Fd}, NodePos) -> +reset_cache_usage(_, nil, _Depth) -> + ok; +reset_cache_usage(#btree{cache_depth = Max}, _, Depth) when Depth > Max -> + ok; +reset_cache_usage(#btree{fd = Fd}, RootPointerInfo, _Depth) -> + Pointer = element(1, RootPointerInfo), + couch_bt_engine_cache:reset({Fd, Pointer}). + +get_node(#btree{fd = Fd, cache_depth = Max}, NodePos, Depth) when Depth =< Max -> + case couch_bt_engine_cache:lookup({Fd, NodePos}) of + undefined -> + {ok, {NodeType, NodeList}} = couch_file:pread_term(Fd, NodePos), + case NodeType of + kp_node -> + Priority = max(1, ?ROOT_NODE_CACHE_PRIORITY - Depth), + couch_bt_engine_cache:insert({Fd, NodePos}, NodeList, Priority); + kv_node -> + ok + end, + {NodeType, NodeList}; + NodeList -> + {kp_node, NodeList} + end; +get_node(#btree{fd = Fd}, NodePos, _Depth) -> {ok, {NodeType, NodeList}} = couch_file:pread_term(Fd, NodePos), {NodeType, NodeList}. @@ -546,9 +637,9 @@ old_list_is_prefix([KV | Rest1], [KV | Rest2], Acc) -> old_list_is_prefix(_OldList, _NewList, _Acc) -> false. -modify_kpnode(Bt, {}, _LowerBound, Actions, [], QueryOutput) -> - modify_node(Bt, nil, Actions, QueryOutput); -modify_kpnode(_Bt, NodeTuple, LowerBound, [], ResultNode, QueryOutput) -> +modify_kpnode(Bt, {}, _LowerBound, Actions, [], QueryOutput, Depth) -> + modify_node(Bt, nil, Actions, QueryOutput, Depth); +modify_kpnode(_Bt, NodeTuple, LowerBound, [], ResultNode, QueryOutput, _Depth) -> {ok, lists:reverse( ResultNode, @@ -566,7 +657,8 @@ modify_kpnode( LowerBound, [{_, FirstActionKey, _} | _] = Actions, ResultNode, - QueryOutput + QueryOutput, + Depth ) -> Sz = tuple_size(NodeTuple), N = find_first_gteq(Bt, NodeTuple, LowerBound, Sz, FirstActionKey), @@ -575,7 +667,7 @@ modify_kpnode( % perform remaining actions on last node {_, PointerInfo} = element(Sz, NodeTuple), {ok, ChildKPs, QueryOutput2} = - modify_node(Bt, PointerInfo, Actions, QueryOutput), + modify_node(Bt, PointerInfo, Actions, QueryOutput, Depth), NodeList = lists:reverse( ResultNode, bounded_tuple_to_list( @@ -593,7 +685,7 @@ modify_kpnode( end, {LessEqQueries, GreaterQueries} = lists:splitwith(SplitFun, Actions), {ok, ChildKPs, QueryOutput2} = - modify_node(Bt, PointerInfo, LessEqQueries, QueryOutput), + modify_node(Bt, PointerInfo, LessEqQueries, QueryOutput, Depth), ResultNode2 = lists:reverse( ChildKPs, bounded_tuple_to_revlist( @@ -603,7 +695,7 @@ modify_kpnode( ResultNode ) ), - modify_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, ResultNode2, QueryOutput2) + modify_kpnode(Bt, NodeTuple, N + 1, GreaterQueries, ResultNode2, QueryOutput2, Depth) end. bounded_tuple_to_revlist(_Tuple, Start, End, Tail) when Start > End -> @@ -631,7 +723,7 @@ find_first_gteq(Bt, Tuple, Start, End, Key) -> find_first_gteq(Bt, Tuple, Start, Mid, Key) end. -modify_kvnode(_Bt, NodeTuple, LowerBound, [], ResultNode, QueryOutput) -> +modify_kvnode(_Bt, NodeTuple, LowerBound, [], ResultNode, QueryOutput, _Depth) -> {ok, lists:reverse( ResultNode, bounded_tuple_to_list(NodeTuple, LowerBound, tuple_size(NodeTuple), []) @@ -643,7 +735,8 @@ modify_kvnode( LowerBound, [{ActionType, ActionKey, ActionValue} | RestActions], ResultNode, - QueryOutput + QueryOutput, + Depth ) when LowerBound > tuple_size(NodeTuple) -> case ActionType of insert -> @@ -653,16 +746,25 @@ modify_kvnode( LowerBound, RestActions, [{ActionKey, ActionValue} | ResultNode], - QueryOutput + QueryOutput, + Depth ); remove -> % just drop the action - modify_kvnode(Bt, NodeTuple, LowerBound, RestActions, ResultNode, QueryOutput); + modify_kvnode(Bt, NodeTuple, LowerBound, RestActions, ResultNode, QueryOutput, Depth); fetch -> % the key/value must not exist in the tree - modify_kvnode(Bt, NodeTuple, LowerBound, RestActions, ResultNode, [ - {not_found, {ActionKey, nil}} | QueryOutput - ]) + modify_kvnode( + Bt, + NodeTuple, + LowerBound, + RestActions, + ResultNode, + [ + {not_found, {ActionKey, nil}} | QueryOutput + ], + Depth + ) end; modify_kvnode( Bt, @@ -670,7 +772,8 @@ modify_kvnode( LowerBound, [{ActionType, ActionKey, ActionValue} | RestActions], AccNode, - QueryOutput + QueryOutput, + Depth ) -> N = find_first_gteq(Bt, NodeTuple, LowerBound, tuple_size(NodeTuple), ActionKey), {Key, Value} = element(N, NodeTuple), @@ -686,16 +789,25 @@ modify_kvnode( N, RestActions, [{ActionKey, ActionValue} | ResultNode], - QueryOutput + QueryOutput, + Depth ); remove -> % ActionKey is less than the Key, just drop the action - modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, QueryOutput); + modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, QueryOutput, Depth); fetch -> % ActionKey is less than the Key, the key/value must not exist in the tree - modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, [ - {not_found, {ActionKey, nil}} | QueryOutput - ]) + modify_kvnode( + Bt, + NodeTuple, + N, + RestActions, + ResultNode, + [ + {not_found, {ActionKey, nil}} | QueryOutput + ], + Depth + ) end; false -> % ActionKey and Key are maybe equal. @@ -709,18 +821,27 @@ modify_kvnode( N + 1, RestActions, [{ActionKey, ActionValue} | ResultNode], - QueryOutput + QueryOutput, + Depth ); remove -> modify_kvnode( - Bt, NodeTuple, N + 1, RestActions, ResultNode, QueryOutput + Bt, NodeTuple, N + 1, RestActions, ResultNode, QueryOutput, Depth ); fetch -> % ActionKey is equal to the Key, insert into the QueryOuput, but re-process the node % since an identical action key can follow it. - modify_kvnode(Bt, NodeTuple, N, RestActions, ResultNode, [ - {ok, assemble(Bt, Key, Value)} | QueryOutput - ]) + modify_kvnode( + Bt, + NodeTuple, + N, + RestActions, + ResultNode, + [ + {ok, assemble(Bt, Key, Value)} | QueryOutput + ], + Depth + ) end; true -> modify_kvnode( @@ -729,7 +850,8 @@ modify_kvnode( N + 1, [{ActionType, ActionKey, ActionValue} | RestActions], [{Key, Value} | ResultNode], - QueryOutput + QueryOutput, + Depth ) end end. @@ -745,7 +867,8 @@ reduce_stream_node( GroupedRedsAcc, _KeyGroupFun, _Fun, - Acc + Acc, + _Depth ) -> {ok, Acc, GroupedRedsAcc, GroupedKVsAcc, GroupedKey}; reduce_stream_node( @@ -759,10 +882,12 @@ reduce_stream_node( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth0 ) -> + Depth = Depth0 + 1, P = element(1, Node), - case get_node(Bt, P) of + case get_node(Bt, P, Depth) of {kp_node, NodeList} -> NodeList2 = adjust_dir(Dir, NodeList), reduce_stream_kp_node( @@ -776,7 +901,8 @@ reduce_stream_node( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ); {kv_node, KVs} -> KVs2 = adjust_dir(Dir, KVs), @@ -791,7 +917,8 @@ reduce_stream_node( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ) end. @@ -806,7 +933,8 @@ reduce_stream_kv_node( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ) -> GTEKeyStartKVs = case KeyStart of @@ -833,7 +961,8 @@ reduce_stream_kv_node( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ). reduce_stream_kv_node2( @@ -844,7 +973,8 @@ reduce_stream_kv_node2( GroupedRedsAcc, _KeyGroupFun, _Fun, - Acc + Acc, + _Depth ) -> {ok, Acc, GroupedRedsAcc, GroupedKVsAcc, GroupedKey}; reduce_stream_kv_node2( @@ -855,7 +985,8 @@ reduce_stream_kv_node2( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ) -> case GroupedKey of undefined -> @@ -867,7 +998,8 @@ reduce_stream_kv_node2( [], KeyGroupFun, Fun, - Acc + Acc, + Depth ); _ -> case KeyGroupFun(GroupedKey, Key) of @@ -880,7 +1012,8 @@ reduce_stream_kv_node2( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ); false -> case Fun(GroupedKey, {GroupedKVsAcc, GroupedRedsAcc}, Acc) of @@ -893,7 +1026,8 @@ reduce_stream_kv_node2( [], KeyGroupFun, Fun, - Acc2 + Acc2, + Depth ); {stop, Acc2} -> throw({stop, Acc2}) @@ -912,7 +1046,8 @@ reduce_stream_kp_node( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ) -> Nodes = case KeyStart of @@ -955,7 +1090,8 @@ reduce_stream_kp_node( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ). reduce_stream_kp_node2( @@ -969,7 +1105,8 @@ reduce_stream_kp_node2( [], KeyGroupFun, Fun, - Acc + Acc, + Depth ) -> {ok, Acc2, GroupedRedsAcc2, GroupedKVsAcc2, GroupedKey2} = reduce_stream_node( @@ -983,7 +1120,8 @@ reduce_stream_kp_node2( [], KeyGroupFun, Fun, - Acc + Acc, + Depth ), reduce_stream_kp_node2( Bt, @@ -996,7 +1134,8 @@ reduce_stream_kp_node2( GroupedRedsAcc2, KeyGroupFun, Fun, - Acc2 + Acc2, + Depth ); reduce_stream_kp_node2( Bt, @@ -1009,7 +1148,8 @@ reduce_stream_kp_node2( GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ) -> {Grouped0, Ungrouped0} = lists:splitwith( fun({Key, _}) -> @@ -1040,7 +1180,8 @@ reduce_stream_kp_node2( GroupedReds ++ GroupedRedsAcc, KeyGroupFun, Fun, - Acc + Acc, + Depth ), reduce_stream_kp_node2( Bt, @@ -1053,7 +1194,8 @@ reduce_stream_kp_node2( GroupedRedsAcc2, KeyGroupFun, Fun, - Acc2 + Acc2, + Depth ); [] -> {ok, Acc, GroupedReds ++ GroupedRedsAcc, GroupedKVsAcc, GroupedKey} @@ -1064,40 +1206,46 @@ adjust_dir(fwd, List) -> adjust_dir(rev, List) -> lists:reverse(List). -stream_node(Bt, Reds, Node, StartKey, InRange, Dir, Fun, Acc) -> +stream_node(Bt, Reds, Node, StartKey, InRange, Dir, Fun, Acc, Depth0) -> + Depth = Depth0 + 1, Pointer = element(1, Node), - {NodeType, NodeList} = get_node(Bt, Pointer), + {NodeType, NodeList} = get_node(Bt, Pointer, Depth), case NodeType of kp_node -> - stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), StartKey, InRange, Dir, Fun, Acc); + stream_kp_node( + Bt, Reds, adjust_dir(Dir, NodeList), StartKey, InRange, Dir, Fun, Acc, Depth + ); kv_node -> - stream_kv_node(Bt, Reds, adjust_dir(Dir, NodeList), StartKey, InRange, Dir, Fun, Acc) + stream_kv_node( + Bt, Reds, adjust_dir(Dir, NodeList), StartKey, InRange, Dir, Fun, Acc, Depth + ) end. -stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc) -> +stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc, Depth0) -> + Depth = Depth0 + 1, Pointer = element(1, Node), - {NodeType, NodeList} = get_node(Bt, Pointer), + {NodeType, NodeList} = get_node(Bt, Pointer, Depth), case NodeType of kp_node -> - stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc); + stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc, Depth); kv_node -> - stream_kv_node2(Bt, Reds, [], adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc) + stream_kv_node2(Bt, Reds, [], adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc, Depth) end. -stream_kp_node(_Bt, _Reds, [], _InRange, _Dir, _Fun, Acc) -> +stream_kp_node(_Bt, _Reds, [], _InRange, _Dir, _Fun, Acc, _Depth) -> {ok, Acc}; -stream_kp_node(Bt, Reds, [{Key, Node} | Rest], InRange, Dir, Fun, Acc) -> +stream_kp_node(Bt, Reds, [{Key, Node} | Rest], InRange, Dir, Fun, Acc, Depth) -> Red = element(2, Node), case Fun(traverse, Key, Red, Acc) of {ok, Acc2} -> - case stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc2) of + case stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc2, Depth) of {ok, Acc3} -> - stream_kp_node(Bt, [Red | Reds], Rest, InRange, Dir, Fun, Acc3); + stream_kp_node(Bt, [Red | Reds], Rest, InRange, Dir, Fun, Acc3, Depth); {stop, LastReds, Acc3} -> {stop, LastReds, Acc3} end; {skip, Acc2} -> - stream_kp_node(Bt, [Red | Reds], Rest, InRange, Dir, Fun, Acc2); + stream_kp_node(Bt, [Red | Reds], Rest, InRange, Dir, Fun, Acc2, Depth); {stop, Acc2} -> {stop, Reds, Acc2} end. @@ -1112,7 +1260,7 @@ drop_nodes(Bt, Reds, StartKey, [{NodeKey, Node} | RestKPs]) -> {Reds, [{NodeKey, Node} | RestKPs]} end. -stream_kp_node(Bt, Reds, KPs, StartKey, InRange, Dir, Fun, Acc) -> +stream_kp_node(Bt, Reds, KPs, StartKey, InRange, Dir, Fun, Acc, Depth) -> {NewReds, NodesToStream} = case Dir of fwd -> @@ -1135,16 +1283,16 @@ stream_kp_node(Bt, Reds, KPs, StartKey, InRange, Dir, Fun, Acc) -> [] -> {ok, Acc}; [{_Key, Node} | Rest] -> - case stream_node(Bt, NewReds, Node, StartKey, InRange, Dir, Fun, Acc) of + case stream_node(Bt, NewReds, Node, StartKey, InRange, Dir, Fun, Acc, Depth) of {ok, Acc2} -> Red = element(2, Node), - stream_kp_node(Bt, [Red | NewReds], Rest, InRange, Dir, Fun, Acc2); + stream_kp_node(Bt, [Red | NewReds], Rest, InRange, Dir, Fun, Acc2, Depth); {stop, LastReds, Acc2} -> {stop, LastReds, Acc2} end end. -stream_kv_node(Bt, Reds, KVs, StartKey, InRange, Dir, Fun, Acc) -> +stream_kv_node(Bt, Reds, KVs, StartKey, InRange, Dir, Fun, Acc, Depth) -> DropFun = case Dir of fwd -> @@ -1154,11 +1302,11 @@ stream_kv_node(Bt, Reds, KVs, StartKey, InRange, Dir, Fun, Acc) -> end, {LTKVs, GTEKVs} = lists:splitwith(DropFun, KVs), AssembleLTKVs = [assemble(Bt, K, V) || {K, V} <- LTKVs], - stream_kv_node2(Bt, Reds, AssembleLTKVs, GTEKVs, InRange, Dir, Fun, Acc). + stream_kv_node2(Bt, Reds, AssembleLTKVs, GTEKVs, InRange, Dir, Fun, Acc, Depth). -stream_kv_node2(_Bt, _Reds, _PrevKVs, [], _InRange, _Dir, _Fun, Acc) -> +stream_kv_node2(_Bt, _Reds, _PrevKVs, [], _InRange, _Dir, _Fun, Acc, _Depth) -> {ok, Acc}; -stream_kv_node2(Bt, Reds, PrevKVs, [{K, V} | RestKVs], InRange, Dir, Fun, Acc) -> +stream_kv_node2(Bt, Reds, PrevKVs, [{K, V} | RestKVs], InRange, Dir, Fun, Acc, Depth) -> case InRange(K) of false -> {stop, {PrevKVs, Reds}, Acc}; @@ -1167,7 +1315,7 @@ stream_kv_node2(Bt, Reds, PrevKVs, [{K, V} | RestKVs], InRange, Dir, Fun, Acc) - case Fun(visit, AssembledKV, {PrevKVs, Reds}, Acc) of {ok, Acc2} -> stream_kv_node2( - Bt, Reds, [AssembledKV | PrevKVs], RestKVs, InRange, Dir, Fun, Acc2 + Bt, Reds, [AssembledKV | PrevKVs], RestKVs, InRange, Dir, Fun, Acc2, Depth ); {stop, Acc2} -> {stop, {PrevKVs, Reds}, Acc2} diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index e33e695c02..fe540ec867 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -67,6 +67,9 @@ set_security/2, set_user_ctx/2, + get_props/1, + update_props/3, + load_validation_funs/1, reload_validation_funs/1, @@ -152,6 +155,11 @@ % Purge client max lag window in seconds (defaulting to 24 hours) -define(PURGE_LAG_SEC, 86400). +% DB props which cannot be dynamically updated after db creation +-define(PROP_PARTITIONED, partitioned). +-define(PROP_HASH, hash). +-define(STATIC_PROPS, [?PROP_PARTITIONED, ?PROP_HASH]). + start_link(Engine, DbName, Filepath, Options) -> Arg = {Engine, DbName, Filepath, Options}, proc_lib:start_link(couch_db_updater, init, [Arg]). @@ -227,9 +235,9 @@ is_clustered(#db{}) -> is_clustered(?OLD_DB_REC = Db) -> ?OLD_DB_MAIN_PID(Db) == undefined. -is_partitioned(#db{options = Options}) -> - Props = couch_util:get_value(props, Options, []), - couch_util:get_value(partitioned, Props, false). +is_partitioned(#db{} = Db) -> + Props = get_props(Db), + couch_util:get_value(?PROP_PARTITIONED, Props, false). close(#db{} = Db) -> ok = couch_db_engine:decref(Db); @@ -254,7 +262,7 @@ monitored_by(Db) -> end. monitor(#db{main_pid = MainPid}) -> - erlang:monitor(process, MainPid). + monitor(process, MainPid). start_compact(#db{} = Db) -> gen_server:call(Db#db.main_pid, start_compact). @@ -269,7 +277,7 @@ wait_for_compaction(#db{main_pid = Pid} = Db, Timeout) -> Start = os:timestamp(), case gen_server:call(Pid, compactor_pid) of CPid when is_pid(CPid) -> - Ref = erlang:monitor(process, CPid), + Ref = monitor(process, CPid), receive {'DOWN', Ref, _, _, normal} when Timeout == infinity -> wait_for_compaction(Db, Timeout); @@ -279,7 +287,7 @@ wait_for_compaction(#db{main_pid = Pid} = Db, Timeout) -> {'DOWN', Ref, _, _, Reason} -> {error, Reason} after Timeout -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), {error, Timeout} end; _ -> @@ -420,7 +428,7 @@ purge_docs(#db{main_pid = Pid} = Db, UUIDsIdsRevs, Options) -> % Gather any existing purges with the same UUIDs UUIDs = element(1, lists:unzip3(UUIDsIdsRevs1)), Old1 = get_purge_infos(Db, UUIDs), - Old2 = maps:from_list([{UUID, {Id, Revs}} || {_, UUID, Id, Revs} <- Old1]), + Old2 = #{UUID => {Id, Revs} || {_, UUID, Id, Revs} <- Old1}, % Filter out all the purges which have already been processed FilterCheckFun = fun({UUID, Id, Revs}) -> case maps:is_key(UUID, Old2) of @@ -478,7 +486,7 @@ get_minimum_purge_seq(#db{} = Db) -> CS when is_integer(CS) -> case purge_client_exists(DbName, DocId, Props) of true -> - {ok, erlang:min(CS, SeqAcc)}; + {ok, min(CS, SeqAcc)}; false -> Fmt1 = "Missing or stale purge doc '~s' on ~p " @@ -489,7 +497,7 @@ get_minimum_purge_seq(#db{} = Db) -> _ -> Fmt2 = "Invalid purge doc '~s' on ~p with purge_seq '~w'", couch_log:error(Fmt2, [DocId, DbName, ClientSeq]), - {ok, erlang:min(OldestPurgeSeq, SeqAcc)} + {ok, min(OldestPurgeSeq, SeqAcc)} end; _ -> {stop, SeqAcc} @@ -503,7 +511,7 @@ get_minimum_purge_seq(#db{} = Db) -> FinalSeq = case MinIdxSeq < PurgeSeq - PurgeInfosLimit of true -> MinIdxSeq; - false -> erlang:max(0, PurgeSeq - PurgeInfosLimit) + false -> max(0, PurgeSeq - PurgeInfosLimit) end, % Log a warning if we've got a purge sequence exceeding the % configured threshold. @@ -626,11 +634,7 @@ get_db_info(Db) -> undefined -> null; Else1 -> Else1 end, - Props = - case couch_db_engine:get_props(Db) of - undefined -> null; - Else2 -> {Else2} - end, + Props = get_props(Db), InfoList = [ {db_name, Name}, {engine, couch_db_engine:get_engine(Db)}, @@ -644,7 +648,7 @@ get_db_info(Db) -> {disk_format_version, DiskVersion}, {committed_update_seq, CommittedUpdateSeq}, {compacted_seq, CompactedSeq}, - {props, Props}, + {props, {Props}}, {uuid, Uuid} ], {ok, InfoList}. @@ -837,6 +841,24 @@ set_revs_limit(#db{main_pid = Pid} = Db, Limit) when Limit > 0 -> set_revs_limit(_Db, _Limit) -> throw(invalid_revs_limit). +get_props(#db{options = Options}) -> + couch_util:get_value(props, Options, []). + +update_props(#db{main_pid = Pid} = Db, K, V) -> + check_is_admin(Db), + case lists:member(K, ?STATIC_PROPS) of + true -> + throw({bad_request, <<"cannot update static property">>}); + false -> + Props = get_props(Db), + Props1 = + case V of + undefined -> lists:keydelete(K, 1, Props); + _ -> lists:keystore(K, 1, Props, {K, V}) + end, + gen_server:call(Pid, {set_props, Props1}, infinity) + end. + name(#db{name = Name}) -> Name; name(?OLD_DB_REC = Db) -> @@ -977,7 +999,7 @@ load_validation_funs(#db{main_pid = Pid, name = <<"shards/", _/binary>>} = Db) - Funs; {'DOWN', Ref, _, _, {database_does_not_exist, _StackTrace}} -> ok = couch_server:close_db_if_idle(Db#db.name), - erlang:error(database_does_not_exist); + error(database_does_not_exist); {'DOWN', Ref, _, _, Reason} -> couch_log:error("could not load validation funs ~p", [Reason]), throw(internal_server_error) @@ -1255,7 +1277,7 @@ new_revid(#doc{body = Body, revs = {OldStart, OldRevs}, atts = Atts, deleted = D case DigestedAtts of Atts2 when length(Atts) =/= length(Atts2) -> % We must have old style non-md5 attachments - ?l2b(integer_to_list(couch_util:rand32())); + integer_to_binary(couch_util:rand32()); Atts2 -> OldRev = case OldRevs of @@ -1340,8 +1362,13 @@ update_docs(Db, Docs0, Options, ?REPLICATED_CHANGES) -> {ok, DocErrors}; update_docs(Db, Docs0, Options, ?INTERACTIVE_EDIT) -> BlockInteractiveDatabaseWrites = couch_disk_monitor:block_interactive_database_writes(), + InternalReplication = + case get(io_priority) of + {internal_repl, _} -> true; + _Else -> false + end, if - BlockInteractiveDatabaseWrites -> + not InternalReplication andalso BlockInteractiveDatabaseWrites -> {ok, [{insufficient_storage, <<"database_dir is too full">>} || _ <- Docs0]}; true -> update_docs_interactive(Db, Docs0, Options) @@ -1466,7 +1493,7 @@ write_and_commit( ) -> DocBuckets = prepare_doc_summaries(Db, DocBuckets1), ReplicatedChanges = lists:member(?REPLICATED_CHANGES, Options), - MRef = erlang:monitor(process, Pid), + MRef = monitor(process, Pid), try Pid ! {update_docs, self(), DocBuckets, LocalDocs, ReplicatedChanges}, case collect_results_with_metrics(Pid, MRef, []) of @@ -1490,7 +1517,7 @@ write_and_commit( end end after - erlang:demonitor(MRef, [flush]) + demonitor(MRef, [flush]) end. prepare_doc_summaries(Db, BucketList) -> @@ -1751,12 +1778,12 @@ validate_epochs(Epochs) -> %% Assert uniqueness. case length(Epochs) == length(lists:ukeysort(2, Epochs)) of true -> ok; - false -> erlang:error(duplicate_epoch) + false -> error(duplicate_epoch) end, %% Assert order. case Epochs == lists:sort(fun({_, A}, {_, B}) -> B =< A end, Epochs) of true -> ok; - false -> erlang:error(epoch_order) + false -> error({epoch_order, Epochs}) end. is_prefix(Pattern, Subject) -> @@ -2352,7 +2379,10 @@ is_owner_test() -> ?assertNot(is_owner(bar, 99, [{baz, 200}, {bar, 100}, {foo, 1}])), ?assertNot(is_owner(baz, 199, [{baz, 200}, {bar, 100}, {foo, 1}])), ?assertError(duplicate_epoch, validate_epochs([{foo, 1}, {bar, 1}])), - ?assertError(epoch_order, validate_epochs([{foo, 100}, {bar, 200}])). + ?assertError( + {epoch_order, [{foo, 100}, {bar, 200}]}, + validate_epochs([{foo, 100}, {bar, 200}]) + ). to_binary(DbName) when is_list(DbName) -> ?l2b(DbName); diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index 3f6c8886dc..b6dfc818da 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -92,6 +92,12 @@ handle_call({set_purge_infos_limit, Limit}, _From, Db) -> {ok, Db2} = couch_db_engine:set_purge_infos_limit(Db, Limit), ok = couch_server:db_updated(Db2), {reply, ok, Db2}; +handle_call({set_props, Props}, _From, Db) -> + {ok, Db1} = couch_db_engine:set_props(Db, Props), + Db2 = options_set_props(Db1, Props), + {ok, Db3} = couch_db_engine:commit_data(Db2), + ok = couch_server:db_updated(Db3), + {reply, ok, Db3}; handle_call({purge_docs, [], _}, _From, Db) -> {reply, {ok, []}, Db}; handle_call({purge_docs, PurgeReqs0, Options}, _From, Db) -> @@ -308,13 +314,17 @@ init_db(DbName, FilePath, EngineState, Options) -> after_doc_read = ADR }, - DbProps = couch_db_engine:get_props(InitDb), - - InitDb#db{ + Db = InitDb#db{ committed_update_seq = couch_db_engine:get_update_seq(InitDb), security = couch_db_engine:get_security(InitDb), - options = lists:keystore(props, 1, NonCreateOpts, {props, DbProps}) - }. + options = NonCreateOpts + }, + DbProps = couch_db_engine:get_props(Db), + options_set_props(Db, DbProps). + +options_set_props(#db{options = Options} = Db, Props) -> + Options1 = lists:keystore(props, 1, Options, {props, Props}), + Db#db{options = Options1}. refresh_validate_doc_funs(#db{name = <<"shards/", _/binary>> = Name} = Db) -> spawn(fabric, reset_validation_funs, [mem3:dbname(Name)]), @@ -795,22 +805,22 @@ purge_docs(Db, PurgeReqs) -> FDIs = couch_db_engine:open_docs(Db, Ids), USeq = couch_db_engine:get_update_seq(Db), - IdFDIs = lists:zip(Ids, FDIs), + IdFDIs = maps:from_list(lists:zip(Ids, FDIs)), {NewIdFDIs, Replies} = apply_purge_reqs(PurgeReqs, IdFDIs, USeq, []), - Pairs = lists:flatmap( - fun({DocId, OldFDI}) -> - {DocId, NewFDI} = lists:keyfind(DocId, 1, NewIdFDIs), - case {OldFDI, NewFDI} of - {not_found, not_found} -> - []; - {#full_doc_info{} = A, #full_doc_info{} = A} -> - []; - {#full_doc_info{}, _} -> - [{OldFDI, NewFDI}] - end - end, - IdFDIs + Pairs = lists:sort( + maps:fold( + fun(DocId, OldFDI, Acc) -> + #{DocId := NewFDI} = NewIdFDIs, + case {OldFDI, NewFDI} of + {not_found, not_found} -> Acc; + {#full_doc_info{} = A, #full_doc_info{} = A} -> Acc; + {#full_doc_info{}, _} -> [{OldFDI, NewFDI} | Acc] + end + end, + [], + IdFDIs + ) ), PSeq = couch_db_engine:get_purge_seq(Db), @@ -834,7 +844,7 @@ apply_purge_reqs([], IdFDIs, _USeq, Replies) -> {IdFDIs, lists:reverse(Replies)}; apply_purge_reqs([Req | RestReqs], IdFDIs, USeq, Replies) -> {_UUID, DocId, Revs} = Req, - {value, {_, FDI0}, RestIdFDIs} = lists:keytake(DocId, 1, IdFDIs), + #{DocId := FDI0} = IdFDIs, {NewFDI, RemovedRevs, NewUSeq} = case FDI0 of #full_doc_info{rev_tree = Tree} -> @@ -872,9 +882,8 @@ apply_purge_reqs([Req | RestReqs], IdFDIs, USeq, Replies) -> % Not found means nothing to change {not_found, [], USeq} end, - NewIdFDIs = [{DocId, NewFDI} | RestIdFDIs], NewReplies = [{ok, RemovedRevs} | Replies], - apply_purge_reqs(RestReqs, NewIdFDIs, NewUSeq, NewReplies). + apply_purge_reqs(RestReqs, IdFDIs#{DocId := NewFDI}, NewUSeq, NewReplies). commit_data(Db) -> {ok, Db1} = couch_db_engine:commit_data(Db), diff --git a/src/couch/src/couch_debug.erl b/src/couch/src/couch_debug.erl index ff864210f2..2cc0a94b41 100644 --- a/src/couch/src/couch_debug.erl +++ b/src/couch/src/couch_debug.erl @@ -48,7 +48,8 @@ dead_nodes/1, ping/1, ping/2, - ping_nodes/0, + ping_live_cluster_nodes/0, + ping_live_cluster_nodes/1, ping_nodes/1, ping_nodes/2, node_events/0 @@ -57,9 +58,12 @@ -export([ print_table/2, print_report/1, - print_report_with_info_width/2 + print_report_with_info_width/2, + print_tree/2 ]). +-define(PING_TIMEOUT_IN_MS, 60000). + -type throw(_Reason) :: no_return(). -type process_name() :: atom(). @@ -83,18 +87,23 @@ help() -> process_name, get_pid, link_tree, - mapfold, - map, - fold, + mapfold_tree, + fold_tree, + map_tree, linked_processes_info, print_linked_processes, memory_info, + resource_hoggers, + resource_hoggers_snapshot, + analyze_resource_hoggers, print_table, print_report, print_report_with_info_width, + print_tree, restart, restart_busy, dead_nodes, + ping, ping_nodes, node_events ]. @@ -230,7 +239,7 @@ help(link_tree) -> The function doesn't recurse to pids older than initial one. The Pids which are lesser than initial Pid are still shown in the output. The info argument is a list of process_info_item() as documented in - erlang:process_info/2. We don't do any attempts to prevent dangerous items. + process_info/2. We don't do any attempts to prevent dangerous items. Be warn that passing some of them such as `messages` for example can be dangerous in a very busy system. --- @@ -288,7 +297,7 @@ help(linked_processes_info) -> use of link_tree. - Pid: initial Pid to start from - Info: a list of process_info_item() as documented - in erlang:process_info/2. + in process_info/2. --- ", []); @@ -467,18 +476,27 @@ help(ping) -> Ping a node and return either a time in microseconds or an error term. + --- + ", []); +help(ping_live_cluster_nodes) -> + io:format(" + ping_live_cluster_nodes() + ping_live_cluster_nodes(Timeout) + -------------------------------- + + Ping the currently connected cluster nodes. Returns a list of + {Node, Result} tuples or an empty list. + --- ", []); help(ping_nodes) -> io:format(" - ping_nodes() - ping_nodes(Timeout) + ping_nodes(Nodes) ping_nodes(Nodes, Timeout) -------------------------------- - Ping the list of currently connected nodes. Return a list of {Node, - Result} tuples where Result is either a time in microseconds or an - error term. + Ping the list of nodes. Return a list of {Node, Result} tuples where + Result is either a time in microseconds or an error term. --- ", []); @@ -768,6 +786,7 @@ info_size(InfoKV) -> {binary, BinInfos} -> lists:sum([S || {_, S, _} <- BinInfos]); {_, V} -> V end. + resource_hoggers(MemoryInfo, InfoKey) -> KeyFun = fun ({_Pid, _Id, undefined}) -> undefined; @@ -976,12 +995,15 @@ ping(Node) -> ping(Node, Timeout) -> mem3:ping(Node, Timeout). -ping_nodes() -> +ping_live_cluster_nodes() -> mem3:ping_nodes(). -ping_nodes(Timeout) -> +ping_live_cluster_nodes(Timeout) -> mem3:ping_nodes(Timeout). +ping_nodes(Nodes) -> + mem3:ping_nodes(Nodes, ?PING_TIMEOUT_IN_MS). + ping_nodes(Nodes, Timeout) -> mem3:ping_nodes(Nodes, Timeout). @@ -1007,6 +1029,7 @@ print_table(Rows, TableSpec) -> end, Rows ), + io:format("~n", []), ok. print_report(Report) -> @@ -1124,8 +1147,8 @@ random_processes(Acc, Depth) -> end); open_port -> spawn_link(fun() -> - Port = erlang:open_port({spawn, "sleep 10"}, [hide]), - true = erlang:link(Port), + Port = open_port({spawn, "sleep 10"}, [hide]), + true = link(Port), Caller ! {Ref, random_processes(Depth - 1)}, receive looper -> ok diff --git a/src/couch/src/couch_doc.erl b/src/couch/src/couch_doc.erl index 7b867f08d1..7ad0f76b63 100644 --- a/src/couch/src/couch_doc.erl +++ b/src/couch/src/couch_doc.erl @@ -43,7 +43,7 @@ to_branch(Doc, [RevId | Rest]) -> to_json_rev(0, []) -> []; to_json_rev(Start, [FirstRevId | _]) -> - [{<<"_rev">>, ?l2b([integer_to_list(Start), "-", revid_to_str(FirstRevId)])}]. + [{<<"_rev">>, rev_to_str({Start, FirstRevId})}]. to_json_body(true, {Body}) -> Body ++ [{<<"_deleted">>, true}]; @@ -75,11 +75,13 @@ to_json_revisions(Options, Start, RevIds0) -> revid_to_str(RevId) when size(RevId) =:= 16 -> couch_util:to_hex_bin(RevId); -revid_to_str(RevId) -> - RevId. +revid_to_str(RevId) when is_binary(RevId) -> + RevId; +revid_to_str(RevId) when is_list(RevId) -> + list_to_binary(RevId). rev_to_str({Pos, RevId}) -> - ?l2b([integer_to_list(Pos), "-", revid_to_str(RevId)]). + <<(integer_to_binary(Pos))/binary, $-, (revid_to_str(RevId))/binary>>. revs_to_strs([]) -> []; @@ -95,7 +97,7 @@ to_json_meta(Meta) -> JsonObj = {[ {<<"rev">>, rev_to_str({PosAcc, RevId})}, - {<<"status">>, ?l2b(atom_to_list(Status))} + {<<"status">>, atom_to_binary(Status)} ]}, {JsonObj, PosAcc - 1} end, @@ -186,12 +188,10 @@ from_json_obj({Props}, DbName) -> from_json_obj(_Other, _) -> throw({bad_request, "Document must be a JSON object"}). -parse_revid(RevId) when size(RevId) =:= 32 -> - RevInt = erlang:list_to_integer(?b2l(RevId), 16), - <>; -parse_revid(RevId) when length(RevId) =:= 32 -> - RevInt = erlang:list_to_integer(RevId, 16), - <>; +parse_revid(RevId) when is_binary(RevId), size(RevId) =:= 32 -> + binary:decode_hex(RevId); +parse_revid(RevId) when is_list(RevId), length(RevId) =:= 32 -> + binary:decode_hex(list_to_binary(RevId)); parse_revid(RevId) when is_binary(RevId) -> RevId; parse_revid(RevId) when is_list(RevId) -> @@ -388,13 +388,13 @@ max_seq(Tree, UpdateSeq) -> case Value of {_Deleted, _DiskPos, OldTreeSeq} -> % Older versions didn't track data sizes. - erlang:max(MaxOldSeq, OldTreeSeq); + max(MaxOldSeq, OldTreeSeq); % necessary clause? {_Deleted, _DiskPos, OldTreeSeq, _Size} -> % Older versions didn't store #leaf records. - erlang:max(MaxOldSeq, OldTreeSeq); + max(MaxOldSeq, OldTreeSeq); #leaf{seq = OldTreeSeq} -> - erlang:max(MaxOldSeq, OldTreeSeq); + max(MaxOldSeq, OldTreeSeq); _ -> MaxOldSeq end @@ -566,7 +566,7 @@ restart_open_doc_revs(Parser, Ref, NewRef) -> unlink(Parser), exit(Parser, kill), flush_parser_messages(Ref), - erlang:error({restart_open_doc_revs, NewRef}). + error({restart_open_doc_revs, NewRef}). flush_parser_messages(Ref) -> receive diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl index c1a069edd4..2d0920f7e8 100644 --- a/src/couch/src/couch_file.erl +++ b/src/couch/src/couch_file.erl @@ -47,11 +47,11 @@ -export([append_term/2, append_term/3]). -export([pread_terms/2]). -export([append_terms/2, append_terms/3]). --export([write_header/2, read_header/1]). +-export([write_header/2, write_header/3, read_header/1]). -export([delete/2, delete/3, nuke_dir/2, init_delete_dir/1]). % gen_server callbacks --export([init/1, terminate/2, format_status/2]). +-export([init/1, terminate/2]). -export([handle_call/3, handle_cast/2, handle_info/2]). %% helper functions @@ -290,20 +290,20 @@ sync(Filepath) when is_list(Filepath) -> ok -> ok; {error, Reason} -> - erlang:error({fsync_error, Reason}) + error({fsync_error, Reason}) end after ok = file:close(Fd) end; {error, Error} -> - erlang:error(Error) + error(Error) end; sync(Fd) -> case gen_server:call(Fd, sync, infinity) of ok -> ok; {error, Reason} -> - erlang:error({fsync_error, Reason}) + error({fsync_error, Reason}) end. %%---------------------------------------------------------------------- @@ -410,9 +410,7 @@ delete_dir(RootDelDir, Dir) -> init_delete_dir(RootDir) -> Dir = filename:join(RootDir, ".delete"), - % note: ensure_dir requires an actual filename companent, which is the - % reason for "foo". - filelib:ensure_dir(filename:join(Dir, "foo")), + filelib:ensure_path(Dir), spawn(fun() -> filelib:fold_files( Dir, @@ -435,11 +433,16 @@ read_header(Fd) -> end. write_header(Fd, Data) -> + write_header(Fd, Data, []). + +% Only the sync option is currently supported +% +write_header(Fd, Data, Opts) when is_list(Opts) -> Bin = ?term_to_bin(Data), Checksum = generate_checksum(Bin), % now we assemble the final header binary and write to disk FinalBin = <>, - ioq:call(Fd, {write_header, FinalBin}, erlang:get(io_priority)). + ioq:call(Fd, {write_header, FinalBin, Opts}, erlang:get(io_priority)). init_status_error(ReturnPid, Ref, Error) -> ReturnPid ! {Ref, self(), Error}, @@ -582,20 +585,21 @@ handle_call({append_bins, Bins}, _From, #file{} = File) -> {{ok, Resps}, File1} -> {reply, {ok, Resps}, File1}; {Error, File1} -> {reply, Error, File1} end; -handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> - BinSize = byte_size(Bin), - case Pos rem ?SIZE_BLOCK of - 0 -> - Padding = <<>>; - BlockOffset -> - Padding = <<0:(8 * (?SIZE_BLOCK - BlockOffset))>> - end, - FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])], - case file:write(Fd, FinalBin) of - ok -> - {reply, ok, File#file{eof = Pos + iolist_size(FinalBin)}}; - Error -> - {reply, Error, reset_eof(File)} +handle_call({write_header, Bin, Opts}, _From, #file{} = File) -> + try + ok = header_fsync(File, Opts), + case handle_write_header(Bin, File) of + {ok, NewFile} -> + ok = header_fsync(NewFile, Opts), + {reply, ok, NewFile}; + {{error, Err}, NewFile} -> + {reply, {error, Err}, NewFile} + end + catch + error:{fsync_error, Error} -> + % If fsync error happens we stop. See comment in + % handle_call(sync, ...) why we're dropping the fd + {stop, {error, Error}, {error, Error}, #file{fd = nil}} end; handle_call(find_header, _From, #file{fd = Fd, eof = Pos} = File) -> {reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}. @@ -617,10 +621,6 @@ handle_info({'DOWN', Ref, process, _Pid, _Info}, #file{db_monitor = Ref} = File) false -> {noreply, File} end. -format_status(_Opt, [PDict, #file{} = File]) -> - {_Fd, FilePath} = couch_util:get_value(couch_file_fd, PDict), - [{data, [{"State", File}, {"InitialFilePath", FilePath}]}]. - eof(#file{fd = Fd}) -> file:position(Fd, eof). @@ -661,6 +661,17 @@ pread(#file{} = File, PosL) -> Extracted = lists:zipwith(ZipFun, DataSizes, Resps), {ok, Extracted}. +header_fsync(#file{fd = Fd}, Opts) when is_list(Opts) -> + case proplists:get_value(sync, Opts) of + true -> + case fsync(Fd) of + ok -> ok; + {error, Err} -> error({fsync_error, Err}) + end; + _ -> + ok + end. + fsync(Fd) -> T0 = erlang:monotonic_time(), % We do not rely on mtime/atime for our safety/consitency so we can use @@ -759,6 +770,18 @@ find_newest_header(Fd, [{Location, Size} | LocationSizes]) -> find_newest_header(Fd, LocationSizes) end. +handle_write_header(Bin, #file{fd = Fd, eof = Pos} = File) -> + BinSize = byte_size(Bin), + case Pos rem ?SIZE_BLOCK of + 0 -> Padding = <<>>; + BlockOffset -> Padding = <<0:(8 * (?SIZE_BLOCK - BlockOffset))>> + end, + FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])], + case file:write(Fd, FinalBin) of + ok -> {ok, File#file{eof = Pos + iolist_size(FinalBin)}}; + {error, Error} -> {{error, Error}, reset_eof(File)} + end. + read_multi_raw_iolists_int(#file{fd = Fd, eof = Eof} = File, PosLens) -> MapFun = fun({Pos, Len}) -> get_pread_locnum(File, Pos, Len) end, LocNums = lists:map(MapFun, PosLens), diff --git a/src/couch/src/couch_flags_config.erl b/src/couch/src/couch_flags_config.erl index a50f4411f9..4281f2266e 100644 --- a/src/couch/src/couch_flags_config.erl +++ b/src/couch/src/couch_flags_config.erl @@ -134,7 +134,7 @@ parse_flags(_Tokens, _) -> parse_flags_term(FlagsBin) -> {Flags, Errors} = lists:splitwith( - fun erlang:is_atom/1, + fun is_atom/1, [parse_flag(F) || F <- split_by_comma(FlagsBin)] ), case Errors of diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl index 4566157da0..7c6a60d2b9 100644 --- a/src/couch/src/couch_httpd_auth.erl +++ b/src/couch/src/couch_httpd_auth.erl @@ -381,7 +381,7 @@ cookie_authentication_handler(#httpd{mochi_req = MochiReq} = Req, AuthModule) -> "timeout", 600 ), couch_log:debug("timeout ~p", [Timeout]), - case (catch erlang:list_to_integer(TimeStr, 16)) of + case (catch list_to_integer(TimeStr, 16)) of TimeStamp when CurrentTime < TimeStamp + Timeout -> case lists:any(VerifyHash, HashAlgorithms) of true -> @@ -438,7 +438,7 @@ cookie_auth_header(_Req, _Headers) -> []. cookie_auth_cookie(Req, User, Secret, TimeStamp) -> - SessionItems = [User, erlang:integer_to_list(TimeStamp, 16)], + SessionItems = [User, integer_to_list(TimeStamp, 16)], cookie_auth_cookie(Req, Secret, SessionItems). cookie_auth_cookie(Req, Secret, SessionItems) when is_list(SessionItems) -> diff --git a/src/couch/src/couch_httpd_db.erl b/src/couch/src/couch_httpd_db.erl index 5c566ed789..733991d842 100644 --- a/src/couch/src/couch_httpd_db.erl +++ b/src/couch/src/couch_httpd_db.erl @@ -817,7 +817,7 @@ receive_request_data(Req) -> receive_request_data(Req, couch_httpd:body_length(Req)). receive_request_data(Req, LenLeft) when LenLeft > 0 -> - Len = erlang:min(4096, LenLeft), + Len = min(4096, LenLeft), Data = couch_httpd:recv(Req, Len), {Data, fun() -> receive_request_data(Req, LenLeft - iolist_size(Data)) end}; receive_request_data(_Req, _) -> diff --git a/src/couch/src/couch_httpd_multipart.erl b/src/couch/src/couch_httpd_multipart.erl index 80fc48a75f..87c588fc8d 100644 --- a/src/couch/src/couch_httpd_multipart.erl +++ b/src/couch/src/couch_httpd_multipart.erl @@ -27,7 +27,7 @@ decode_multipart_stream(ContentType, DataFun, Ref) -> Parent = self(), NumMpWriters = num_mp_writers(), {Parser, ParserRef} = spawn_monitor(fun() -> - ParentRef = erlang:monitor(process, Parent), + ParentRef = monitor(process, Parent), put(mp_parent_ref, ParentRef), num_mp_writers(NumMpWriters), {<<"--", _/binary>>, _, _} = couch_httpd:parse_multipart_request( @@ -214,7 +214,7 @@ maybe_send_data({Ref, Chunks, Offset, Counters, Waiting}) -> end. handle_hello(WriterPid, Counters) -> - WriterRef = erlang:monitor(process, WriterPid), + WriterRef = monitor(process, WriterPid), orddict:store(WriterPid, {WriterRef, 0}, Counters). update_writer(WriterPid, Counters) -> @@ -222,7 +222,7 @@ update_writer(WriterPid, Counters) -> {ok, {WriterRef, Count}} -> orddict:store(WriterPid, {WriterRef, Count + 1}, Counters); error -> - WriterRef = erlang:monitor(process, WriterPid), + WriterRef = monitor(process, WriterPid), orddict:store(WriterPid, {WriterRef, 1}, Counters) end. @@ -274,7 +274,7 @@ atts_to_mp( WriteFun, AttFun ) -> - LengthBin = list_to_binary(integer_to_list(Len)), + LengthBin = integer_to_binary(Len), % write headers WriteFun(<<"\r\nContent-Disposition: attachment; filename=\"", Name/binary, "\"">>), WriteFun(<<"\r\nContent-Type: ", Type/binary>>), @@ -344,7 +344,7 @@ length_multipart_stream(Boundary, JsonBytes, Atts) -> end. abort_multipart_stream(Parser) -> - MonRef = erlang:monitor(process, Parser), + MonRef = monitor(process, Parser), Parser ! abort_parsing, receive {'DOWN', MonRef, _, _, _} -> ok diff --git a/src/couch/src/couch_httpd_vhost.erl b/src/couch/src/couch_httpd_vhost.erl index 170e1a2f74..91da711afe 100644 --- a/src/couch/src/couch_httpd_vhost.erl +++ b/src/couch/src/couch_httpd_vhost.erl @@ -329,7 +329,7 @@ split_host_port(HostAsString) -> N -> HostPart = string:substr(HostAsString, 1, N - 1), case - (catch erlang:list_to_integer( + (catch list_to_integer( string:substr( HostAsString, N + 1, diff --git a/src/couch/src/couch_key_tree.erl b/src/couch/src/couch_key_tree.erl index 87504c78c7..6a3dffc040 100644 --- a/src/couch/src/couch_key_tree.erl +++ b/src/couch/src/couch_key_tree.erl @@ -113,7 +113,7 @@ merge_tree([{Depth, Nodes} | Rest], {IDepth, INodes} = Tree, MergeAcc) -> % value that's used throughout this module. case merge_at([Nodes], Depth - IDepth, [INodes]) of {[Merged], Result} -> - NewDepth = erlang:min(Depth, IDepth), + NewDepth = min(Depth, IDepth), {Rest ++ [{NewDepth, Merged} | MergeAcc], Result}; fail -> merge_tree(Rest, Tree, [{Depth, Nodes} | MergeAcc]) @@ -507,12 +507,12 @@ stem_tree(Depth, {Key, Val, Children}, Limit, Seen0) -> {SeenAcc, LimitPosAcc, ChildAcc, BranchAcc} = Acc, case stem_tree(Depth + 1, Child, Limit, SeenAcc) of {NewSeenAcc, LimitPos, NewChild, NewBranches} -> - NewLimitPosAcc = erlang:max(LimitPos, LimitPosAcc), + NewLimitPosAcc = max(LimitPos, LimitPosAcc), NewChildAcc = [NewChild | ChildAcc], NewBranchAcc = NewBranches ++ BranchAcc, {NewSeenAcc, NewLimitPosAcc, NewChildAcc, NewBranchAcc}; {NewSeenAcc, LimitPos, NewBranches} -> - NewLimitPosAcc = erlang:max(LimitPos, LimitPosAcc), + NewLimitPosAcc = max(LimitPos, LimitPosAcc), NewBranchAcc = NewBranches ++ BranchAcc, {NewSeenAcc, NewLimitPosAcc, ChildAcc, NewBranchAcc} end diff --git a/src/couch/src/couch_multidb_changes.erl b/src/couch/src/couch_multidb_changes.erl index 2ed200314d..bb8a3fe47c 100644 --- a/src/couch/src/couch_multidb_changes.erl +++ b/src/couch/src/couch_multidb_changes.erl @@ -241,7 +241,7 @@ should_wait_for_shard_map(<<_/binary>>) -> -spec register_with_event_server(pid()) -> reference(). register_with_event_server(Server) -> - Ref = erlang:monitor(process, couch_event_server), + Ref = monitor(process, couch_event_server), couch_event:register_all(Server), Ref. @@ -475,7 +475,7 @@ setup_all() -> meck:expect(couch_db, close, 1, ok), mock_changes_reader(), % create process to stand in for couch_event_server - % mocking erlang:monitor doesn't work, so give it real process to monitor + % mocking monitor doesn't work, so give it real process to monitor EvtPid = spawn_link(fun() -> receive looper -> ok @@ -709,7 +709,7 @@ t_handle_info_change_feed_exited_and_need_rescan(_) -> t_spawn_changes_reader(_) -> Pid = start_changes_reader(?DBNAME, 3), - ?assert(erlang:is_process_alive(Pid)), + ?assert(is_process_alive(Pid)), ChArgs = kill_mock_changes_reader_and_get_its_args(Pid), ?assertEqual({self(), ?DBNAME}, ChArgs), ?assert(meck:validate(couch_db)), @@ -1135,7 +1135,7 @@ kill_mock_changes_reader_and_get_its_args(Pid) -> {'DOWN', Ref, _, Pid, {Server, DbName}} -> {Server, DbName} after 1000 -> - erlang:error(spawn_change_reader_timeout) + error(spawn_change_reader_timeout) end. mock_changes_reader() -> diff --git a/src/couch/src/couch_native_process.erl b/src/couch/src/couch_native_process.erl index c9280f1d76..9d555c8a42 100644 --- a/src/couch/src/couch_native_process.erl +++ b/src/couch/src/couch_native_process.erl @@ -111,7 +111,7 @@ handle_call({prompt, Data}, _From, State) -> end. handle_cast(garbage_collect, State) -> - erlang:garbage_collect(), + garbage_collect(), {noreply, State, State#evstate.idle}; handle_cast(stop, State) -> {stop, normal, State}; @@ -120,7 +120,7 @@ handle_cast(_Msg, State) -> handle_info(timeout, State) -> couch_proc_manager:os_proc_idle(self()), - erlang:garbage_collect(), + garbage_collect(), {noreply, State, State#evstate.idle}; handle_info({'EXIT', _, normal}, State) -> {noreply, State, State#evstate.idle}; @@ -478,6 +478,6 @@ to_binary(true) -> to_binary(false) -> false; to_binary(Data) when is_atom(Data) -> - list_to_binary(atom_to_list(Data)); + atom_to_binary(Data); to_binary(Data) -> Data. diff --git a/src/couch/src/couch_os_process.erl b/src/couch/src/couch_os_process.erl index 59ceeca13a..339c93f72f 100644 --- a/src/couch/src/couch_os_process.erl +++ b/src/couch/src/couch_os_process.erl @@ -152,7 +152,7 @@ init([Command]) -> couch_stats:increment_counter([couchdb, query_server, process_starts]), spawn(fun() -> % this ensure the real os process is killed when this process dies. - erlang:monitor(process, Pid), + monitor(process, Pid), killer(OsPid) end), {ok, OsProc, IdleLimit}. @@ -191,7 +191,7 @@ handle_call({prompt, Data}, _From, #os_proc{idle = Idle} = OsProc) -> end. handle_cast(garbage_collect, #os_proc{idle = Idle} = OsProc) -> - erlang:garbage_collect(), + garbage_collect(), {noreply, OsProc, Idle}; handle_cast(stop, OsProc) -> {stop, normal, OsProc}; @@ -201,7 +201,7 @@ handle_cast(Msg, #os_proc{idle = Idle} = OsProc) -> handle_info(timeout, #os_proc{idle = Idle} = OsProc) -> couch_proc_manager:os_proc_idle(self()), - erlang:garbage_collect(), + garbage_collect(), {noreply, OsProc, Idle}; handle_info({Port, {exit_status, 0}}, #os_proc{port = Port} = OsProc) -> couch_log:info("OS Process terminated normally", []), diff --git a/src/couch/src/couch_primary_sup.erl b/src/couch/src/couch_primary_sup.erl index 32ee282d68..93a1d2112f 100644 --- a/src/couch/src/couch_primary_sup.erl +++ b/src/couch/src/couch_primary_sup.erl @@ -18,6 +18,7 @@ start_link() -> supervisor:start_link({local, couch_primary_services}, ?MODULE, []). init([]) -> + ok = couch_bt_engine_cache:create_tables(), Children = [ {couch_task_status, {couch_task_status, start_link, []}, permanent, brutal_kill, worker, @@ -45,7 +46,7 @@ init([]) -> ] ]}, permanent, 5000, worker, [ets_lru]} - ] ++ couch_servers(), + ] ++ couch_bt_engine_cache:sup_children() ++ couch_servers(), {ok, {{one_for_one, 10, 3600}, Children}}. couch_servers() -> diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl index d90463bf32..aa538e23e7 100644 --- a/src/couch/src/couch_proc_manager.erl +++ b/src/couch/src/couch_proc_manager.erl @@ -162,7 +162,7 @@ handle_call({get_proc, #client{} = Client}, From, State) -> {noreply, State}; handle_call({ret_proc, #proc{} = Proc}, From, State) -> #proc{client = Ref, pid = Pid} = Proc, - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), gen_server:reply(From, true), case ets:lookup(?PROCS, Pid) of [#proc{} = ProcInt] -> @@ -615,7 +615,7 @@ make_proc(Pid, Lang, Mod) when is_binary(Lang) -> {ok, Proc}. assign_proc(Pid, #proc{client = undefined} = Proc0) when is_pid(Pid) -> - Proc = Proc0#proc{client = erlang:monitor(process, Pid)}, + Proc = Proc0#proc{client = monitor(process, Pid)}, % It's important to insert the proc here instead of doing an update_element % as we might have updated the db_key or ddoc_keys in teach_ddoc/4 ets:insert(?PROCS, Proc), @@ -730,13 +730,25 @@ remove_waiting_client(#client{wait_key = Key}) -> ets:delete(?WAITERS, Key). get_proc_config() -> - Limit = config:get_boolean("query_server_config", "reduce_limit", true), - Timeout = get_os_process_timeout(), {[ - {<<"reduce_limit">>, Limit}, - {<<"timeout">>, Timeout} + {<<"reduce_limit">>, get_reduce_limit()}, + {<<"reduce_limit_threshold">>, couch_query_servers:reduce_limit_threshold()}, + {<<"reduce_limit_ratio">>, couch_query_servers:reduce_limit_ratio()}, + {<<"timeout">>, get_os_process_timeout()} ]}. +% Reduce limit is a tri-state value of true, false or log. The default value if +% is true. That's also the value if anything other than those 3 values are +% specified. +% +get_reduce_limit() -> + case config:get("query_server_config", "reduce_limit", "true") of + "false" -> false; + "log" -> log; + "true" -> true; + Other when is_list(Other) -> true + end. + get_hard_limit() -> config:get_integer("query_server_config", "os_process_limit", 100). diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl index 3b222e0810..1436501221 100644 --- a/src/couch/src/couch_query_servers.erl +++ b/src/couch/src/couch_query_servers.erl @@ -19,6 +19,7 @@ -export([filter_view/4]). -export([finalize/2]). -export([rewrite/3]). +-export([reduce_limit_threshold/0, reduce_limit_ratio/0]). -export([with_ddoc_proc/3, proc_prompt/2, ddoc_prompt/4, ddoc_proc_prompt/3, json_doc/1]). @@ -278,7 +279,7 @@ sum_arrays(Else, _) -> throw_sum_error(Else). check_sum_overflow(InSize, OutSize, Sum) -> - Overflowed = OutSize > 4906 andalso OutSize * 2 > InSize, + Overflowed = OutSize > reduce_limit_threshold() andalso OutSize * reduce_limit_ratio() > InSize, case config:get("query_server_config", "reduce_limit", "true") of "true" when Overflowed -> Msg = log_sum_overflow(InSize, OutSize), @@ -302,6 +303,12 @@ log_sum_overflow(InSize, OutSize) -> couch_log:error(Msg, []), Msg. +reduce_limit_threshold() -> + config:get_integer("query_server_config", "reduce_limit_threshold", 5000). + +reduce_limit_ratio() -> + config:get_float("query_server_config", "reduce_limit_ratio", 2.0). + builtin_stats(_, []) -> {0, 0, 0, 0, 0}; builtin_stats(_, [[_, First] | Rest]) -> @@ -327,8 +334,8 @@ stat_values(Value, Acc) when is_tuple(Value), is_tuple(Acc) -> { Sum0 + Sum1, Cnt0 + Cnt1, - erlang:min(Min0, Min1), - erlang:max(Max0, Max1), + min(Min0, Min1), + max(Max0, Max1), Sqr0 + Sqr1 }; stat_values(Else, _Acc) -> @@ -461,18 +468,18 @@ builtin_cmp_last(A, B) -> validate_doc_update(Db, DDoc, EditDoc, DiskDoc, Ctx, SecObj) -> JsonEditDoc = couch_doc:to_json_obj(EditDoc, [revs]), JsonDiskDoc = json_doc(DiskDoc), - Resp = ddoc_prompt( - Db, - DDoc, - [<<"validate_doc_update">>], - [JsonEditDoc, JsonDiskDoc, Ctx, SecObj] - ), - if - Resp == 1 -> ok; - true -> couch_stats:increment_counter([couchdb, query_server, vdu_rejects], 1) - end, + Args = [JsonEditDoc, JsonDiskDoc, Ctx, SecObj], + + Resp = + case ddoc_prompt(Db, DDoc, [<<"validate_doc_update">>], Args) of + Code when Code =:= 1; Code =:= ok; Code =:= true -> + ok; + Other -> + couch_stats:increment_counter([couchdb, query_server, vdu_rejects], 1), + Other + end, case Resp of - RespCode when RespCode =:= 1; RespCode =:= ok; RespCode =:= true -> + ok -> ok; {[{<<"forbidden">>, Message}]} -> throw({forbidden, Message}); diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index ca12a56fa9..246cdd1cba 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -303,7 +303,12 @@ init([N]) -> "couchdb", "update_lru_on_read", false ), ok = config:listen_for_changes(?MODULE, N), - ok = couch_file:init_delete_dir(RootDir), + % Spawn async .deleted files recursive cleaner, but only + % for the first sharded couch_server instance + case N of + 1 -> ok = couch_file:init_delete_dir(RootDir); + _ -> ok + end, ets:new(couch_dbs(N), [ set, protected, @@ -400,7 +405,7 @@ handle_config_terminate(_Server, _Reason, N) -> erlang:send_after(?RELISTEN_DELAY, whereis(?MODULE), {restart_config_listener, N}). per_couch_server(X) -> - erlang:max(1, X div num_servers()). + max(1, X div num_servers()). all_databases() -> {ok, DbList} = all_databases( @@ -858,7 +863,7 @@ get_engine(Server, DbName) -> [Engine] -> {ok, Engine}; _ -> - erlang:error(engine_conflict) + error(engine_conflict) end. get_possible_engines(DbName, RootDir, Engines) -> diff --git a/src/couch/src/couch_stream.erl b/src/couch/src/couch_stream.erl index ea375101fb..2998cd3ffa 100644 --- a/src/couch/src/couch_stream.erl +++ b/src/couch/src/couch_stream.erl @@ -185,7 +185,7 @@ init({Engine, OpenerPid, OpenerPriority, Options}) -> end, {ok, #stream{ engine = Engine, - opener_monitor = erlang:monitor(process, OpenerPid), + opener_monitor = monitor(process, OpenerPid), md5 = couch_hash:md5_hash_init(), identity_md5 = couch_hash:md5_hash_init(), encoding_fun = EncodingFun, @@ -269,7 +269,7 @@ handle_call(close, _From, Stream) -> StreamLen = WrittenLen + iolist_size(WriteBin2), {do_finalize(NewEngine), StreamLen, IdenLen, Md5Final, IdenMd5Final} end, - erlang:demonitor(MonRef), + demonitor(MonRef), {stop, normal, Result, Stream}. handle_cast(_Msg, State) -> diff --git a/src/couch/src/couch_task_status.erl b/src/couch/src/couch_task_status.erl index 29ab73692d..8577c304d5 100644 --- a/src/couch/src/couch_task_status.erl +++ b/src/couch/src/couch_task_status.erl @@ -107,7 +107,7 @@ handle_call({add_task, TaskProps}, {From, _}, Server) -> case ets:lookup(?MODULE, From) of [] -> true = ets:insert(?MODULE, {From, TaskProps}), - erlang:monitor(process, From), + monitor(process, From), {reply, ok, Server}; [_] -> {reply, {add_task_error, already_registered}, Server} @@ -130,7 +130,7 @@ handle_cast({update_status, Pid, NewProps}, Server) -> {noreply, Server}. handle_info({'DOWN', _MonitorRef, _Type, Pid, _Info}, Server) -> - %% should we also erlang:demonitor(_MonitorRef), ? + %% should we also demonitor(_MonitorRef), ? ets:delete(?MODULE, Pid), {noreply, Server}. diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index bdd5c14f3b..e92caab93a 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -137,7 +137,7 @@ to_existing_atom(V) when is_list(V) -> end; to_existing_atom(V) when is_binary(V) -> try - list_to_existing_atom(?b2l(V)) + binary_to_existing_atom(V) catch _:_ -> V end; @@ -147,7 +147,7 @@ to_existing_atom(V) when is_atom(V) -> shutdown_sync(Pid) when not is_pid(Pid) -> ok; shutdown_sync(Pid) -> - MRef = erlang:monitor(process, Pid), + MRef = monitor(process, Pid), try catch unlink(Pid), catch exit(Pid, shutdown), @@ -156,7 +156,7 @@ shutdown_sync(Pid) -> ok end after - erlang:demonitor(MRef, [flush]) + demonitor(MRef, [flush]) end. validate_utf8(Data) when is_list(Data) -> @@ -208,31 +208,8 @@ to_hex(Binary) when is_binary(Binary) -> to_hex(List) when is_list(List) -> binary_to_list(to_hex_bin(list_to_binary(List))). -% Optimized encode_hex/1 function from Erlang/OTP binary module starting with OTP 24+ [1]. -% One exception is we are emitting lower case hex characters instead of upper case ones. -% -% [1] https://github.com/erlang/otp/blob/master/lib/stdlib/src/binary.erl#L365. -% - --define(HEX(X), (hex(X)):16). - -%% erlfmt-ignore -to_hex_bin(Data) when byte_size(Data) rem 8 =:= 0 -> - << <> || <> <= Data >>; -to_hex_bin(Data) when byte_size(Data) rem 7 =:= 0 -> - << <> || <> <= Data >>; -to_hex_bin(Data) when byte_size(Data) rem 6 =:= 0 -> - << <> || <> <= Data >>; -to_hex_bin(Data) when byte_size(Data) rem 5 =:= 0 -> - << <> || <> <= Data >>; -to_hex_bin(Data) when byte_size(Data) rem 4 =:= 0 -> - << <> || <> <= Data >>; -to_hex_bin(Data) when byte_size(Data) rem 3 =:= 0 -> - << <> || <> <= Data >>; -to_hex_bin(Data) when byte_size(Data) rem 2 =:= 0 -> - << <> || <> <= Data >>; to_hex_bin(Data) when is_binary(Data) -> - << <> || <> <= Data >>. + binary:encode_hex(Data, lowercase). parse_term(Bin) when is_binary(Bin) -> parse_term(binary_to_list(Bin)); @@ -251,8 +228,14 @@ get_value(Key, List, Default) -> Default end. -set_value(Key, List, Value) -> - lists:keyreplace(Key, 1, List, {Key, Value}). +% insert or update a {Key, Value} tuple in a list of tuples +-spec set_value(Key, TupleList1, Value) -> TupleList2 when + Key :: term(), + TupleList1 :: [tuple()], + Value :: term(), + TupleList2 :: [tuple()]. +set_value(Key, TupleList1, Value) -> + lists:keystore(Key, 1, TupleList1, {Key, Value}). get_nested_json_value({Props}, [Key | Keys]) -> case couch_util:get_value(Key, Props, nil) of @@ -446,16 +429,16 @@ to_binary(V) when is_list(V) -> list_to_binary(io_lib:format("~p", [V])) end; to_binary(V) when is_atom(V) -> - list_to_binary(atom_to_list(V)); + atom_to_binary(V); to_binary(V) -> list_to_binary(io_lib:format("~p", [V])). to_integer(V) when is_integer(V) -> V; to_integer(V) when is_list(V) -> - erlang:list_to_integer(V); + list_to_integer(V); to_integer(V) when is_binary(V) -> - erlang:list_to_integer(binary_to_list(V)). + binary_to_integer(V). to_list(V) when is_list(V) -> V; @@ -528,13 +511,15 @@ reorder_results(Keys, SortedResults, Default) -> Map = maps:from_list(SortedResults), [maps:get(Key, Map, Default) || Key <- Keys]. -url_strip_password(Url) -> +url_strip_password(Url) when is_list(Url) -> re:replace( Url, "(http|https|socks5)://([^:]+):[^@]+@(.*)$", "\\1://\\2:*****@\\3", [{return, list}] - ). + ); +url_strip_password(Other) -> + Other. encode_doc_id(#doc{id = Id}) -> encode_doc_id(Id); @@ -568,7 +553,7 @@ with_db(Db, Fun) -> true -> Fun(Db); false -> - erlang:error({invalid_db, Db}) + error({invalid_db, Db}) end. rfc1123_date() -> @@ -645,7 +630,7 @@ find_in_binary(_B, <<>>) -> find_in_binary(B, Data) -> case binary:match(Data, [B], []) of nomatch -> - MatchLength = erlang:min(byte_size(B), byte_size(Data)), + MatchLength = min(byte_size(B), byte_size(Data)), match_prefix_at_end( binary:part(B, {0, MatchLength}), binary:part(Data, {byte_size(Data), -MatchLength}), @@ -739,7 +724,7 @@ ensure_loaded(_Module) -> %% a function that does a receive as it would hijack incoming messages. with_proc(M, F, A, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> - exit({reply, erlang:apply(M, F, A)}) + exit({reply, apply(M, F, A)}) end), receive {'DOWN', Ref, process, Pid, {reply, Resp}} -> @@ -747,7 +732,7 @@ with_proc(M, F, A, Timeout) -> {'DOWN', Ref, process, Pid, Error} -> {error, Error} after Timeout -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), {error, timeout} end. @@ -787,39 +772,9 @@ version_to_binary(Ver) when is_tuple(Ver) -> version_to_binary(Ver) when is_list(Ver) -> IsZero = fun(N) -> N == 0 end, Ver1 = lists:reverse(lists:dropwhile(IsZero, lists:reverse(Ver))), - Ver2 = [erlang:integer_to_list(N) || N <- Ver1], + Ver2 = [integer_to_list(N) || N <- Ver1], ?l2b(lists:join(".", Ver2)). --compile({inline, [hex/1]}). - -%% erlfmt-ignore -hex(X) -> - % We are encoding hex directly as lower case ASCII here and it's just a lookup table - % for all the 00..ff combinations. 0x00 -> "00", 0xf1-> "f1", etc. - % - % 00, ..., 0f, - % .. .. - % f0, ..., ff - % - element(X + 1, { - 16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3061, 16#3062, 16#3063, 16#3064, 16#3065, 16#3066, - 16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3161, 16#3162, 16#3163, 16#3164, 16#3165, 16#3166, - 16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3261, 16#3262, 16#3263, 16#3264, 16#3265, 16#3266, - 16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3361, 16#3362, 16#3363, 16#3364, 16#3365, 16#3366, - 16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3461, 16#3462, 16#3463, 16#3464, 16#3465, 16#3466, - 16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3561, 16#3562, 16#3563, 16#3564, 16#3565, 16#3566, - 16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3661, 16#3662, 16#3663, 16#3664, 16#3665, 16#3666, - 16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3761, 16#3762, 16#3763, 16#3764, 16#3765, 16#3766, - 16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3861, 16#3862, 16#3863, 16#3864, 16#3865, 16#3866, - 16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3961, 16#3962, 16#3963, 16#3964, 16#3965, 16#3966, - 16#6130, 16#6131, 16#6132, 16#6133, 16#6134, 16#6135, 16#6136, 16#6137, 16#6138, 16#6139, 16#6161, 16#6162, 16#6163, 16#6164, 16#6165, 16#6166, - 16#6230, 16#6231, 16#6232, 16#6233, 16#6234, 16#6235, 16#6236, 16#6237, 16#6238, 16#6239, 16#6261, 16#6262, 16#6263, 16#6264, 16#6265, 16#6266, - 16#6330, 16#6331, 16#6332, 16#6333, 16#6334, 16#6335, 16#6336, 16#6337, 16#6338, 16#6339, 16#6361, 16#6362, 16#6363, 16#6364, 16#6365, 16#6366, - 16#6430, 16#6431, 16#6432, 16#6433, 16#6434, 16#6435, 16#6436, 16#6437, 16#6438, 16#6439, 16#6461, 16#6462, 16#6463, 16#6464, 16#6465, 16#6466, - 16#6530, 16#6531, 16#6532, 16#6533, 16#6534, 16#6535, 16#6536, 16#6537, 16#6538, 16#6539, 16#6561, 16#6562, 16#6563, 16#6564, 16#6565, 16#6566, - 16#6630, 16#6631, 16#6632, 16#6633, 16#6634, 16#6635, 16#6636, 16#6637, 16#6638, 16#6639, 16#6661, 16#6662, 16#6663, 16#6664, 16#6665, 16#6666 - }). - verify_hash_names(HashAlgorithms, SupportedHashes) -> verify_hash_names(HashAlgorithms, SupportedHashes, []). verify_hash_names([], _, HashNames) -> @@ -856,11 +811,11 @@ remove_sensitive_data(KVList) -> lists:keyreplace(password, 1, KVList1, {password, <<"****">>}). ejson_to_map(#{} = Val) -> - maps:map(fun(_, V) -> ejson_to_map(V) end, Val); + #{K => ejson_to_map(V) || K := V <- Val}; ejson_to_map(Val) when is_list(Val) -> - lists:map(fun(V) -> ejson_to_map(V) end, Val); + [ejson_to_map(V) || V <- Val]; ejson_to_map({Val}) when is_list(Val) -> - maps:from_list(lists:map(fun({K, V}) -> {K, ejson_to_map(V)} end, Val)); + #{K => ejson_to_map(V) || {K, V} <- Val}; ejson_to_map(Val) -> Val. diff --git a/src/couch/src/couch_uuids.erl b/src/couch/src/couch_uuids.erl index 588136159c..f007eab91c 100644 --- a/src/couch/src/couch_uuids.erl +++ b/src/couch/src/couch_uuids.erl @@ -17,7 +17,7 @@ -export([start/0, stop/0]). -export([new/0, random/0]). - +-export([v7_hex/0, v7_bin/0]). -export([init/1]). -export([handle_call/3, handle_cast/2, handle_info/2]). @@ -44,6 +44,8 @@ init([]) -> handle_call(create, _From, random) -> {reply, random(), random}; +handle_call(create, _From, uuid_v7) -> + {reply, v7_hex(), uuid_v7}; handle_call(create, _From, {utc_random, ClockSeq}) -> {UtcRandom, NewClockSeq} = utc_random(ClockSeq), {reply, UtcRandom, {utc_random, NewClockSeq}}; @@ -84,6 +86,32 @@ handle_config_terminate(_Server, _Reason, _State) -> gen_server:cast(?MODULE, change), erlang:send_after(?RELISTEN_DELAY, whereis(?MODULE), restart_config_listener). +%% UUID Version 7 +%% https://datatracker.ietf.org/doc/html/rfc9562#name-uuid-version-7 +%% +%% 0 1 2 3 +%% 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +%% +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +%% | unix_ts_ms | +%% +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +%% | unix_ts_ms | ver | rand_a | +%% +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +%% |var| rand_b | +%% +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +%% | rand_b | +%% +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +%% +%% ver = 0111 = 7 +%% var = 10 = 2 +%% +v7_bin() -> + MSec = os:system_time(millisecond), + <> = crypto:strong_rand_bytes(10), + <>. + +v7_hex() -> + couch_util:to_hex_bin(v7_bin()). + new_prefix() -> couch_util:to_hex((crypto:strong_rand_bytes(13))). @@ -104,6 +132,8 @@ state() -> {utc_id, UtcIdSuffix, ClockSeq}; sequential -> {sequential, new_prefix(), inc()}; + uuid_v7 -> + uuid_v7; Unknown -> throw({unknown_uuid_algorithm, Unknown}) end. diff --git a/src/couch/src/test_util.erl b/src/couch/src/test_util.erl index 430c54788a..b8001840fb 100644 --- a/src/couch/src/test_util.erl +++ b/src/couch/src/test_util.erl @@ -155,7 +155,7 @@ stop_sync(Name, Reason, Timeout) when is_atom(Name) -> stop_sync(Pid, Reason, Timeout) when is_atom(Reason) and is_pid(Pid) -> stop_sync(Pid, fun() -> exit(Pid, Reason) end, Timeout); stop_sync(Pid, Fun, Timeout) when is_function(Fun) and is_pid(Pid) -> - MRef = erlang:monitor(process, Pid), + MRef = monitor(process, Pid), try begin catch unlink(Pid), @@ -168,7 +168,7 @@ stop_sync(Pid, Fun, Timeout) when is_function(Fun) and is_pid(Pid) -> end end after - erlang:demonitor(MRef, [flush]) + demonitor(MRef, [flush]) end; stop_sync(_, _, _) -> error(badarg). @@ -380,7 +380,7 @@ load_applications_with_stats() -> stats_file_to_app(File) -> [_Desc, _Priv, App | _] = lists:reverse(filename:split(File)), - erlang:list_to_atom(App). + list_to_atom(App). calculate_start_order(Apps) -> AllApps = calculate_start_order(sort_apps(Apps), []), diff --git a/src/couch/test/eunit/couch_bt_engine_cache_test.erl b/src/couch/test/eunit/couch_bt_engine_cache_test.erl new file mode 100644 index 0000000000..0eade12a41 --- /dev/null +++ b/src/couch/test/eunit/couch_bt_engine_cache_test.erl @@ -0,0 +1,102 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_bt_engine_cache_test). + +-include_lib("couch/include/couch_eunit.hrl"). + +couch_bt_engine_cache_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_created), + ?TDEF_FE(t_insert_and_lookup), + ?TDEF_FE(t_decay_and_removal_works, 10), + ?TDEF_FE(t_pid_cleanup_works, 10) + ] + }. + +setup() -> + Ctx = test_util:start_applications([config]), + couch_bt_engine_cache:create_tables(), + #{shard_count := N} = couch_bt_engine_cache:info(), + Pids = lists:map( + fun(I) -> + {ok, Pid} = couch_bt_engine_cache:start_link(I), + Pid + end, + lists:seq(1, N) + ), + {Ctx, Pids}. + +teardown({Ctx, [_ | _] = Pids}) -> + lists:foreach( + fun(Pid) -> + catch unlink(Pid), + exit(Pid, kill) + end, + Pids + ), + clear_tables(), + config:delete("bt_engine_cache", "max_size", false), + config:delete("bt_engine_cache", "leave_percent", false), + test_util:stop_applications(Ctx). + +clear_tables() -> + lists:foreach(fun ets:delete/1, couch_bt_engine_cache:tables()). + +t_created(_) -> + Info = couch_bt_engine_cache:info(), + #{size := Size, memory := Mem, shard_count := N} = Info, + ?assert(N >= 16, "shard count is greater or equal to 16"), + ?assertEqual(0, Size), + ?assert(is_integer(Mem)), + ?assert(Mem >= 0). + +t_insert_and_lookup(_) -> + ?assertError(function_clause, couch_bt_engine_cache:insert(not_a_pid, 1, foo)), + ?assertError(function_clause, couch_bt_engine_cache:insert(self(), xyz, foo)), + ?assertMatch(#{size := 0}, couch_bt_engine_cache:info()), + ?assert(couch_bt_engine_cache:insert({pid, 42}, term)), + ?assertMatch(#{size := 1}, couch_bt_engine_cache:info()), + ?assertNot(couch_bt_engine_cache:insert({pid, 42}, term)), + ?assertEqual(term, couch_bt_engine_cache:lookup({pid, 42})), + ?assertEqual(undefined, couch_bt_engine_cache:lookup({pid, 43})). + +t_decay_and_removal_works(_) -> + config:set("bt_engine_cache", "leave_percent", "0", false), + Term = [foo, bar, baz, lists:seq(1, 100)], + [couch_bt_engine_cache:insert({pid, I}, Term) || I <- lists:seq(1, 10000)], + WaitFun = fun() -> + #{size := Size} = couch_bt_engine_cache:info(), + case Size > 0 of + true -> wait; + false -> ok + end + end, + test_util:wait(WaitFun, 7500), + ?assertMatch(#{size := 0}, couch_bt_engine_cache:info()). + +t_pid_cleanup_works(_) -> + Pid = spawn(fun() -> timer:sleep(2000) end), + [couch_bt_engine_cache:insert({Pid, I}, baz) || I <- lists:seq(1, 1000)], + WaitFun = fun() -> + #{size := Size} = couch_bt_engine_cache:info(), + case Size > 0 of + true -> wait; + false -> ok + end + end, + test_util:wait(WaitFun, 7500), + ?assertMatch(#{size := 0}, couch_bt_engine_cache:info()). diff --git a/src/couch/test/eunit/couch_bt_engine_compactor_ev_tests.erl b/src/couch/test/eunit/couch_bt_engine_compactor_ev_tests.erl index 007c74d06e..803d6295f2 100644 --- a/src/couch/test/eunit/couch_bt_engine_compactor_ev_tests.erl +++ b/src/couch/test/eunit/couch_bt_engine_compactor_ev_tests.erl @@ -264,7 +264,7 @@ run_successful_compaction(DbName) -> {ok, ContinueFun} = ?EV_MOD:set_wait(init), {ok, Db} = couch_db:open_int(DbName, []), {ok, CPid} = couch_db:start_compact(Db), - Ref = erlang:monitor(process, CPid), + Ref = monitor(process, CPid), ContinueFun(CPid), receive {'DOWN', Ref, _, _, normal} -> ok @@ -278,7 +278,7 @@ wait_db_cleared(Db) -> wait_db_cleared(Db, 5). wait_db_cleared(Db, N) when N < 0 -> - erlang:error({db_clear_timeout, couch_db:name(Db)}); + error({db_clear_timeout, couch_db:name(Db)}); wait_db_cleared(Db, N) -> Tab = couch_server:couch_dbs(couch_db:name(Db)), case ets:lookup(Tab, couch_db:name(Db)) of @@ -299,7 +299,7 @@ wait_db_cleared(Db, N) -> populate_db(_Db, NumDocs) when NumDocs =< 0 -> ok; populate_db(Db, NumDocs) -> - String = [$a || _ <- lists:seq(1, erlang:min(NumDocs, 500))], + String = [$a || _ <- lists:seq(1, min(NumDocs, 500))], Docs = lists:map( fun(_) -> couch_doc:from_json_obj( diff --git a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl index 4ed57668ef..a7fec17db2 100644 --- a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl +++ b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl @@ -89,7 +89,7 @@ check_db_validity(DbName) -> with_mecked_emsort(Fun) -> meck:new(couch_emsort, [passthrough]), - meck:expect(couch_emsort, iter, fun(_) -> erlang:error(kaboom) end), + meck:expect(couch_emsort, iter, fun(_) -> error(kaboom) end), try Fun() after @@ -131,7 +131,7 @@ wait_db_compact_done(_DbName, 0) -> {line, ?LINE}, {reason, "DB compaction failed to finish"} ], - erlang:error({assertion_failed, Failure}); + error({assertion_failed, Failure}); wait_db_compact_done(DbName, N) -> IsDone = couch_util:with_db(DbName, fun(Db) -> not is_pid(couch_db:get_compactor_pid(Db)) diff --git a/src/couch/test/eunit/couch_btree_prop_tests.erl b/src/couch/test/eunit/couch_btree_prop_tests.erl new file mode 100644 index 0000000000..1a9bd4f753 --- /dev/null +++ b/src/couch/test/eunit/couch_btree_prop_tests.erl @@ -0,0 +1,225 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_btree_prop_tests). + +-ifdef(WITH_PROPER). + +-export([ + command/1, + initial_state/0, + next_state/3, + precondition/2, + postcondition/3, + query_modify/3, + lookup/1 +]). + +% Process dict keys +-define(BTREE, btree). +-define(BTREE_FILENAME, btree_filename). + +-include_lib("couch/include/couch_eunit_proper.hrl"). +-include_lib("couch/include/couch_eunit.hrl"). + +btree_property_test_() -> + ?EUNIT_QUICKCHECK(90, 3000). + +% +% Properties +% + +prop_btree_() -> + ?FORALL( + Cmds, + commands(?MODULE), + begin + setup_btree(), + {Hist, St, Res} = run_commands(?MODULE, Cmds), + cleanup_btree(), + ?WHENFAIL( + on_failure(Hist, St, Res), + aggregate(command_names(Cmds), Res =:= ok) + ) + end + ). + +% +% Setup, teardown and proxy calls to couch_btree. +% + +% PropEr is a bit awkward when it comes to test setup and teardown, so to make +% everything easier use plain function calls in the test and keep the btree +% state in the process dictionary. + +setup_btree() -> + Filename = ?tempfile(), + {ok, Fd} = couch_file:open(Filename, [create, overwrite]), + {ok, Bt} = couch_btree:open(nil, Fd, [{compression, none}]), + put(?BTREE, Bt), + put(?BTREE_FILENAME, Filename). + +cleanup_btree() -> + Bt = get(?BTREE), + Fd = couch_btree:get_fd(Bt), + ok = couch_file:close(Fd), + ok = file:delete(get(?BTREE_FILENAME)), + erase(?BTREE), + erase(?BTREE_FILENAME). + +get_btree() -> + Bt = get(?BTREE), + % Sanity check. The btree didn't somehow disappear. + true = couch_btree:is_btree(Bt), + Bt. + +update_btree(NewBt) -> + OldBt = get(?BTREE), + % Sanity check. Expect an old btree instance to exist. + true = couch_btree:is_btree(OldBt), + true = couch_btree:is_btree(NewBt), + put(?BTREE, NewBt), + ok. + +% add/2 and add_remove/3 call query_modify/4, so just test that +% +query_modify(Lookups, Inserts, Removes) -> + Bt = get_btree(), + {ok, QueryResults, Bt1} = couch_btree:query_modify(Bt, Lookups, Inserts, Removes), + ok = update_btree(Bt1), + {ok, QueryResults, Bt1}. + +lookup(Keys) -> + Bt = get_btree(), + couch_btree:lookup(Bt, Keys). + +foldl() -> + Fun = fun({K, V}, Acc) -> {ok, [{K, V} | Acc]} end, + {ok, _, Acc1} = couch_btree:foldl(get_btree(), Fun, []), + lists:reverse(Acc1). + +% +% PropEr state callbacks +% + +% Model the btree as a simple orddict. +% +initial_state() -> + orddict:new(). + +command(Model) when is_list(Model) -> + frequency([ + {1, {call, ?MODULE, query_modify, [keys(), kvs(), remove_keys()]}}, + {1, {call, ?MODULE, lookup, [keys()]}} + ]). + +% These are called during command generation, the test with an actual model and +% during shrinking to guide the shrinking behavior. +% +precondition(_Model, {call, ?MODULE, query_modify, [_, _, _]}) -> + true; +precondition(Model, {call, ?MODULE, lookup, [Keys]}) -> + % Avoid exploring too many useless cases and only look up keys if we know + % we have some keys to look at + orddict:size(Model) > 0 andalso length(Keys) > 0. + +% Assuming the postcondition passed, advance to the next state +% +next_state(Model, _, {call, ?MODULE, query_modify, [_Lookups, Inserts, Removes]}) -> + model_add_remove(Model, Inserts, Removes); +next_state(Model, _, {call, ?MODULE, lookup, [_]}) -> + Model. + +% The Model is *before* the call is applied. The Result is *after* the command +% was applied to the actual btree we're testing. This is where the "model" vs +% "real" btree check happens. +% +postcondition(Model, {call, ?MODULE, query_modify, [Lookups, Inserts, Removes]}, Result) -> + {ok, QueryResults, _Bt} = Result, + ModelUpdate = model_add_remove(Model, Inserts, Removes), + ModelLookup = model_query(Model, Lookups), + ModelLookup == lists:sort(QueryResults) andalso ModelUpdate == foldl(); +postcondition(Model, {call, ?MODULE, lookup, [Lookups]}, Result) -> + model_lookup(Model, Lookups) == Result. + +% +% Generators +% + +key() -> + integer(1, 1000). + +val() -> + elements([a, b, c]). + +kvs() -> + list({key(), val()}). + +keys() -> + list(key()). + +remove_keys() -> + % Bias a bit towards not removing keys + frequency([{4, []}, {1, keys()}]). + +% +% Helper functions +% + +% Model (orddict) helpers + +model_add_remove(Model, Inserts, Removes) -> + % Keep this in sync with the op_order/1 from the + % couch_btree model: + % op_order(fetch) -> 1; + % op_order(remove) -> 2; + % op_order(insert) -> 3. + Model1 = model_remove(Model, Removes), + Model2 = model_insert(Model1, Inserts), + Model2. + +model_insert(Model, KVs) -> + UsortFun = fun({K1, _}, {K2, _}) -> K1 =< K2 end, + FoldFun = fun({K, V}, M) -> orddict:store(K, V, M) end, + lists:foldl(FoldFun, Model, lists:usort(UsortFun, KVs)). + +model_remove(Model, Keys) -> + FoldFun = fun(K, M) -> orddict:erase(K, M) end, + lists:foldl(FoldFun, Model, lists:usort(Keys)). + +model_lookup(Model, Keys) -> + Fun = + fun(K) -> + case orddict:find(K, Model) of + {ok, V} -> {ok, {K, V}}; + error -> not_found + end + end, + lists:map(Fun, Keys). + +model_query(Model, Keys) -> + Fun = + fun(K) -> + case orddict:find(K, Model) of + {ok, V} -> {ok, {K, V}}; + error -> {not_found, {K, nil}} + end + end, + lists:sort(lists:map(Fun, Keys)). + +% Other helpers + +on_failure(History, St, Res) -> + Msg = "~nWHENFAIL: History: ~p\nState: ~p\nResult: ~p\n", + io:format(standard_error, Msg, [History, St, Res]). + +-endif. diff --git a/src/couch/test/eunit/couch_btree_tests.erl b/src/couch/test/eunit/couch_btree_tests.erl index 1ec92818bb..f365a37cf1 100644 --- a/src/couch/test/eunit/couch_btree_tests.erl +++ b/src/couch/test/eunit/couch_btree_tests.erl @@ -13,7 +13,6 @@ -module(couch_btree_tests). -include_lib("couch/include/couch_eunit.hrl"). --include_lib("couch/include/couch_db.hrl"). -define(ROWS, 1000). % seconds @@ -30,6 +29,35 @@ setup() -> setup_kvs(_) -> setup(). +setup_kvs_with_cache(_) -> + {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]), + {ok, Btree} = couch_btree:open(nil, Fd, [ + {compression, none}, + {reduce, fun reduce_fun/2}, + {cache_depth, 2} + ]), + {Fd, Btree}. + +setup_kvs_with_small_chunk_size(_) -> + % Less than a 1/4 than current default 1279 + config:set("couchdb", "btree_chunk_size", "300", false), + {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]), + {ok, Btree} = couch_btree:open(nil, Fd, [ + {compression, none}, + {reduce, fun reduce_fun/2} + ]), + {Fd, Btree}. + +setup_kvs_with_large_chunk_size(_) -> + % About 4x than current default 1279 + config:set("couchdb", "btree_chunk_size", "5000", false), + {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]), + {ok, Btree} = couch_btree:open(nil, Fd, [ + {compression, none}, + {reduce, fun reduce_fun/2} + ]), + {Fd, Btree}. + setup_red() -> {_, EvenOddKVs} = lists:foldl( fun(Idx, {Key, Acc}) -> @@ -48,6 +76,7 @@ setup_red(_) -> setup_red(). teardown(Fd) when is_pid(Fd) -> + config:delete("couchdb", "btree_chunk_size", false), ok = couch_file:close(Fd); teardown({Fd, _}) -> teardown(Fd). @@ -81,7 +110,7 @@ btree_open_test_() -> {ok, Btree} = couch_btree:open(nil, Fd, [{compression, none}]), { "Ensure that created btree is really a btree record", - ?_assert(is_record(Btree, btree)) + ?_assert(couch_btree:is_btree(Btree)) }. sorted_kvs_test_() -> @@ -105,7 +134,7 @@ sorted_kvs_test_() -> rsorted_kvs_test_() -> Sorted = [{Seq, rand:uniform()} || Seq <- lists:seq(1, ?ROWS)], Funs = kvs_test_funs(), - Reversed = Sorted, + Reversed = lists:reverse(Sorted), { "BTree with backward sorted keys", { @@ -140,6 +169,60 @@ shuffled_kvs_test_() -> } }. +sorted_kvs_with_cache_test_() -> + Funs = kvs_test_funs(), + Sorted = [{Seq, rand:uniform()} || Seq <- lists:seq(1, ?ROWS)], + { + "BTree with a cache and sorted keys", + { + setup, + fun() -> test_util:start_couch() end, + fun test_util:stop/1, + { + foreachx, + fun setup_kvs_with_cache/1, + fun teardown/2, + [{Sorted, Fun} || Fun <- Funs] + } + } + }. + +sorted_kvs_small_chunk_size_test_() -> + Funs = kvs_test_funs(), + Sorted = [{Seq, rand:uniform()} || Seq <- lists:seq(1, ?ROWS)], + { + "BTree with a small chunk size and sorted keys", + { + setup, + fun() -> test_util:start_couch() end, + fun test_util:stop/1, + { + foreachx, + fun setup_kvs_with_small_chunk_size/1, + fun teardown/2, + [{Sorted, Fun} || Fun <- Funs] + } + } + }. + +sorted_kvs_large_chunk_size_test_() -> + Funs = kvs_test_funs(), + Sorted = [{Seq, rand:uniform()} || Seq <- lists:seq(1, ?ROWS)], + { + "BTree with a large chunk size and sorted keys", + { + setup, + fun() -> test_util:start_couch() end, + fun test_util:stop/1, + { + foreachx, + fun setup_kvs_with_large_chunk_size/1, + fun teardown/2, + [{Sorted, Fun} || Fun <- Funs] + } + } + }. + reductions_test_() -> { "BTree reductions", @@ -189,10 +272,10 @@ reductions_test_() -> }. should_set_fd_correctly(_, {Fd, Btree}) -> - ?_assertMatch(Fd, Btree#btree.fd). + ?_assertMatch(Fd, couch_btree:get_fd(Btree)). should_set_root_correctly(_, {_, Btree}) -> - ?_assertMatch(nil, Btree#btree.root). + ?_assertMatch(nil, couch_btree:get_state(Btree)). should_create_zero_sized_btree(_, {_, Btree}) -> ?_assertMatch(0, couch_btree:size(Btree)). @@ -200,7 +283,7 @@ should_create_zero_sized_btree(_, {_, Btree}) -> should_set_reduce_option(_, {_, Btree}) -> ReduceFun = fun reduce_fun/2, Btree1 = couch_btree:set_options(Btree, [{reduce, ReduceFun}]), - ?_assertMatch(ReduceFun, Btree1#btree.reduce). + ?_assertMatch(ReduceFun, couch_btree:get_reduce_fun(Btree1)). should_fold_over_empty_btree(_, {_, Btree}) -> {ok, _, EmptyRes} = couch_btree:foldl(Btree, fun(_, X) -> {ok, X + 1} end, 0), @@ -228,7 +311,7 @@ should_have_lesser_size_than_file(Fd, Btree) -> should_keep_root_pointer_to_kp_node(Fd, Btree) -> ?_assertMatch( {ok, {kp_node, _}}, - couch_file:pread_term(Fd, element(1, Btree#btree.root)) + couch_file:pread_term(Fd, element(1, couch_btree:get_state(Btree))) ). should_remove_all_keys(KeyValues, Btree) -> @@ -615,18 +698,15 @@ test_key_access(Btree, List) -> FoldFun = fun(Element, {[HAcc | TAcc], Count}) -> case Element == HAcc of true -> {ok, {TAcc, Count + 1}}; - _ -> {ok, {TAcc, Count + 1}} + _ -> {ok, {TAcc, Count}} end end, Length = length(List), Sorted = lists:sort(List), {ok, _, {[], Length}} = couch_btree:foldl(Btree, FoldFun, {Sorted, 0}), - {ok, _, {[], Length}} = couch_btree:fold( - Btree, - FoldFun, - {Sorted, 0}, - [{dir, rev}] - ), + Reversed = lists:reverse(Sorted), + RevOpts = [{dir, rev}], + {ok, _, {[], Length}} = couch_btree:fold(Btree, FoldFun, {Reversed, 0}, RevOpts), ok. test_lookup_access(Btree, KeyValues) -> diff --git a/src/couch/test/eunit/couch_changes_tests.erl b/src/couch/test/eunit/couch_changes_tests.erl index bbeeac26ae..917689e860 100644 --- a/src/couch/test/eunit/couch_changes_tests.erl +++ b/src/couch/test/eunit/couch_changes_tests.erl @@ -1654,7 +1654,7 @@ wait_finished({_, ConsumerRef}) -> {'DOWN', ConsumerRef, _, _, Msg} when Msg == normal; Msg == ok -> ok; {'DOWN', ConsumerRef, _, _, Msg} -> - erlang:error( + error( {consumer_died, [ {module, ?MODULE}, {line, ?LINE}, @@ -1662,7 +1662,7 @@ wait_finished({_, ConsumerRef}) -> ]} ) after ?TIMEOUT -> - erlang:error( + error( {consumer_died, [ {module, ?MODULE}, {line, ?LINE}, @@ -1752,7 +1752,7 @@ maybe_pause(Parent, Acc) -> Parent ! {ok, Ref}, throw({stop, Acc}); V when V /= updated -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch/test/eunit/couch_file_tests.erl b/src/couch/test/eunit/couch_file_tests.erl index 5b74816360..c138a9c0ab 100644 --- a/src/couch/test/eunit/couch_file_tests.erl +++ b/src/couch/test/eunit/couch_file_tests.erl @@ -946,3 +946,86 @@ legacy_stats() -> reset_legacy_checksum_stats() -> Counter = couch_stats:sample([couchdb, legacy_checksums]), couch_stats:decrement_counter([couchdb, legacy_checksums], Counter). + +write_header_sync_test_() -> + { + "Test sync options for write_header", + { + setup, + fun test_util:start_couch/0, + fun test_util:stop_couch/1, + { + foreach, + fun unlinked_setup/0, + fun teardown/1, + [ + ?TDEF_FE(should_handle_sync_option), + ?TDEF_FE(should_not_sync_by_default), + ?TDEF_FE(should_handle_error_of_the_first_sync), + ?TDEF_FE(should_handle_error_of_the_second_sync), + ?TDEF_FE(should_handle_error_of_the_file_write) + ] + } + } + }. + +unlinked_setup() -> + Self = self(), + ReqId = make_ref(), + meck:new(file, [passthrough, unstick]), + spawn(fun() -> + {ok, Fd} = couch_file:open(?tempfile(), [create, overwrite]), + Self ! {ReqId, Fd} + end), + receive + {ReqId, Result} -> Result + end. + +should_handle_sync_option(Fd) -> + ok = couch_file:write_header(Fd, {<<"some_data">>, 32}, [sync]), + ?assertMatch({ok, {<<"some_data">>, 32}}, couch_file:read_header(Fd)), + ?assertEqual(2, meck:num_calls(file, datasync, ['_'])), + ok. + +should_not_sync_by_default(Fd) -> + ok = couch_file:write_header(Fd, {<<"some_data">>, 32}), + ?assertMatch({ok, {<<"some_data">>, 32}}, couch_file:read_header(Fd)), + ?assertEqual(0, meck:num_calls(file, datasync, ['_'])), + ok. + +should_handle_error_of_the_first_sync(Fd) -> + meck:expect( + file, + datasync, + ['_'], + meck:val({error, terminated}) + ), + ?assertEqual({error, terminated}, couch_file:write_header(Fd, {<<"some_data">>, 32}, [sync])), + ?assertEqual(1, meck:num_calls(file, datasync, ['_'])), + ok. + +should_handle_error_of_the_second_sync(Fd) -> + meck:expect( + file, + datasync, + ['_'], + meck:seq([ + meck:val(ok), + meck:val({error, terminated}) + ]) + ), + ?assertEqual({error, terminated}, couch_file:write_header(Fd, {<<"some_data">>, 32}, [sync])), + ?assertEqual(2, meck:num_calls(file, datasync, ['_'])), + ok. + +should_handle_error_of_the_file_write(Fd) -> + meck:expect( + file, + write, + ['_', '_'], + meck:val({error, terminated}) + ), + ?assertEqual({error, terminated}, couch_file:write_header(Fd, {<<"some_data">>, 32}, [sync])), + ?assertEqual(1, meck:num_calls(file, datasync, ['_'])), + ?assertEqual(1, meck:num_calls(file, write, ['_', '_'])), + ok. diff --git a/src/couch/test/eunit/couch_index_tests.erl b/src/couch/test/eunit/couch_index_tests.erl index 368f7a0596..3c078ca157 100644 --- a/src/couch/test/eunit/couch_index_tests.erl +++ b/src/couch/test/eunit/couch_index_tests.erl @@ -245,7 +245,7 @@ tracer_new() -> ok. tracer_delete() -> - dbg:stop_clear(), + dbg:stop(), (catch ets:delete(?MODULE)), ok. diff --git a/src/couch/test/eunit/couch_query_servers_tests.erl b/src/couch/test/eunit/couch_query_servers_tests.erl index 1ade40b670..a2fffb446d 100644 --- a/src/couch/test/eunit/couch_query_servers_tests.erl +++ b/src/couch/test/eunit/couch_query_servers_tests.erl @@ -16,96 +16,110 @@ -include_lib("couch/include/couch_eunit.hrl"). setup() -> - meck:new([config, couch_log]). + meck:new(couch_log, [passthrough]), + Ctx = test_util:start_couch([ioq]), + config:set("query_server_config", "reduce_limit", "true", false), + config:set("log", "level", "info", false), + Ctx. -teardown(_) -> +teardown(Ctx) -> + config:delete("query_server_config", "reduce_limit", true), + config:delete("query_server_config", "reduce_limit_threshold", true), + config:delete("query_server_config", "reduce_limit_ratio", true), + config:delete("log", "level", false), + test_util:stop_couch(Ctx), meck:unload(). -setup_oom() -> - test_util:start_couch([ioq]). - -teardown_oom(Ctx) -> - meck:unload(), - test_util:stop_couch(Ctx). - -sum_overflow_test_() -> +query_server_limits_test_() -> { - "Test overflow detection in the _sum reduce function", + "Test overflow detection and batch splitting in query server", { - setup, + foreach, fun setup/0, fun teardown/1, [ - fun should_return_error_on_overflow/0, - fun should_return_object_on_log/0, - fun should_return_object_on_false/0 - ] - } - }. - -filter_oom_test_() -> - { - "Test recovery from oom in filters", - { - setup, - fun setup_oom/0, - fun teardown_oom/1, - [ - fun should_split_large_batches/0 + ?TDEF_FE(builtin_should_return_error_on_overflow), + ?TDEF_FE(builtin_should_not_return_error_with_generous_overflow_threshold), + ?TDEF_FE(builtin_should_not_return_error_with_generous_overflow_ratio), + ?TDEF_FE(builtin_should_return_object_on_log), + ?TDEF_FE(builtin_should_return_object_on_false), + ?TDEF_FE(js_reduce_should_return_error_on_overflow), + ?TDEF_FE(js_reduce_should_return_object_on_log), + ?TDEF_FE(js_reduce_should_return_object_on_false), + ?TDEF_FE(should_split_large_batches) ] } }. -should_return_error_on_overflow() -> - meck:reset([config, couch_log]), - meck:expect( - config, - get, - ["query_server_config", "reduce_limit", "true"], - "true" - ), - meck:expect(couch_log, error, ['_', '_'], ok), +builtin_should_return_error_on_overflow(_) -> + config:set("query_server_config", "reduce_limit", "true", false), + meck:reset(couch_log), KVs = gen_sum_kvs(), {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), ?assertMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result), - ?assert(meck:called(config, get, '_')), ?assert(meck:called(couch_log, error, '_')). -should_return_object_on_log() -> - meck:reset([config, couch_log]), - meck:expect( - config, - get, - ["query_server_config", "reduce_limit", "true"], - "log" - ), - meck:expect(couch_log, error, ['_', '_'], ok), +builtin_should_not_return_error_with_generous_overflow_threshold(_) -> + config:set("query_server_config", "reduce_limit", "true", false), + config:set_integer("query_server_config", "reduce_limit_threshold", 1000000, false), + meck:reset(couch_log), + KVs = gen_sum_kvs(), + {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), + ?assertNotMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result). + +builtin_should_not_return_error_with_generous_overflow_ratio(_) -> + config:set("query_server_config", "reduce_limit", "true", false), + config:set_float("query_server_config", "reduce_limit_ratio", 0.1, false), + meck:reset(couch_log), + KVs = gen_sum_kvs(), + {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), + ?assertNotMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result). + +builtin_should_return_object_on_log(_) -> + config:set("query_server_config", "reduce_limit", "log", false), + meck:reset(couch_log), KVs = gen_sum_kvs(), {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), ?assertMatch({[_ | _]}, Result), Keys = [K || {K, _} <- element(1, Result)], ?assert(not lists:member(<<"error">>, Keys)), - ?assert(meck:called(config, get, '_')), ?assert(meck:called(couch_log, error, '_')). -should_return_object_on_false() -> - meck:reset([config, couch_log]), - meck:expect( - config, - get, - ["query_server_config", "reduce_limit", "true"], - "false" - ), - meck:expect(couch_log, error, ['_', '_'], ok), +builtin_should_return_object_on_false(_) -> + config:set("query_server_config", "reduce_limit", "false", false), + meck:reset(couch_log), KVs = gen_sum_kvs(), {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs), ?assertMatch({[_ | _]}, Result), Keys = [K || {K, _} <- element(1, Result)], ?assert(not lists:member(<<"error">>, Keys)), - ?assert(meck:called(config, get, '_')), ?assertNot(meck:called(couch_log, error, '_')). -should_split_large_batches() -> +js_reduce_should_return_error_on_overflow(_) -> + config:set("query_server_config", "reduce_limit", "true", false), + meck:reset(couch_log), + KVs = gen_sum_kvs(), + {ok, [Result]} = couch_query_servers:reduce(<<"javascript">>, [sum_js()], KVs), + ?assertMatch({[{reduce_overflow_error, <<"Reduce output must shrink", _/binary>>}]}, Result), + ?assert(meck:called(couch_log, error, '_')). + +js_reduce_should_return_object_on_log(_) -> + config:set("query_server_config", "reduce_limit", "log", false), + meck:reset(couch_log), + KVs = gen_sum_kvs(), + {ok, [Result]} = couch_query_servers:reduce(<<"javascript">>, [sum_js()], KVs), + ?assertMatch([<<"result">>, [_ | _]], Result), + ?assert(meck:called(couch_log, info, '_')). + +js_reduce_should_return_object_on_false(_) -> + config:set("query_server_config", "reduce_limit", "false", false), + meck:reset(couch_log), + KVs = gen_sum_kvs(), + {ok, [Result]} = couch_query_servers:reduce(<<"javascript">>, [sum_js()], KVs), + ?assertMatch([<<"result">>, [_ | _]], Result), + ?assertNot(meck:called(couch_log, info, '_')). + +should_split_large_batches(_) -> Req = {json_req, {[]}}, Db = <<"somedb">>, DDoc = #doc{ @@ -152,3 +166,13 @@ gen_sum_kvs() -> end, lists:seq(1, 10) ). + +sum_js() -> + % Don't do this in real views + << + "\n" + " function(keys, vals, rereduce) {\n" + " return ['result', vals.concat(vals)]\n" + " }\n" + " " + >>. diff --git a/src/couch/test/eunit/couch_server_tests.erl b/src/couch/test/eunit/couch_server_tests.erl index a43106d890..77ddfde213 100644 --- a/src/couch/test/eunit/couch_server_tests.erl +++ b/src/couch/test/eunit/couch_server_tests.erl @@ -224,7 +224,7 @@ t_interleaved_create_delete_open(DbName) -> % Now monitor and resume the couch_server and assert that % couch_server does not crash while processing OpenResultMsg - CSRef = erlang:monitor(process, CouchServer), + CSRef = monitor(process, CouchServer), erlang:resume_process(CouchServer), check_monitor_not_triggered(CSRef), @@ -253,7 +253,7 @@ get_opener_pid(DbName) -> wait_for_open_async_result(CouchServer, Opener) -> WaitFun = fun() -> - {_, Messages} = erlang:process_info(CouchServer, messages), + {_, Messages} = process_info(CouchServer, messages), Found = lists:foldl( fun(Msg, Acc) -> case Msg of @@ -276,7 +276,7 @@ wait_for_open_async_result(CouchServer, Opener) -> check_monitor_not_triggered(Ref) -> receive {'DOWN', Ref, _, _, Reason0} -> - erlang:error({monitor_triggered, Reason0}) + error({monitor_triggered, Reason0}) after 100 -> ok end. @@ -286,5 +286,5 @@ get_next_message() -> Msg -> Msg after 5000 -> - erlang:error(timeout) + error(timeout) end. diff --git a/src/couch/test/eunit/couch_stream_tests.erl b/src/couch/test/eunit/couch_stream_tests.erl index f29c04c82c..6bc2c7952c 100644 --- a/src/couch/test/eunit/couch_stream_tests.erl +++ b/src/couch/test/eunit/couch_stream_tests.erl @@ -124,7 +124,7 @@ should_stop_on_normal_exit_of_stream_opener({Fd, _}) -> end, ?assertNot(is_process_alive(OpenerPid)), % Verify the stream itself has also died - StreamRef = erlang:monitor(process, StreamPid), + StreamRef = monitor(process, StreamPid), receive {'DOWN', StreamRef, _, _, _} -> ok end, diff --git a/src/couch/test/eunit/couch_task_status_tests.erl b/src/couch/test/eunit/couch_task_status_tests.erl index e5d8ead980..90e25362ac 100644 --- a/src/couch/test/eunit/couch_task_status_tests.erl +++ b/src/couch/test/eunit/couch_task_status_tests.erl @@ -181,7 +181,7 @@ loop() -> end. call(Pid, done) -> - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), Pid ! {done, self()}, Res = wait(Pid), receive @@ -222,7 +222,7 @@ get_task_prop(Pid, Prop) -> ), case couch_util:get_value(Prop, hd(Element), nil) of nil -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch/test/eunit/couch_work_queue_tests.erl b/src/couch/test/eunit/couch_work_queue_tests.erl index c41cf598e4..85c3f1a3c8 100644 --- a/src/couch/test/eunit/couch_work_queue_tests.erl +++ b/src/couch/test/eunit/couch_work_queue_tests.erl @@ -375,7 +375,7 @@ produce_term(Q, Producer, Term, Wait) -> {item, Ref, Item} -> Item after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -395,7 +395,7 @@ produce(Q, Producer, Size, Wait) -> {item, Ref, Item} -> Item after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch/test/eunit/couchdb_attachments_tests.erl b/src/couch/test/eunit/couchdb_attachments_tests.erl index 5de1eae9e0..103a023268 100644 --- a/src/couch/test/eunit/couchdb_attachments_tests.erl +++ b/src/couch/test/eunit/couchdb_attachments_tests.erl @@ -646,7 +646,7 @@ wait_compaction(DbName, Kind, Line) -> end, case test_util:wait(WaitFun, ?TIMEOUT) of timeout -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, Line}, diff --git a/src/couch/test/eunit/couchdb_file_compression_tests.erl b/src/couch/test/eunit/couchdb_file_compression_tests.erl index 122900d4ba..1016cd2de2 100644 --- a/src/couch/test/eunit/couchdb_file_compression_tests.erl +++ b/src/couch/test/eunit/couchdb_file_compression_tests.erl @@ -222,7 +222,7 @@ wait_compaction(DbName, Kind, Line) -> end, case test_util:wait(WaitFun, ?TIMEOUT * 1000) of timeout -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, Line}, diff --git a/src/couch/test/eunit/couchdb_os_proc_pool.erl b/src/couch/test/eunit/couchdb_os_proc_pool.erl index 1a8479497d..e230e761a1 100644 --- a/src/couch/test/eunit/couchdb_os_proc_pool.erl +++ b/src/couch/test/eunit/couchdb_os_proc_pool.erl @@ -562,7 +562,7 @@ get_client_proc({Pid, Ref}, ClientName) -> receive {proc, Ref, Proc} -> Proc after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -574,7 +574,7 @@ get_client_proc({Pid, Ref}, ClientName) -> end. stop_client({Pid, Ref}) -> - MRef = erlang:monitor(process, Pid), + MRef = monitor(process, Pid), Pid ! stop, receive {stop, Ref} -> @@ -583,12 +583,12 @@ stop_client({Pid, Ref}) -> end, ok after ?TIMEOUT -> - erlang:demonitor(MRef, [flush]), + demonitor(MRef, [flush]), timeout end. kill_client({Pid, Ref}) -> - MRef = erlang:monitor(process, Pid), + MRef = monitor(process, Pid), Pid ! die, receive {die, Ref} -> @@ -597,7 +597,7 @@ kill_client({Pid, Ref}) -> end, ok after ?TIMEOUT -> - erlang:demonitor(MRef, [flush]), + demonitor(MRef, [flush]), timeout end. diff --git a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl index 0722103a4e..fc7884ed90 100644 --- a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl +++ b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl @@ -120,7 +120,7 @@ concurrent_doc_update(NumClients, DbName, InitRev) -> ]} ), Pid = spawn_client(DbName, ClientDoc), - {Value, Pid, erlang:monitor(process, Pid)} + {Value, Pid, monitor(process, Pid)} end, lists:seq(1, NumClients) ), @@ -135,7 +135,7 @@ concurrent_doc_update(NumClients, DbName, InitRev) -> {'DOWN', MonRef, process, Pid, conflict} -> {AccConflicts + 1, AccValue}; {'DOWN', MonRef, process, Pid, Error} -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -146,7 +146,7 @@ concurrent_doc_update(NumClients, DbName, InitRev) -> ]} ) after ?TIMEOUT div 2 -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch/test/eunit/couchdb_vhosts_tests.erl b/src/couch/test/eunit/couchdb_vhosts_tests.erl index d1b7589140..e005537cba 100644 --- a/src/couch/test/eunit/couchdb_vhosts_tests.erl +++ b/src/couch/test/eunit/couchdb_vhosts_tests.erl @@ -96,7 +96,7 @@ should_return_database_info({Url, DbName}) -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"db_name">>, JsonBody)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -120,7 +120,7 @@ should_return_revs_info({Url, DbName}) -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"_revs_info">>, JsonBody)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -146,7 +146,7 @@ should_return_virtual_request_path_field_in_request({Url, DbName}) -> proplists:get_value(<<"requested_path">>, Json) ); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -170,7 +170,7 @@ should_return_real_request_path_field_in_request({Url, DbName}) -> Path = ?l2b("/" ++ DbName ++ "/_design/doc1/_show/test"), ?assertEqual(Path, proplists:get_value(<<"path">>, Json)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -194,7 +194,7 @@ should_match_wildcard_vhost({Url, DbName}) -> Path = ?l2b("/" ++ DbName ++ "/_design/doc1/_show/test"), ?assertEqual(Path, proplists:get_value(<<"path">>, Json)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -218,7 +218,7 @@ should_return_db_info_for_wildcard_vhost_for_custom_db({Url, DbName}) -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"db_name">>, JsonBody)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -243,7 +243,7 @@ should_replace_rewrite_variables_for_db_and_doc({Url, DbName}) -> Path = ?l2b("/" ++ DbName ++ "/_design/doc1/_show/test"), ?assertEqual(Path, proplists:get_value(<<"path">>, Json)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -267,7 +267,7 @@ should_return_db_info_for_vhost_with_resource({Url, DbName}) -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"db_name">>, JsonBody)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -291,7 +291,7 @@ should_return_revs_info_for_vhost_with_resource({Url, DbName}) -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"_revs_info">>, JsonBody)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -311,7 +311,7 @@ should_return_db_info_for_vhost_with_wildcard_resource({Url, DbName}) -> {JsonBody} = jiffy:decode(Body), ?assert(proplists:is_defined(<<"db_name">>, JsonBody)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -335,7 +335,7 @@ should_return_path_for_vhost_with_wildcard_host({Url, DbName}) -> Path = ?l2b("/" ++ DbName ++ "/_design/doc1/_show/test"), ?assertEqual(Path, proplists:get_value(<<"path">>, Json)); Else -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch/test/eunit/couchdb_views_tests.erl b/src/couch/test/eunit/couchdb_views_tests.erl index 0d32d7fcf5..8b1950948c 100644 --- a/src/couch/test/eunit/couchdb_views_tests.erl +++ b/src/couch/test/eunit/couchdb_views_tests.erl @@ -728,7 +728,7 @@ couchdb_1283() -> ), % Start and pause compacton - WaitRef = erlang:make_ref(), + WaitRef = make_ref(), meck:expect(couch_mrview_index, compact, fun(Db, State, Opts) -> receive {WaitRef, From, init} -> ok @@ -741,7 +741,7 @@ couchdb_1283() -> end), {ok, CPid} = gen_server:call(Pid, compact), - CRef = erlang:monitor(process, CPid), + CRef = monitor(process, CPid), ?assert(is_process_alive(CPid)), % Make sure that our compactor is waiting for us @@ -773,7 +773,7 @@ wait_for_process_shutdown(Pid, ExpectedReason, Error) -> {'DOWN', Pid, process, _, Reason} -> ?assertEqual(ExpectedReason, Reason) after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [{module, ?MODULE}, {line, ?LINE}, Error]} ) end. @@ -994,7 +994,7 @@ compact_db(DbName) -> wait_db_compact_done(DbName, ?WAIT_DELAY_COUNT). wait_db_compact_done(_DbName, 0) -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -1020,7 +1020,7 @@ compact_view_group(DbName, DDocId) when is_binary(DDocId) -> wait_view_compact_done(DbName, DDocId, 10). wait_view_compact_done(_DbName, _DDocId, 0) -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -1052,7 +1052,7 @@ read_header(File) -> stop_indexer(StopFun, Pid, Line, Reason) -> case test_util:stop_sync(Pid, StopFun) of timeout -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, Line}, diff --git a/src/couch_event/src/couch_event_listener.erl b/src/couch_event/src/couch_event_listener.erl index 22bf233274..87d17b6338 100644 --- a/src/couch_event/src/couch_event_listener.erl +++ b/src/couch_event/src/couch_event_listener.erl @@ -47,7 +47,7 @@ term(). start(Mod, Arg, Options) -> - Pid = erlang:spawn(?MODULE, do_init, [Mod, Arg, Options]), + Pid = spawn(?MODULE, do_init, [Mod, Arg, Options]), {ok, Pid}. start(Name, Mod, Arg, Options) -> @@ -59,7 +59,7 @@ start(Name, Mod, Arg, Options) -> end. start_link(Mod, Arg, Options) -> - Pid = erlang:spawn_link(?MODULE, do_init, [Mod, Arg, Options]), + Pid = spawn_link(?MODULE, do_init, [Mod, Arg, Options]), {ok, Pid}. start_link(Name, Mod, Arg, Options) -> @@ -87,7 +87,7 @@ do_init(Module, Arg, Options) -> {ok, State, Timeout} when is_integer(Timeout), Timeout >= 0 -> ?MODULE:loop(#st{module = Module, state = State}, Timeout); Else -> - erlang:exit(Else) + exit(Else) end. loop(St, Timeout) -> @@ -109,7 +109,7 @@ maybe_name_process(Options) -> true -> ok; {false, Pid} -> - erlang:error({already_started, Pid}) + error({already_started, Pid}) end; none -> ok @@ -133,7 +133,7 @@ do_event(#st{module = Module, state = State} = St, DbName, Event) -> {stop, Reason, NewState} -> do_terminate(Reason, St#st{state = NewState}); Else -> - erlang:error(Else) + error(Else) end. do_cast(#st{module = Module, state = State} = St, Message) -> @@ -145,7 +145,7 @@ do_cast(#st{module = Module, state = State} = St, Message) -> {stop, Reason, NewState} -> do_terminate(Reason, St#st{state = NewState}); Else -> - erlang:error(Else) + error(Else) end. do_info(#st{module = Module, state = State} = St, Message) -> @@ -157,7 +157,7 @@ do_info(#st{module = Module, state = State} = St, Message) -> {stop, Reason, NewState} -> do_terminate(Reason, St#st{state = NewState}); Else -> - erlang:error(Else) + error(Else) end. do_terminate(Reason, #st{module = Module, state = State}) -> @@ -173,7 +173,7 @@ do_terminate(Reason, #st{module = Module, state = State}) -> ignore -> normal; Else -> Else end, - erlang:exit(Status). + exit(Status). where({local, Name}) -> whereis(Name). @@ -192,7 +192,7 @@ get_all_dbnames(Options) -> end. get_all_dbnames([], []) -> - erlang:error(no_dbnames_provided); + error(no_dbnames_provided); get_all_dbnames([], Acc) -> lists:usort(convert_dbname_list(Acc)); get_all_dbnames([{dbname, DbName} | Rest], Acc) -> @@ -209,4 +209,4 @@ convert_dbname_list([DbName | Rest]) when is_binary(DbName) -> convert_dbname_list([DbName | Rest]) when is_list(DbName) -> [list_to_binary(DbName) | convert_dbname_list(Rest)]; convert_dbname_list([DbName | _]) -> - erlang:error({invalid_dbname, DbName}). + error({invalid_dbname, DbName}). diff --git a/src/couch_event/src/couch_event_listener_mfa.erl b/src/couch_event/src/couch_event_listener_mfa.erl index 5ec465cf75..55099c2c4c 100644 --- a/src/couch_event/src/couch_event_listener_mfa.erl +++ b/src/couch_event/src/couch_event_listener_mfa.erl @@ -47,7 +47,7 @@ enter_loop(Mod, Func, State, Options) -> Parent = case proplists:get_value(parent, Options) of P when is_pid(P) -> - erlang:monitor(process, P), + monitor(process, P), P; _ -> undefined @@ -64,7 +64,7 @@ stop(Pid) -> couch_event_listener:cast(Pid, shutdown). init({Parent, Mod, Func, State}) -> - erlang:monitor(process, Parent), + monitor(process, Parent), {ok, #st{ mod = Mod, func = Func, @@ -86,7 +86,7 @@ handle_event(DbName, Event, #st{mod = Mod, func = Func, state = State} = St) -> couch_log:error("~p: else in handle_event for db ~p, event ~p, else ~p", [ ?MODULE, DbName, Event, Else ]), - erlang:error(Else) + error(Else) end catch Class:Reason:Stack -> diff --git a/src/couch_event/src/couch_event_server.erl b/src/couch_event/src/couch_event_server.erl index dfdf0ac020..f147306883 100644 --- a/src/couch_event/src/couch_event_server.erl +++ b/src/couch_event/src/couch_event_server.erl @@ -42,7 +42,7 @@ init(_) -> handle_call({register, Pid, NewDbNames}, _From, St) -> case maps:get(Pid, St#st.by_pid, undefined) of undefined -> - NewRef = erlang:monitor(process, Pid), + NewRef = monitor(process, Pid), {reply, ok, register(St, NewRef, Pid, NewDbNames)}; {ReuseRef, OldDbNames} -> unregister(St, Pid, OldDbNames), @@ -53,7 +53,7 @@ handle_call({unregister, Pid}, _From, #st{by_pid = ByPid} = St) -> undefined -> {reply, not_registered, St}; {Ref, OldDbNames} -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), {reply, ok, unregister(St, Pid, OldDbNames)} end; handle_call(Msg, From, St) -> @@ -229,7 +229,7 @@ t_invalid_gen_server_messages(_) -> kill_sync(Pid) -> unlink(Pid), - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), exit(Pid, kill), receive {'DOWN', Ref, _, _, _} -> ok diff --git a/src/couch_index/src/couch_index.erl b/src/couch_index/src/couch_index.erl index b8bb351832..0d0e62fb5f 100644 --- a/src/couch_index/src/couch_index.erl +++ b/src/couch_index/src/couch_index.erl @@ -61,7 +61,7 @@ compact(Pid) -> compact(Pid, Options) -> {ok, CPid} = gen_server:call(Pid, compact), case lists:member(monitor, Options) of - true -> {ok, erlang:monitor(process, CPid)}; + true -> {ok, monitor(process, CPid)}; false -> ok end. @@ -381,7 +381,7 @@ send_replies(Waiters, UpdateSeq, IdxState) -> assert_signature_match(Mod, OldIdxState, NewIdxState) -> case {Mod:get(signature, OldIdxState), Mod:get(signature, NewIdxState)} of {Sig, Sig} -> ok; - _ -> erlang:error(signature_mismatch) + _ -> error(signature_mismatch) end. commit_compacted(NewIdxState, State) -> diff --git a/src/couch_index/src/couch_index_server.erl b/src/couch_index/src/couch_index_server.erl index 973b063374..d4593ee0d5 100644 --- a/src/couch_index/src/couch_index_server.erl +++ b/src/couch_index/src/couch_index_server.erl @@ -78,7 +78,7 @@ get_index(Module, <<"shards/", _/binary>> = DbName, DDoc) -> {'DOWN', Ref, process, Pid, Error} -> Error after 61000 -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), {error, timeout} end; get_index(Module, DbName, DDoc) when is_binary(DbName) -> diff --git a/src/couch_index/src/couch_index_util.erl b/src/couch_index/src/couch_index_util.erl index db8aad470e..9a16d06d67 100644 --- a/src/couch_index/src/couch_index_util.erl +++ b/src/couch_index/src/couch_index_util.erl @@ -14,6 +14,7 @@ -export([root_dir/0, index_dir/2, index_file/3]). -export([load_doc/3, sort_lib/1, hexsig/1]). +-export([get_purge_checkpoints/2, cleanup_purges/3]). -include_lib("couch/include/couch_db.hrl"). @@ -72,3 +73,49 @@ sort_lib([{LName, LCode} | Rest], LAcc) -> hexsig(Sig) -> couch_util:to_hex(Sig). + +% Helper function for indexes to get their purge checkpoints as signatures. +% +get_purge_checkpoints(DbName, Type) when is_binary(DbName), is_binary(Type) -> + couch_util:with_db(DbName, fun(Db) -> get_purge_checkpoints(Db, Type) end); +get_purge_checkpoints(Db, Type) when is_binary(Type) -> + Prefix = <>, + PrefixSize = byte_size(Prefix), + FoldFun = fun(#doc{id = Id}, Acc) -> + case Id of + <> -> {ok, Acc#{Sig => Id}}; + _ -> {stop, Acc} + end + end, + Opts = [{start_key, Prefix}], + {ok, Signatures = #{}} = couch_db:fold_local_docs(Db, FoldFun, #{}, Opts), + Signatures. + +% Helper functions for indexes to clean their purge checkpoints. +% +cleanup_purges(DbName, #{} = Sigs, #{} = Checkpoints) when is_binary(DbName) -> + couch_util:with_db(DbName, fun(Db) -> + cleanup_purges(Db, Sigs, Checkpoints) + end); +cleanup_purges(Db, #{} = Sigs, #{} = CheckpointsMap) -> + InactiveMap = maps:without(maps:keys(Sigs), CheckpointsMap), + InactiveCheckpoints = maps:values(InactiveMap), + DeleteFun = fun(DocId) -> delete_checkpoint(Db, DocId) end, + lists:foreach(DeleteFun, InactiveCheckpoints). + +delete_checkpoint(Db, DocId) -> + DbName = couch_db:name(Db), + LogMsg = "~p : deleting inactive purge checkpoint ~s : ~s", + couch_log:debug(LogMsg, [?MODULE, DbName, DocId]), + try couch_db:open_doc(Db, DocId, []) of + {ok, Doc = #doc{}} -> + Deleted = Doc#doc{deleted = true, body = {[]}}, + couch_db:update_doc(Db, Deleted, [?ADMIN_CTX]); + {not_found, _} -> + ok + catch + Tag:Error -> + ErrLog = "~p : error deleting checkpoint ~s : ~s error: ~p:~p", + couch_log:error(ErrLog, [?MODULE, DbName, DocId, Tag, Error]), + ok + end. diff --git a/src/couch_index/test/eunit/couch_index_ddoc_updated_tests.erl b/src/couch_index/test/eunit/couch_index_ddoc_updated_tests.erl index cbdb719545..6b7fe5a4a3 100644 --- a/src/couch_index/test/eunit/couch_index_ddoc_updated_tests.erl +++ b/src/couch_index/test/eunit/couch_index_ddoc_updated_tests.erl @@ -88,7 +88,7 @@ check_all_indexers_exit_on_ddoc_change({_Ctx, DbName}) -> IndexesBefore = get_indexes_by_ddoc(DDocID, N), ?assertEqual(N, length(IndexesBefore)), - AliveBefore = lists:filter(fun erlang:is_process_alive/1, IndexesBefore), + AliveBefore = lists:filter(fun is_process_alive/1, IndexesBefore), ?assertEqual(N, length(AliveBefore)), % update ddoc @@ -121,7 +121,7 @@ check_all_indexers_exit_on_ddoc_change({_Ctx, DbName}) -> ?assertEqual(0, length(IndexesAfter)), %% assert that previously running indexes are gone - AliveAfter = lists:filter(fun erlang:is_process_alive/1, IndexesBefore), + AliveAfter = lists:filter(fun is_process_alive/1, IndexesBefore), ?assertEqual(0, length(AliveAfter)), ok end). diff --git a/src/couch_log/src/couch_log_config.erl b/src/couch_log/src/couch_log_config.erl index c3fd8fe85c..265e4d15fd 100644 --- a/src/couch_log/src/couch_log_config.erl +++ b/src/couch_log/src/couch_log_config.erl @@ -108,7 +108,7 @@ transform(filter_fields, FieldsStr) -> Default = [pid, registered_name, error_info, messages], case parse_term(FieldsStr) of {ok, List} when is_list(List) -> - case lists:all(fun erlang:is_atom/1, List) of + case lists:all(fun is_atom/1, List) of true -> List; false -> diff --git a/src/couch_log/src/couch_log_trunc_io.erl b/src/couch_log/src/couch_log_trunc_io.erl index c330edb336..72ce5390e0 100644 --- a/src/couch_log/src/couch_log_trunc_io.erl +++ b/src/couch_log/src/couch_log_trunc_io.erl @@ -69,7 +69,7 @@ format(Fmt, Args, Max, Options) -> couch_log_trunc_io_fmt:format(Fmt, Args, Max, Options) catch _What:_Why -> - erlang:error(badarg, [Fmt, Args]) + error(badarg, [Fmt, Args]) end. %% @doc Returns an flattened list containing the ASCII representation of the given @@ -114,7 +114,7 @@ print(Term, Max, Options) when is_list(Options) -> print(Term, _Max, #print_options{force_strings = true}) when not is_list(Term), not is_binary(Term), not is_atom(Term) -> - erlang:error(badarg); + error(badarg); print(_, Max, _Options) when Max < 0 -> {"...", 3}; print(_, _, #print_options{depth = 0}) -> {"...", 3}; @@ -249,7 +249,7 @@ print(Fun, Max, _Options) when is_function(Fun) -> L = erlang:fun_to_list(Fun), case length(L) > Max of true -> - S = erlang:max(5, Max), + S = max(5, Max), Res = string:substr(L, 1, S) ++ "..>", {Res, length(Res)}; _ -> @@ -347,7 +347,7 @@ list_bodyc(X, Max, Options, _Tuple) -> {[$|, List], Len + 1}. map_body(Map, Max, #print_options{depth = Depth}) when Max < 4; Depth =:= 0 -> - case erlang:map_size(Map) of + case map_size(Map) of 0 -> {[], 0}; _ -> {"...", 3} end; @@ -474,7 +474,7 @@ alist([H | T], Max, Options = #print_options{force_strings = true}) when is_bina {[List | Final], FLen + Len} end; alist(_, _, #print_options{force_strings = true}) -> - erlang:error(badarg); + error(badarg); alist([H | _L], _Max, _Options) -> throw({unprintable, H}); alist(H, _Max, _Options) -> diff --git a/src/couch_log/src/couch_log_trunc_io_fmt.erl b/src/couch_log/src/couch_log_trunc_io_fmt.erl index 40f3248c23..3c772bd6d4 100644 --- a/src/couch_log/src/couch_log_trunc_io_fmt.erl +++ b/src/couch_log/src/couch_log_trunc_io_fmt.erl @@ -56,10 +56,10 @@ format(FmtStr, Args, MaxLen, Opts) when is_list(FmtStr) -> ), build2(Cs2, Count, MaxLen2 - StrLen); false -> - erlang:error(badarg) + error(badarg) end; format(_FmtStr, _Args, _MaxLen, _Opts) -> - erlang:error(badarg). + error(badarg). collect([$~ | Fmt0], Args0) -> {C, Fmt1, Args1} = collect_cseq(Fmt0, Args0), @@ -324,7 +324,7 @@ term(T, F, Adj, P0, Pad) -> L = lists:flatlength(T), P = case P0 of - none -> erlang:min(L, F); + none -> min(L, F); _ -> P0 end, if @@ -501,10 +501,10 @@ unprefixed_integer(Int, F, Adj, Base, Pad, Lowercase) when -> if Int < 0 -> - S = cond_lowercase(erlang:integer_to_list(-Int, Base), Lowercase), + S = cond_lowercase(integer_to_list(-Int, Base), Lowercase), term([$- | S], F, Adj, none, Pad); true -> - S = cond_lowercase(erlang:integer_to_list(Int, Base), Lowercase), + S = cond_lowercase(integer_to_list(Int, Base), Lowercase), term(S, F, Adj, none, Pad) end. @@ -516,10 +516,10 @@ prefixed_integer(Int, F, Adj, Base, Pad, Prefix, Lowercase) when -> if Int < 0 -> - S = cond_lowercase(erlang:integer_to_list(-Int, Base), Lowercase), + S = cond_lowercase(integer_to_list(-Int, Base), Lowercase), term([$-, Prefix | S], F, Adj, none, Pad); true -> - S = cond_lowercase(erlang:integer_to_list(Int, Base), Lowercase), + S = cond_lowercase(integer_to_list(Int, Base), Lowercase), term([Prefix | S], F, Adj, none, Pad) end. diff --git a/src/couch_log/test/eunit/couch_log_config_listener_test.erl b/src/couch_log/test/eunit/couch_log_config_listener_test.erl index c955972ff7..2e4a45be32 100644 --- a/src/couch_log/test/eunit/couch_log_config_listener_test.erl +++ b/src/couch_log/test/eunit/couch_log_config_listener_test.erl @@ -29,14 +29,14 @@ check_restart_listener() -> Handler1 = get_handler(), ?assertNotEqual(not_found, Handler1), - Ref = erlang:monitor(process, Listener1), + Ref = monitor(process, Listener1), ok = gen_event:delete_handler(config_event, get_handler(), testing), receive {'DOWN', Ref, process, _, _} -> ?assertNot(is_process_alive(Listener1)) after ?TIMEOUT -> - erlang:error({timeout, config_listener_mon_death}) + error({timeout, config_listener_mon_death}) end, NewHandler = test_util:wait( diff --git a/src/couch_log/test/eunit/couch_log_formatter_test.erl b/src/couch_log/test/eunit/couch_log_formatter_test.erl index cdb7eae312..d09f3b38aa 100644 --- a/src/couch_log/test/eunit/couch_log_formatter_test.erl +++ b/src/couch_log/test/eunit/couch_log_formatter_test.erl @@ -53,7 +53,7 @@ crashing_formatting_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** Generic server and some stuff", @@ -76,7 +76,7 @@ gen_server_error_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** Generic server and some stuff", @@ -102,7 +102,7 @@ gen_server_error_with_extra_args_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** Generic server and some stuff", @@ -128,7 +128,7 @@ gen_fsm_error_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** State machine did a thing", @@ -154,7 +154,7 @@ gen_fsm_error_with_extra_args_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** State machine did a thing", @@ -180,7 +180,7 @@ gen_event_error_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** gen_event handler did a thing", @@ -210,7 +210,7 @@ gen_event_error_test() -> emulator_error_test() -> Event = { error, - erlang:group_leader(), + group_leader(), { emulator, "~s~n", @@ -230,7 +230,7 @@ normal_error_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "format thing: ~w ~w", @@ -253,7 +253,7 @@ error_report_std_error_test() -> Pid = self(), Event = { error_report, - erlang:group_leader(), + group_leader(), { Pid, std_error, @@ -274,7 +274,7 @@ supervisor_report_test() -> % A standard supervisor report Event1 = { error_report, - erlang:group_leader(), + group_leader(), { Pid, supervisor_report, @@ -307,7 +307,7 @@ supervisor_report_test() -> % in the offender blob. Event2 = { error_report, - erlang:group_leader(), + group_leader(), { Pid, supervisor_report, @@ -339,7 +339,7 @@ supervisor_report_test() -> % A supervisor_bridge Event3 = { error_report, - erlang:group_leader(), + group_leader(), { Pid, supervisor_report, @@ -370,7 +370,7 @@ supervisor_report_test() -> % Any other supervisor report Event4 = { error_report, - erlang:group_leader(), + group_leader(), { Pid, supervisor_report, @@ -391,7 +391,7 @@ crash_report_test() -> % A standard crash report Event1 = { error_report, - erlang:group_leader(), + group_leader(), { Pid, crash_report, @@ -425,7 +425,7 @@ crash_report_test() -> % A registered process crash report Event2 = { error_report, - erlang:group_leader(), + group_leader(), { Pid, crash_report, @@ -450,7 +450,7 @@ crash_report_test() -> % A non-exit crash report Event3 = { error_report, - erlang:group_leader(), + group_leader(), { Pid, crash_report, @@ -475,7 +475,7 @@ crash_report_test() -> % A extra report info Event4 = { error_report, - erlang:group_leader(), + group_leader(), { Pid, crash_report, @@ -504,7 +504,7 @@ warning_report_test() -> % A warning message Event1 = { warning_msg, - erlang:group_leader(), + group_leader(), { Pid, "a ~s string ~w", @@ -522,7 +522,7 @@ warning_report_test() -> % A warning report Event2 = { warning_report, - erlang:group_leader(), + group_leader(), { Pid, std_warning, @@ -543,7 +543,7 @@ info_report_test() -> % An info message Event1 = { info_msg, - erlang:group_leader(), + group_leader(), { Pid, "an info ~s string ~w", @@ -561,7 +561,7 @@ info_report_test() -> % Application exit info Event2 = { info_report, - erlang:group_leader(), + group_leader(), { Pid, std_info, @@ -583,7 +583,7 @@ info_report_test() -> % Any other std_info message Event3 = { info_report, - erlang:group_leader(), + group_leader(), { Pid, std_info, @@ -604,7 +604,7 @@ info_report_test() -> % Non-list other report Event4 = { info_report, - erlang:group_leader(), + group_leader(), { Pid, std_info, @@ -625,7 +625,7 @@ progress_report_test() -> % Application started Event1 = { info_report, - erlang:group_leader(), + group_leader(), { Pid, progress, @@ -643,7 +643,7 @@ progress_report_test() -> % Supervisor started child Event2 = { info_report, - erlang:group_leader(), + group_leader(), { Pid, progress, @@ -669,7 +669,7 @@ progress_report_test() -> % Other progress report Event3 = { info_report, - erlang:group_leader(), + group_leader(), { Pid, progress, @@ -787,7 +787,7 @@ format_reason_test_() -> "bad argument in call to j:k(a, 2) at y:x/1" }, { - {{badarity, {fun erlang:spawn/1, [a, b]}}, [{y, x, 1}]}, + {{badarity, {fun spawn/1, [a, b]}}, [{y, x, 1}]}, "function called with wrong arity of 2 instead of 1 at y:x/1" }, { @@ -837,7 +837,7 @@ coverage_test() -> do_format( { error_report, - erlang:group_leader(), + group_leader(), {self(), std_error, "foobar"} } ) @@ -852,7 +852,7 @@ coverage_test() -> do_format( { error_report, - erlang:group_leader(), + group_leader(), {self(), std_error, dang} } ) @@ -862,7 +862,7 @@ gen_server_error_with_last_msg_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** Generic server and some stuff", @@ -890,7 +890,7 @@ gen_event_error_with_last_msg_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** gen_event handler did a thing", @@ -923,7 +923,7 @@ gen_fsm_error_with_last_msg_test() -> Pid = self(), Event = { error, - erlang:group_leader(), + group_leader(), { Pid, "** State machine did a thing", diff --git a/src/couch_log/test/eunit/couch_log_server_test.erl b/src/couch_log/test/eunit/couch_log_server_test.erl index df768f26ea..ff9dbc3f0d 100644 --- a/src/couch_log/test/eunit/couch_log_server_test.erl +++ b/src/couch_log/test/eunit/couch_log_server_test.erl @@ -39,7 +39,7 @@ check_can_reconfigure() -> check_can_restart() -> Pid1 = whereis(couch_log_server), - Ref = erlang:monitor(process, Pid1), + Ref = monitor(process, Pid1), ?assert(is_process_alive(Pid1)), supervisor:terminate_child(couch_log_sup, couch_log_server), @@ -48,7 +48,7 @@ check_can_restart() -> receive {'DOWN', Ref, _, _, _} -> ok after 1000 -> - erlang:error(timeout_restarting_couch_log_server) + error(timeout_restarting_couch_log_server) end, ?assert(not is_process_alive(Pid1)), diff --git a/src/couch_log/test/eunit/couch_log_test_util.erl b/src/couch_log/test/eunit/couch_log_test_util.erl index 9a170bdbdb..743cdf6eff 100644 --- a/src/couch_log/test/eunit/couch_log_test_util.erl +++ b/src/couch_log/test/eunit/couch_log_test_util.erl @@ -64,7 +64,7 @@ wait_for_config() -> receive couch_log_config_change_finished -> ok after 1000 -> - erlang:error(config_change_timeout) + error(config_change_timeout) end. with_meck(Mods, Fun) -> @@ -119,7 +119,7 @@ disable_logs_from(Name) when is_atom(Name) -> P when is_pid(P) -> disable_logs_from(P); undefined -> - erlang:error({unknown_pid_name, Name}) + error({unknown_pid_name, Name}) end. last_log_key() -> diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl index 037c35cdd1..244f668af0 100644 --- a/src/couch_mrview/src/couch_mrview.erl +++ b/src/couch_mrview/src/couch_mrview.erl @@ -62,7 +62,7 @@ validate_ddoc_fields(DDoc) -> [{<<"rewrites">>, [string, array]}], [{<<"shows">>, object}, {any, [object, string]}], [{<<"updates">>, object}, {any, [object, string]}], - [{<<"validate_doc_update">>, string}], + [{<<"validate_doc_update">>, [string, object]}], [{<<"views">>, object}, {<<"lib">>, object}], [{<<"views">>, object}, {any, object}, {<<"map">>, MapFuncType}], [{<<"views">>, object}, {any, object}, {<<"reduce">>, string}] @@ -183,7 +183,7 @@ map_function_type({Props}) -> end. format_type(Type) when is_atom(Type) -> - ?l2b(atom_to_list(Type)); + atom_to_binary(Type); format_type(Types) when is_list(Types) -> iolist_to_binary(join(lists:map(fun atom_to_list/1, Types), <<" or ">>)). @@ -287,12 +287,22 @@ query_all_docs(Db, Args0, Callback, Acc) -> couch_index_util:hexsig(couch_hash:md5_hash(?term_to_bin(Info))) end), Args1 = Args0#mrargs{view_type = map}, + + % TODO: Compatibility clause. Remove after upgrading to next minor release + % after 3.5.0. + % + % As of commit 7aa8a4, args are validated in fabric. However, during online + % cluster upgrades, old nodes will still expect args to be validated on + % workers, so keep the clause around until the next minor version then + % remove. + % + Args2 = couch_mrview_util:validate_all_docs_args(Db, Args1), {ok, Acc1} = - case Args1#mrargs.preflight_fun of + case Args2#mrargs.preflight_fun of PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc); _ -> {ok, Acc} end, - all_docs_fold(Db, Args1, Callback, Acc1). + all_docs_fold(Db, Args2, Callback, Acc1). query_view(Db, DDoc, VName) -> Args = #mrargs{extra = [{view_row_map, true}]}, @@ -328,7 +338,7 @@ query_view(Db, {Type, View, Ref}, Args, Callback, Acc) -> red -> red_fold(Db, View, Args, Callback, Acc) end after - erlang:demonitor(Ref, [flush]) + demonitor(Ref, [flush]) end. get_info(Db, DDoc) -> diff --git a/src/couch_mrview/src/couch_mrview_cleanup.erl b/src/couch_mrview/src/couch_mrview_cleanup.erl index 5b5afbdce0..e8a2833a7c 100644 --- a/src/couch_mrview/src/couch_mrview_cleanup.erl +++ b/src/couch_mrview/src/couch_mrview_cleanup.erl @@ -14,12 +14,9 @@ -export([ run/1, - cleanup_purges/3, - cleanup_indices/2 + cleanup/2 ]). --include_lib("couch/include/couch_db.hrl"). - run(Db) -> Indices = couch_mrview_util:get_index_files(Db), Checkpoints = couch_mrview_util:get_purge_checkpoints(Db), @@ -28,15 +25,26 @@ run(Db) -> ok = cleanup_purges(Db1, Sigs, Checkpoints), ok = cleanup_indices(Sigs, Indices). -cleanup_purges(DbName, Sigs, Checkpoints) when is_binary(DbName) -> - couch_util:with_db(DbName, fun(Db) -> - cleanup_purges(Db, Sigs, Checkpoints) - end); -cleanup_purges(Db, #{} = Sigs, #{} = CheckpointsMap) -> - InactiveMap = maps:without(maps:keys(Sigs), CheckpointsMap), - InactiveCheckpoints = maps:values(InactiveMap), - DeleteFun = fun(DocId) -> delete_checkpoint(Db, DocId) end, - lists:foreach(DeleteFun, InactiveCheckpoints). +% erpc endpoint for fabric_index_cleanup:cleanup_indexes/2 +% +cleanup(Dbs, #{} = Sigs) -> + try + lists:foreach( + fun(Db) -> + Indices = couch_mrview_util:get_index_files(Db), + Checkpoints = couch_mrview_util:get_purge_checkpoints(Db), + ok = cleanup_purges(Db, Sigs, Checkpoints), + ok = cleanup_indices(Sigs, Indices) + end, + Dbs + ) + catch + error:database_does_not_exist -> + ok + end. + +cleanup_purges(Db, Sigs, Checkpoints) -> + couch_index_util:cleanup_purges(Db, Sigs, Checkpoints). cleanup_indices(#{} = Sigs, #{} = IndexMap) -> Fun = fun(_, Files) -> lists:foreach(fun delete_file/1, Files) end, @@ -54,20 +62,3 @@ delete_file(File) -> couch_log:error(ErrLog, [?MODULE, File, Tag, Error]), ok end. - -delete_checkpoint(Db, DocId) -> - DbName = couch_db:name(Db), - LogMsg = "~p : deleting inactive purge checkpoint ~s : ~s", - couch_log:debug(LogMsg, [?MODULE, DbName, DocId]), - try couch_db:open_doc(Db, DocId, []) of - {ok, Doc = #doc{}} -> - Deleted = Doc#doc{deleted = true, body = {[]}}, - couch_db:update_doc(Db, Deleted, [?ADMIN_CTX]); - {not_found, _} -> - ok - catch - Tag:Error -> - ErrLog = "~p : error deleting checkpoint ~s : ~s error: ~p:~p", - couch_log:error(ErrLog, [?MODULE, DbName, DocId, Tag, Error]), - ok - end. diff --git a/src/couch_mrview/src/couch_mrview_compactor.erl b/src/couch_mrview/src/couch_mrview_compactor.erl index 28e5a9b3da..a534dd0e8a 100644 --- a/src/couch_mrview/src/couch_mrview_compactor.erl +++ b/src/couch_mrview/src/couch_mrview_compactor.erl @@ -136,11 +136,11 @@ recompact(State) -> recompact(State, recompact_retry_count()). recompact(#mrst{db_name = DbName, idx_name = IdxName}, 0) -> - erlang:error({exceeded_recompact_retry_count, [{db_name, DbName}, {idx_name, IdxName}]}); + error({exceeded_recompact_retry_count, [{db_name, DbName}, {idx_name, IdxName}]}); recompact(State, RetryCount) -> Self = self(), link(State#mrst.fd), - {Pid, Ref} = erlang:spawn_monitor(fun() -> + {Pid, Ref} = spawn_monitor(fun() -> couch_index_updater:update(Self, couch_mrview_index, State) end), recompact_loop(Pid, Ref, State, RetryCount). @@ -240,7 +240,7 @@ swap_compacted(OldState, NewState) -> } = NewState, link(NewState#mrst.fd), - Ref = erlang:monitor(process, NewState#mrst.fd), + Ref = monitor(process, NewState#mrst.fd), RootDir = couch_index_util:root_dir(), IndexFName = couch_mrview_util:index_file(DbName, Sig), @@ -257,7 +257,7 @@ swap_compacted(OldState, NewState) -> ok = file:rename(CompactFName, IndexFName), unlink(OldState#mrst.fd), - erlang:demonitor(OldState#mrst.fd_monitor, [flush]), + demonitor(OldState#mrst.fd_monitor, [flush]), {ok, NewState#mrst{fd_monitor = Ref}}. diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl index 51777480cd..a5080ed763 100644 --- a/src/couch_mrview/src/couch_mrview_index.erl +++ b/src/couch_mrview/src/couch_mrview_index.erl @@ -167,7 +167,7 @@ open(Db, State0) -> end. close(State) -> - erlang:demonitor(State#mrst.fd_monitor, [flush]), + demonitor(State#mrst.fd_monitor, [flush]), couch_file:close(State#mrst.fd). % This called after ddoc_updated event occurrs, and @@ -178,7 +178,7 @@ close(State) -> % couch_file will be closed automatically after all % outstanding queries are done. shutdown(State) -> - erlang:demonitor(State#mrst.fd_monitor, [flush]), + demonitor(State#mrst.fd_monitor, [flush]), unlink(State#mrst.fd). delete(#mrst{db_name = DbName, sig = Sig} = State) -> diff --git a/src/couch_mrview/src/couch_mrview_test_util.erl b/src/couch_mrview/src/couch_mrview_test_util.erl index 349611191b..f021ce58d9 100644 --- a/src/couch_mrview/src/couch_mrview_test_util.erl +++ b/src/couch_mrview/src/couch_mrview_test_util.erl @@ -139,7 +139,7 @@ ddoc(Id) -> doc(Id) -> couch_doc:from_json_obj( {[ - {<<"_id">>, list_to_binary(integer_to_list(Id))}, + {<<"_id">>, integer_to_binary(Id)}, {<<"val">>, Id} ]} ). diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl index 92e66d0fa3..4238e6b7db 100644 --- a/src/couch_mrview/src/couch_mrview_updater.erl +++ b/src/couch_mrview/src/couch_mrview_updater.erl @@ -166,13 +166,13 @@ map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) -> QServer = State1#mrst.qserver, DocFun = fun ({nil, Seq, _}, {SeqAcc, Results}) -> - {erlang:max(Seq, SeqAcc), Results}; + {max(Seq, SeqAcc), Results}; ({Id, Seq, deleted}, {SeqAcc, Results}) -> - {erlang:max(Seq, SeqAcc), [{Id, []} | Results]}; + {max(Seq, SeqAcc), [{Id, []} | Results]}; ({Id, Seq, Doc}, {SeqAcc, Results}) -> couch_stats:increment_counter([couchdb, mrview, map_doc]), {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc), - {erlang:max(Seq, SeqAcc), [{Id, Res} | Results]} + {max(Seq, SeqAcc), [{Id, Res} | Results]} end, FoldFun = fun(Docs, Acc) -> update_task(length(Docs)), @@ -242,7 +242,7 @@ merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys) -> merge_results(RawResults, VKV, DIK) end, {ViewKVs1, DocIdKeys1} = lists:foldl(Fun, {ViewKVs, DocIdKeys}, Results), - merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1). + merge_results(Rest, max(Seq, SeqAcc), ViewKVs1, DocIdKeys1). merge_results({DocId, []}, ViewKVs, DocIdKeys) -> {ViewKVs, [{DocId, []} | DocIdKeys]}; diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index a5c81a0736..5405e8db82 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -16,6 +16,7 @@ -export([get_local_purge_doc_id/1, get_value_from_options/2]). -export([verify_view_filename/1, get_signature_from_filename/1]). -export([get_signatures/1, get_purge_checkpoints/1, get_index_files/1]). +-export([get_signatures_from_ddocs/2]). -export([ddoc_to_mrst/2, init_state/4, reset_index/3]). -export([make_header/1]). -export([index_file/2, compaction_file/2, open_file/1]). @@ -94,40 +95,35 @@ get_signatures(DbName) when is_binary(DbName) -> couch_util:with_db(DbName, fun get_signatures/1); get_signatures(Db) -> DbName = couch_db:name(Db), - % get_design_docs/1 returns ejson for clustered shards, and - % #full_doc_info{}'s for other cases. {ok, DDocs} = couch_db:get_design_docs(Db), + % get_design_docs/1 returns ejson for clustered shards, and + % #full_doc_info{}'s for other cases. Both are transformed to #doc{} records FoldFun = fun ({[_ | _]} = EJsonDoc, Acc) -> Doc = couch_doc:from_json_obj(EJsonDoc), - {ok, Mrst} = ddoc_to_mrst(DbName, Doc), - Sig = couch_util:to_hex_bin(Mrst#mrst.sig), - Acc#{Sig => true}; + [Doc | Acc]; (#full_doc_info{} = FDI, Acc) -> {ok, Doc} = couch_db:open_doc_int(Db, FDI, [ejson_body]), - {ok, Mrst} = ddoc_to_mrst(DbName, Doc), - Sig = couch_util:to_hex_bin(Mrst#mrst.sig), - Acc#{Sig => true} + [Doc | Acc] + end, + DDocs1 = lists:foldl(FoldFun, [], DDocs), + get_signatures_from_ddocs(DbName, DDocs1). + +% From a list of design #doc{} records returns signatures map: #{Sig => true} +% +get_signatures_from_ddocs(DbName, DDocs) when is_list(DDocs) -> + FoldFun = fun(#doc{} = Doc, Acc) -> + {ok, Mrst} = ddoc_to_mrst(DbName, Doc), + Sig = couch_util:to_hex_bin(Mrst#mrst.sig), + Acc#{Sig => true} end, lists:foldl(FoldFun, #{}, DDocs). % Returns a map of `Sig => DocId` elements for all the purge view % checkpoint docs. Sig is a hex-encoded binary. % -get_purge_checkpoints(DbName) when is_binary(DbName) -> - couch_util:with_db(DbName, fun get_purge_checkpoints/1); get_purge_checkpoints(Db) -> - FoldFun = fun(#doc{id = Id}, Acc) -> - case Id of - <> -> - {ok, Acc#{Sig => Id}}; - _ -> - {stop, Acc} - end - end, - Opts = [{start_key, <>}], - {ok, Signatures = #{}} = couch_db:fold_local_docs(Db, FoldFun, #{}, Opts), - Signatures. + couch_index_util:get_purge_checkpoints(Db, <<"mrview">>). % Returns a map of `Sig => [FilePath, ...]` elements. Sig is a hex-encoded % binary and FilePaths are lists as they intended to be passed to couch_file @@ -152,7 +148,7 @@ get_index_files(Db) -> get_view(Db, DDoc, ViewName, Args0) -> case get_view_index_state(Db, DDoc, ViewName, Args0) of {ok, State, Args2} -> - Ref = erlang:monitor(process, State#mrst.fd), + Ref = monitor(process, State#mrst.fd), #mrst{language = Lang, views = Views} = State, {Type, View, Args3} = extract_view(Lang, Args2, ViewName, Views), check_range(Args3, view_cmp(View)), @@ -382,7 +378,7 @@ init_state(Db, Fd, State, Header) -> {ShouldCommit, State#mrst{ fd = Fd, - fd_monitor = erlang:monitor(process, Fd), + fd_monitor = monitor(process, Fd), update_seq = Seq, purge_seq = PurgeSeq, id_btree = IdBtree, @@ -544,7 +540,13 @@ apply_limit(ViewPartitioned, Args) -> mrverror(io_lib:format(Fmt, [MaxLimit])) end. -validate_all_docs_args(Db, Args0) -> +validate_all_docs_args(Db, #mrargs{} = Args) -> + case get_extra(Args, validated, false) of + true -> Args; + false -> validate_all_docs_args_int(Db, Args) + end. + +validate_all_docs_args_int(Db, Args0) -> Args = validate_args(Args0#mrargs{view_type = map}), DbPartitioned = couch_db:is_partitioned(Db), @@ -560,7 +562,13 @@ validate_all_docs_args(Db, Args0) -> apply_limit(false, Args) end. -validate_args(Args) -> +validate_args(#mrargs{} = Args) -> + case get_extra(Args, validated, false) of + true -> Args; + false -> validate_args_int(Args) + end. + +validate_args_int(#mrargs{} = Args) -> GroupLevel = determine_group_level(Args), Reduce = Args#mrargs.reduce, case Reduce == undefined orelse is_boolean(Reduce) of @@ -696,11 +704,12 @@ validate_args(Args) -> _ -> mrverror(<<"Invalid value for `partition`.">>) end, - Args#mrargs{ + Args1 = Args#mrargs{ start_key_docid = SKDocId, end_key_docid = EKDocId, group_level = GroupLevel - }. + }, + set_extra(Args1, validated, true). determine_group_level(#mrargs{group = undefined, group_level = undefined}) -> 0; @@ -1304,7 +1313,8 @@ make_user_reds_reduce_fun(Lang, ReduceFuns, NthRed) -> get_btree_state(nil) -> nil; -get_btree_state(#btree{} = Btree) -> +get_btree_state(Btree) -> + true = couch_btree:is_btree(Btree), couch_btree:get_state(Btree). extract_view_reduce({red, {N, _Lang, #mrview{reduce_funs = Reds}}, _Ref}) -> diff --git a/src/couch_mrview/test/eunit/couch_mrview_collation_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_collation_tests.erl index c00b97b33c..87d4814260 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_collation_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_collation_tests.erl @@ -137,7 +137,7 @@ find_matching_rows(Index, Value) -> ), lists:map( fun({Id, V}) -> - {row, [{id, list_to_binary(integer_to_list(Id))}, {key, V}, {value, 0}]} + {row, [{id, integer_to_binary(Id)}, {key, V}, {value, 0}]} end, Matches ). @@ -206,7 +206,7 @@ make_docs() -> fun(V, {Docs0, Count}) -> Doc = couch_doc:from_json_obj( {[ - {<<"_id">>, list_to_binary(integer_to_list(Count))}, + {<<"_id">>, integer_to_binary(Count)}, {<<"foo">>, V} ]} ), @@ -220,7 +220,7 @@ make_docs() -> rows() -> {Rows, _} = lists:foldl( fun(V, {Rows0, Count}) -> - Id = list_to_binary(integer_to_list(Count)), + Id = integer_to_binary(Count), Row = {row, [{id, Id}, {key, V}, {value, 0}]}, {[Row | Rows0], Count + 1} end, diff --git a/src/couch_mrview/test/eunit/couch_mrview_compact_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_compact_tests.erl index df035c649a..048f6bc701 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_compact_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_compact_tests.erl @@ -55,7 +55,7 @@ should_swap(Db) -> receive {'DOWN', MonRef, process, _, _} -> ok after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -68,7 +68,7 @@ should_swap(Db) -> {QPid, Count} -> ?assertEqual(1000, Count) after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -86,7 +86,7 @@ should_remove(Db) -> ok = couch_index:compact(IndexPid, []), {ok, CompactorPid} = couch_index:get_compactor_pid(IndexPid), {ok, CompactingPid} = couch_index_compactor:get_compacting_pid(CompactorPid), - MonRef = erlang:monitor(process, CompactingPid), + MonRef = monitor(process, CompactingPid), exit(CompactingPid, crash), receive {'DOWN', MonRef, process, _, crash} -> @@ -100,7 +100,7 @@ should_remove(Db) -> ?assert(is_process_alive(IndexPid)), ?assert(is_process_alive(CompactorPid)) after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch_mrview/test/eunit/couch_mrview_ddoc_updated_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_ddoc_updated_tests.erl index 91b24e336a..756434892a 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_ddoc_updated_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_ddoc_updated_tests.erl @@ -90,7 +90,7 @@ check_indexing_stops_on_ddoc_change(Db) -> IndexesBefore = get_indexes_by_ddoc(couch_db:name(Db), DDocID, 1), ?assertEqual(1, length(IndexesBefore)), - AliveBefore = lists:filter(fun erlang:is_process_alive/1, IndexesBefore), + AliveBefore = lists:filter(fun is_process_alive/1, IndexesBefore), ?assertEqual(1, length(AliveBefore)), {ok, DDoc} = couch_db:open_doc(Db, DDocID, [ejson_body, ?ADMIN_CTX]), @@ -117,7 +117,7 @@ check_indexing_stops_on_ddoc_change(Db) -> {QPid, Msg} -> ?assertEqual(Msg, ddoc_updated) after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -129,7 +129,7 @@ check_indexing_stops_on_ddoc_change(Db) -> %% assert that previously running indexes are gone IndexesAfter = get_indexes_by_ddoc(couch_db:name(Db), DDocID, 0), ?assertEqual(0, length(IndexesAfter)), - AliveAfter = lists:filter(fun erlang:is_process_alive/1, IndexesBefore), + AliveAfter = lists:filter(fun is_process_alive/1, IndexesBefore), ?assertEqual(0, length(AliveAfter)) end). diff --git a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl index 3207a3da3d..f3452b55a2 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_fabric_tests.erl @@ -288,7 +288,7 @@ wait_compaction(DbName, Line) -> end, case test_util:wait(WaitFun, 10000) of timeout -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, Line}, diff --git a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl index 44500bdf1b..c4ac9ed139 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl @@ -299,7 +299,7 @@ test_purge_index_reset(Db) -> PurgeInfos = lists:map( fun(I) -> - DocId = list_to_binary(integer_to_list(I)), + DocId = integer_to_binary(I), FDI = couch_db:get_full_doc_info(Db, DocId), Rev = get_rev(FDI), {couch_uuids:random(), DocId, [Rev]} @@ -576,7 +576,7 @@ wait_compaction(DbName, Kind, Line) -> end, case test_util:wait(WaitFun, 10000) of timeout -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, Line}, @@ -600,7 +600,7 @@ fold_fun({_PSeq, _UUID, Id, Revs}, Acc) -> {ok, [{Id, Revs} | Acc]}. docid(I) -> - list_to_binary(integer_to_list(I)). + integer_to_binary(I). uuid(I) -> Str = io_lib:format("UUID~4..0b", [I]), diff --git a/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl index 2562bb511e..c304dcdad3 100644 --- a/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl +++ b/src/couch_mrview/test/eunit/couch_mrview_util_tests.erl @@ -174,3 +174,31 @@ t_get_index_files_clustered({DbName, _Db}) -> ?assertMatch({ok, _}, file:read_file_info(File)), {ok, Info} = couch_mrview:get_info(ShardName1, ?DDOC_ID), ?assertEqual(proplists:get_value(signature, Info), Sig). + +do_not_validate_args_if_already_validated_test() -> + Args = #mrargs{ + view_type = red, + group = true, + group_level = undefined, + extra = [{foo, bar}] + }, + + % Initially if we haven't validated, it's not flagged as such + ?assertNot(couch_mrview_util:get_extra(Args, validated, false)), + + % Do the validation + Args1 = couch_mrview_util:validate_args(Args), + + % Validation worked + ?assertEqual(exact, Args1#mrargs.group_level), + + % Validation flag is set to true + ?assert(couch_mrview_util:get_extra(Args1, validated, false)), + + Args2 = couch_mrview_util:validate_args(Args1), + % No change, as already validated + ?assertEqual(Args1, Args2), + + Args3 = couch_mrview_util:validate_all_docs_args(some_db, Args2), + % No change for all docs validation as already validated + ?assertEqual(Args1, Args3). diff --git a/src/couch_peruser/src/couch_peruser.erl b/src/couch_peruser/src/couch_peruser.erl index 6f4a24cab2..8a7cbe13a8 100644 --- a/src/couch_peruser/src/couch_peruser.erl +++ b/src/couch_peruser/src/couch_peruser.erl @@ -269,7 +269,7 @@ should_handle_doc(ShardName, DocId) -> ) -> boolean(). should_handle_doc_int(ShardName, DocId) -> DbName = mem3:dbname(ShardName), - Live = [erlang:node() | erlang:nodes()], + Live = [erlang:node() | nodes()], Shards = mem3:shards(DbName, DocId), Nodes = [N || #shard{node = N} <- Shards, lists:member(N, Live)], case mem3:owner(DbName, DocId, Nodes) of diff --git a/src/couch_prometheus/src/couch_prometheus.erl b/src/couch_prometheus/src/couch_prometheus.erl index 4d053d1af7..7f3dc494d1 100644 --- a/src/couch_prometheus/src/couch_prometheus.erl +++ b/src/couch_prometheus/src/couch_prometheus.erl @@ -68,7 +68,8 @@ get_system_stats() -> get_internal_replication_jobs_stat(), get_membership_stat(), get_membership_nodes(), - get_distribution_stats() + get_distribution_stats(), + get_bt_engine_cache_stats() ]). get_uptime_stat() -> @@ -119,9 +120,9 @@ get_vm_stats() -> end, erlang:memory() ), - {NumGCs, WordsReclaimed, _} = erlang:statistics(garbage_collection), - CtxSwitches = element(1, erlang:statistics(context_switches)), - Reds = element(1, erlang:statistics(reductions)), + {NumGCs, WordsReclaimed, _} = statistics(garbage_collection), + CtxSwitches = element(1, statistics(context_switches)), + Reds = element(1, statistics(reductions)), ProcCount = erlang:system_info(process_count), ProcLimit = erlang:system_info(process_limit), [ @@ -157,7 +158,7 @@ get_vm_stats() -> ]. get_io_stats() -> - {{input, In}, {output, Out}} = erlang:statistics(io), + {{input, In}, {output, Out}} = statistics(io), [ to_prom( erlang_io_recv_bytes_total, @@ -374,3 +375,12 @@ get_distribution_stats() -> get_ets_stats() -> NumTabs = length(ets:all()), to_prom(erlang_ets_table, gauge, "number of ETS tables", NumTabs). + +get_bt_engine_cache_stats() -> + Stats = couch_bt_engine_cache:info(), + Size = maps:get(size, Stats, 0), + Mem = maps:get(memory, Stats, 0), + [ + to_prom(couchdb_bt_engine_cache_memory, gauge, "memory used by the btree cache", Mem), + to_prom(couchdb_bt_engine_cache_size, gauge, "number of entries in the btree cache", Size) + ]. diff --git a/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl b/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl index c33b379a84..3838717214 100644 --- a/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl +++ b/src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl @@ -114,9 +114,9 @@ t_no_duplicate_metrics(Port) -> % definition, not the values. These lines always start with % a # character. MetricDefs = lists:filter(fun(S) -> string:find(S, "#") =:= S end, Lines), - ?assertNotEqual(erlang:length(MetricDefs), 0), + ?assertNotEqual(length(MetricDefs), 0), Diff = get_duplicates(MetricDefs), - ?assertEqual(erlang:length(Diff), 0). + ?assertEqual(length(Diff), 0). get_duplicates(List) -> List -- sets:to_list(couch_util:set_from_list(List)). @@ -158,7 +158,7 @@ t_starts_with_couchdb(Port) -> {match, _} -> ok; nomatch -> - erlang:error( + error( {assertRegexp_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch_pse_tests/src/cpse_test_ref_counting.erl b/src/couch_pse_tests/src/cpse_test_ref_counting.erl index e563210807..401353f37f 100644 --- a/src/couch_pse_tests/src/cpse_test_ref_counting.erl +++ b/src/couch_pse_tests/src/cpse_test_ref_counting.erl @@ -82,7 +82,7 @@ start_client(Db0) -> {waiting, Pid} -> Pid ! go after 1000 -> - erlang:error(timeout) + error(timeout) end, receive @@ -90,7 +90,7 @@ start_client(Db0) -> couch_db:close(Db1), ok after 1000 -> - erlang:error(timeout) + error(timeout) end end). @@ -99,7 +99,7 @@ wait_client({Pid, _Ref}) -> receive go -> ok after 1000 -> - erlang:error(timeout) + error(timeout) end. close_client({Pid, Ref}) -> @@ -108,5 +108,5 @@ close_client({Pid, Ref}) -> {'DOWN', Ref, _, _, _} -> ok after 1000 -> - erlang:error(timeout) + error(timeout) end. diff --git a/src/couch_pse_tests/src/cpse_util.erl b/src/couch_pse_tests/src/cpse_util.erl index c5aac94c0e..bd17de26bc 100644 --- a/src/couch_pse_tests/src/cpse_util.erl +++ b/src/couch_pse_tests/src/cpse_util.erl @@ -95,13 +95,13 @@ open_db(DbName) -> shutdown_db(Db) -> Pid = couch_db:get_pid(Db), - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), exit(Pid, kill), receive {'DOWN', Ref, _, _, _} -> ok after ?SHUTDOWN_TIMEOUT -> - erlang:error(database_shutdown_timeout) + error(database_shutdown_timeout) end, test_util:wait(fun() -> case @@ -189,7 +189,7 @@ assert_db_props(Module, Line, DbName, Props) when is_binary(DbName) -> catch error:{assertEqual, Props} -> {_, Rest} = proplists:split(Props, [module, line]), - erlang:error({assertEqual, [{module, Module}, {line, Line} | Rest]}) + error({assertEqual, [{module, Module}, {line, Line} | Rest]}) after couch_db:close(Db) end; @@ -199,7 +199,7 @@ assert_db_props(Module, Line, Db, Props) -> catch error:{assertEqual, Props} -> {_, Rest} = proplists:split(Props, [module, line]), - erlang:error({assertEqual, [{module, Module}, {line, Line} | Rest]}) + error({assertEqual, [{module, Module}, {line, Line} | Rest]}) end. assert_each_prop(_Db, []) -> @@ -309,7 +309,7 @@ gen_write(Db, {Action, {<<"_local/", _/binary>> = DocId, Body}}) -> end, {local, #doc{ id = DocId, - revs = {0, [list_to_binary(integer_to_list(RevId))]}, + revs = {0, [integer_to_binary(RevId)]}, body = Body, deleted = Deleted }}; @@ -387,7 +387,7 @@ prep_atts(Db, [{FileName, Data} | Rest]) -> {'DOWN', Ref, _, _, Resp} -> Resp after ?ATTACHMENT_WRITE_TIMEOUT -> - erlang:error(attachment_write_timeout) + error(attachment_write_timeout) end, [Att | prep_atts(Db, Rest)]. @@ -617,7 +617,7 @@ list_diff([T1 | R1], [T2 | R2]) -> compact(Db) -> {ok, Pid} = couch_db:start_compact(Db), - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), % Ideally I'd assert that Pid is linked to us % at this point but its technically possible @@ -630,9 +630,9 @@ compact(Db) -> {'DOWN', Ref, _, _, noproc} -> ok; {'DOWN', Ref, _, _, Reason} -> - erlang:error({compactor_died, Reason}) + error({compactor_died, Reason}) after ?COMPACTOR_TIMEOUT -> - erlang:error(compactor_timed_out) + error(compactor_timed_out) end, test_util:wait(fun() -> diff --git a/src/couch_quickjs/.gitignore b/src/couch_quickjs/.gitignore index dd78afee12..f7d339351f 100644 --- a/src/couch_quickjs/.gitignore +++ b/src/couch_quickjs/.gitignore @@ -16,6 +16,8 @@ /quickjs/qjscalc.c /quickjs/repl.c /quickjs/run-test262 +/quickjs/test262_report.txt +/quickjs/test262/ /quickjs/test_fib.c /quickjs/.github compile_commands.json diff --git a/src/couch_quickjs/c_src/couchjs.c b/src/couch_quickjs/c_src/couchjs.c index 7958acda59..420f5f41d1 100644 --- a/src/couch_quickjs/c_src/couchjs.c +++ b/src/couch_quickjs/c_src/couchjs.c @@ -86,14 +86,20 @@ static CmdType parse_command(char* str, size_t len) { return CMD_VIEW; } -static void add_cx_methods(JSContext* cx) { +// Return true if initializations succeed and false if any fails. Failure +// will shortcut the sequence and will return early. The only thing to do +// then is to free the context and return early +// +static bool add_cx_methods(JSContext* cx) { //TODO: configure some features with env vars of command line switches - JS_AddIntrinsicBaseObjects(cx); - JS_AddIntrinsicEval(cx); - JS_AddIntrinsicJSON(cx); - JS_AddIntrinsicRegExp(cx); - JS_AddIntrinsicMapSet(cx); - JS_AddIntrinsicDate(cx); + return ! ( + JS_AddIntrinsicBaseObjects(cx) || + JS_AddIntrinsicEval(cx) || + JS_AddIntrinsicJSON(cx) || + JS_AddIntrinsicRegExp(cx) || + JS_AddIntrinsicMapSet(cx) || + JS_AddIntrinsicDate(cx) + ); } // Creates a new JSContext with only the provided sandbox function @@ -104,7 +110,12 @@ static JSContext* make_sandbox(JSContext* cx, JSValue sbox) { if(!cx1) { return NULL; } - add_cx_methods(cx1); + + if(!add_cx_methods(cx1)) { + JS_FreeContext(cx1); + return NULL; + } + JSValue global = JS_GetGlobalObject(cx1); int i; diff --git a/src/couch_quickjs/patches/01-spidermonkey-185-mode.patch b/src/couch_quickjs/patches/01-spidermonkey-185-mode.patch index 6b65fe4f6a..f50c02bf9a 100644 --- a/src/couch_quickjs/patches/01-spidermonkey-185-mode.patch +++ b/src/couch_quickjs/patches/01-spidermonkey-185-mode.patch @@ -1,6 +1,6 @@ ---- quickjs-master/quickjs.c 2025-04-30 08:12:10 -+++ quickjs/quickjs.c 2025-04-30 13:50:44 -@@ -30110,10 +30110,24 @@ +--- quickjs-master/quickjs.c 2025-11-05 05:46:20 ++++ quickjs/quickjs.c 2025-11-05 09:54:50 +@@ -31286,10 +31286,24 @@ if (s->token.val == TOK_FUNCTION || (token_is_pseudo_keyword(s, JS_ATOM_async) && peek_token(s, TRUE) == TOK_FUNCTION)) { diff --git a/src/couch_quickjs/patches/02-test262-errors.patch b/src/couch_quickjs/patches/02-test262-errors.patch new file mode 100644 index 0000000000..267bcb6c8b --- /dev/null +++ b/src/couch_quickjs/patches/02-test262-errors.patch @@ -0,0 +1,11 @@ +--- quickjs-master/test262_errors.txt 2025-11-05 05:46:20 ++++ quickjs/test262_errors.txt 2025-11-05 09:54:50 +@@ -19,6 +19,8 @@ + test262/test/language/expressions/compound-assignment/S11.13.2_A6.10_T1.js:24: Test262Error: #1: innerX === 2. Actual: 5 + test262/test/language/expressions/compound-assignment/S11.13.2_A6.11_T1.js:24: Test262Error: #1: innerX === 2. Actual: 5 + test262/test/language/identifier-resolution/assign-to-global-undefined.js:20: strict mode: expected error ++test262/test/language/statements/expression/S12.4_A1.js:15: unexpected error type: Test262: This statement should not be evaluated. ++test262/test/language/statements/expression/S12.4_A1.js:15: strict mode: unexpected error type: Test262: This statement should not be evaluated. + test262/test/staging/sm/Function/arguments-parameter-shadowing.js:14: Test262Error: Expected SameValue(«true», «false») to be true + test262/test/staging/sm/Function/constructor-binding.js:11: Test262Error: Expected SameValue(«"function"», «"undefined"») to be true + test262/test/staging/sm/Function/constructor-binding.js:11: strict mode: Test262Error: Expected SameValue(«"function"», «"undefined"») to be true diff --git a/src/couch_quickjs/quickjs/Changelog b/src/couch_quickjs/quickjs/Changelog index 7cc33993d9..7d6afd6da1 100644 --- a/src/couch_quickjs/quickjs/Changelog +++ b/src/couch_quickjs/quickjs/Changelog @@ -1,3 +1,31 @@ +- micro optimizations (15% faster on bench-v8) +- added resizable array buffers +- added ArrayBuffer.prototype.transfer +- added the Iterator object and methods +- added set methods +- added Atomics.pause +- added added Map and WeakMap upsert methods +- added Math.sumPrecise() +- misc bug fixes + +2025-09-13: + +- added JSON modules and import attributes +- added JS_PrintValue() API +- qjs: pretty print objects in print() and console.log() +- qjs: better promise rejection tracker heuristics +- added RegExp v flag +- added RegExp modifiers +- added RegExp.escape +- added Float16Array +- added Promise.try +- improved JSON parser spec conformance +- qjs: improved compatibility of std.parseExtJSON() with JSON5 and + accept JSON5 modules +- added JS_FreePropertyEnum() and JS_AtomToCStringLen() API +- added Error.isError() +- misc bug fixes + 2025-04-26: - removed the bignum extensions and qjscalc diff --git a/src/couch_quickjs/quickjs/Makefile b/src/couch_quickjs/quickjs/Makefile index 3b1c745947..ba64923d2b 100644 --- a/src/couch_quickjs/quickjs/Makefile +++ b/src/couch_quickjs/quickjs/Makefile @@ -54,6 +54,10 @@ PREFIX?=/usr/local # use UB sanitizer #CONFIG_UBSAN=y +# TEST262 bootstrap config: commit id and shallow "since" parameter +TEST262_COMMIT?=42303c7c2bcf1c1edb9e5375c291c6fbc8a261ab +TEST262_SINCE?=2025-09-01 + OBJDIR=.obj ifdef CONFIG_ASAN @@ -464,6 +468,15 @@ stats: qjs$(EXE) microbench: qjs$(EXE) $(WINE) ./qjs$(EXE) --std tests/microbench.js +ifeq ($(wildcard test262/features.txt),) +test2-bootstrap: + git clone --single-branch --shallow-since=$(TEST262_SINCE) https://github.com/tc39/test262.git + (cd test262 && git checkout -q $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..) +else +test2-bootstrap: + (cd test262 && git fetch && git reset --hard $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..) +endif + ifeq ($(wildcard test262o/tests.txt),) test2o test2o-update: @echo test262o tests not installed diff --git a/src/couch_quickjs/quickjs/VERSION b/src/couch_quickjs/quickjs/VERSION index c76e76d1f1..433b8f85bd 100644 --- a/src/couch_quickjs/quickjs/VERSION +++ b/src/couch_quickjs/quickjs/VERSION @@ -1 +1 @@ -2025-04-26 +2025-09-13 diff --git a/src/couch_quickjs/quickjs/cutils.c b/src/couch_quickjs/quickjs/cutils.c index c038cf44ca..52ff1649bd 100644 --- a/src/couch_quickjs/quickjs/cutils.c +++ b/src/couch_quickjs/quickjs/cutils.c @@ -100,15 +100,20 @@ void dbuf_init(DynBuf *s) dbuf_init2(s, NULL, NULL); } -/* return < 0 if error */ -int dbuf_realloc(DynBuf *s, size_t new_size) +/* Try to allocate 'len' more bytes. return < 0 if error */ +int dbuf_claim(DynBuf *s, size_t len) { - size_t size; + size_t new_size, size; uint8_t *new_buf; + new_size = s->size + len; + if (new_size < len) + return -1; /* overflow case */ if (new_size > s->allocated_size) { if (s->error) return -1; - size = s->allocated_size * 3 / 2; + size = s->allocated_size + (s->allocated_size / 2); + if (size < s->allocated_size) + return -1; /* overflow case */ if (size > new_size) new_size = size; new_buf = s->realloc_func(s->opaque, s->buf, new_size); @@ -122,22 +127,10 @@ int dbuf_realloc(DynBuf *s, size_t new_size) return 0; } -int dbuf_write(DynBuf *s, size_t offset, const uint8_t *data, size_t len) -{ - size_t end; - end = offset + len; - if (dbuf_realloc(s, end)) - return -1; - memcpy(s->buf + offset, data, len); - if (end > s->size) - s->size = end; - return 0; -} - int dbuf_put(DynBuf *s, const uint8_t *data, size_t len) { - if (unlikely((s->size + len) > s->allocated_size)) { - if (dbuf_realloc(s, s->size + len)) + if (unlikely((s->allocated_size - s->size) < len)) { + if (dbuf_claim(s, len)) return -1; } memcpy_no_ub(s->buf + s->size, data, len); @@ -147,8 +140,8 @@ int dbuf_put(DynBuf *s, const uint8_t *data, size_t len) int dbuf_put_self(DynBuf *s, size_t offset, size_t len) { - if (unlikely((s->size + len) > s->allocated_size)) { - if (dbuf_realloc(s, s->size + len)) + if (unlikely((s->allocated_size - s->size) < len)) { + if (dbuf_claim(s, len)) return -1; } memcpy(s->buf + s->size, s->buf + offset, len); @@ -156,11 +149,26 @@ int dbuf_put_self(DynBuf *s, size_t offset, size_t len) return 0; } -int dbuf_putc(DynBuf *s, uint8_t c) +int __dbuf_putc(DynBuf *s, uint8_t c) { return dbuf_put(s, &c, 1); } +int __dbuf_put_u16(DynBuf *s, uint16_t val) +{ + return dbuf_put(s, (uint8_t *)&val, 2); +} + +int __dbuf_put_u32(DynBuf *s, uint32_t val) +{ + return dbuf_put(s, (uint8_t *)&val, 4); +} + +int __dbuf_put_u64(DynBuf *s, uint64_t val) +{ + return dbuf_put(s, (uint8_t *)&val, 8); +} + int dbuf_putstr(DynBuf *s, const char *str) { return dbuf_put(s, (const uint8_t *)str, strlen(str)); @@ -182,7 +190,7 @@ int __attribute__((format(printf, 2, 3))) dbuf_printf(DynBuf *s, /* fast case */ return dbuf_put(s, (uint8_t *)buf, len); } else { - if (dbuf_realloc(s, s->size + len + 1)) + if (dbuf_claim(s, len + 1)) return -1; va_start(ap, fmt); vsnprintf((char *)(s->buf + s->size), s->allocated_size - s->size, diff --git a/src/couch_quickjs/quickjs/cutils.h b/src/couch_quickjs/quickjs/cutils.h index 32b97579db..094a8f1241 100644 --- a/src/couch_quickjs/quickjs/cutils.h +++ b/src/couch_quickjs/quickjs/cutils.h @@ -264,24 +264,58 @@ typedef struct DynBuf { void dbuf_init(DynBuf *s); void dbuf_init2(DynBuf *s, void *opaque, DynBufReallocFunc *realloc_func); -int dbuf_realloc(DynBuf *s, size_t new_size); -int dbuf_write(DynBuf *s, size_t offset, const uint8_t *data, size_t len); +int dbuf_claim(DynBuf *s, size_t len); int dbuf_put(DynBuf *s, const uint8_t *data, size_t len); int dbuf_put_self(DynBuf *s, size_t offset, size_t len); -int dbuf_putc(DynBuf *s, uint8_t c); int dbuf_putstr(DynBuf *s, const char *str); +int __dbuf_putc(DynBuf *s, uint8_t c); +int __dbuf_put_u16(DynBuf *s, uint16_t val); +int __dbuf_put_u32(DynBuf *s, uint32_t val); +int __dbuf_put_u64(DynBuf *s, uint64_t val); + +static inline int dbuf_putc(DynBuf *s, uint8_t val) +{ + if (unlikely((s->allocated_size - s->size) < 1)) { + return __dbuf_putc(s, val); + } else { + s->buf[s->size++] = val; + return 0; + } +} + static inline int dbuf_put_u16(DynBuf *s, uint16_t val) { - return dbuf_put(s, (uint8_t *)&val, 2); + if (unlikely((s->allocated_size - s->size) < 2)) { + return __dbuf_put_u16(s, val); + } else { + put_u16(s->buf + s->size, val); + s->size += 2; + return 0; + } } + static inline int dbuf_put_u32(DynBuf *s, uint32_t val) { - return dbuf_put(s, (uint8_t *)&val, 4); + if (unlikely((s->allocated_size - s->size) < 4)) { + return __dbuf_put_u32(s, val); + } else { + put_u32(s->buf + s->size, val); + s->size += 4; + return 0; + } } + static inline int dbuf_put_u64(DynBuf *s, uint64_t val) { - return dbuf_put(s, (uint8_t *)&val, 8); + if (unlikely((s->allocated_size - s->size) < 8)) { + return __dbuf_put_u64(s, val); + } else { + put_u64(s->buf + s->size, val); + s->size += 8; + return 0; + } } + int __attribute__((format(printf, 2, 3))) dbuf_printf(DynBuf *s, const char *fmt, ...); void dbuf_free(DynBuf *s); @@ -364,4 +398,60 @@ static inline double uint64_as_float64(uint64_t u64) return u.d; } +static inline double fromfp16(uint16_t v) +{ + double d; + uint32_t v1; + v1 = v & 0x7fff; + if (unlikely(v1 >= 0x7c00)) + v1 += 0x1f8000; /* NaN or infinity */ + d = uint64_as_float64(((uint64_t)(v >> 15) << 63) | ((uint64_t)v1 << (52 - 10))); + return d * 0x1p1008; +} + +static inline uint16_t tofp16(double d) +{ + uint64_t a, addend; + uint32_t v, sgn; + int shift; + + a = float64_as_uint64(d); + sgn = a >> 63; + a = a & 0x7fffffffffffffff; + if (unlikely(a > 0x7ff0000000000000)) { + /* nan */ + v = 0x7c01; + } else if (a < 0x3f10000000000000) { /* 0x1p-14 */ + /* subnormal f16 number or zero */ + if (a <= 0x3e60000000000000) { /* 0x1p-25 */ + v = 0x0000; /* zero */ + } else { + shift = 1051 - (a >> 52); + a = ((uint64_t)1 << 52) | (a & (((uint64_t)1 << 52) - 1)); + addend = ((a >> shift) & 1) + (((uint64_t)1 << (shift - 1)) - 1); + v = (a + addend) >> shift; + } + } else { + /* normal number or infinity */ + a -= 0x3f00000000000000; /* adjust the exponent */ + /* round */ + addend = ((a >> (52 - 10)) & 1) + (((uint64_t)1 << (52 - 11)) - 1); + v = (a + addend) >> (52 - 10); + /* overflow ? */ + if (unlikely(v > 0x7c00)) + v = 0x7c00; + } + return v | (sgn << 15); +} + +static inline int isfp16nan(uint16_t v) +{ + return (v & 0x7FFF) > 0x7C00; +} + +static inline int isfp16zero(uint16_t v) +{ + return (v & 0x7FFF) == 0; +} + #endif /* CUTILS_H */ diff --git a/src/couch_quickjs/quickjs/libregexp-opcode.h b/src/couch_quickjs/quickjs/libregexp-opcode.h index f255e09f27..ebab751dfc 100644 --- a/src/couch_quickjs/quickjs/libregexp-opcode.h +++ b/src/couch_quickjs/quickjs/libregexp-opcode.h @@ -26,11 +26,15 @@ DEF(invalid, 1) /* never used */ DEF(char, 3) +DEF(char_i, 3) DEF(char32, 5) +DEF(char32_i, 5) DEF(dot, 1) DEF(any, 1) /* same as dot but match any character including line terminator */ DEF(line_start, 1) +DEF(line_start_m, 1) DEF(line_end, 1) +DEF(line_end_m, 1) DEF(goto, 5) DEF(split_goto_first, 5) DEF(split_next_first, 5) @@ -42,11 +46,17 @@ DEF(loop, 5) /* decrement the top the stack and goto if != 0 */ DEF(push_i32, 5) /* push integer on the stack */ DEF(drop, 1) DEF(word_boundary, 1) +DEF(word_boundary_i, 1) DEF(not_word_boundary, 1) +DEF(not_word_boundary_i, 1) DEF(back_reference, 2) -DEF(backward_back_reference, 2) /* must come after back_reference */ +DEF(back_reference_i, 2) /* must come after */ +DEF(backward_back_reference, 2) /* must come after */ +DEF(backward_back_reference_i, 2) /* must come after */ DEF(range, 3) /* variable length */ +DEF(range_i, 3) /* variable length */ DEF(range32, 3) /* variable length */ +DEF(range32_i, 3) /* variable length */ DEF(lookahead, 5) DEF(negative_lookahead, 5) DEF(push_char_pos, 1) /* push the character position on the stack */ diff --git a/src/couch_quickjs/quickjs/libregexp.c b/src/couch_quickjs/quickjs/libregexp.c index 8c47389852..118d950eb9 100644 --- a/src/couch_quickjs/quickjs/libregexp.c +++ b/src/couch_quickjs/quickjs/libregexp.c @@ -71,7 +71,9 @@ typedef struct { const uint8_t *buf_start; int re_flags; BOOL is_unicode; + BOOL unicode_sets; /* if set, is_unicode is also set */ BOOL ignore_case; + BOOL multi_line; BOOL dotall; int capture_count; int total_capture_count; /* -1 = not computed yet */ @@ -102,11 +104,11 @@ static const REOpCode reopcode_info[REOP_COUNT] = { }; #define RE_HEADER_FLAGS 0 -#define RE_HEADER_CAPTURE_COUNT 1 -#define RE_HEADER_STACK_SIZE 2 -#define RE_HEADER_BYTECODE_LEN 3 +#define RE_HEADER_CAPTURE_COUNT 2 +#define RE_HEADER_STACK_SIZE 3 +#define RE_HEADER_BYTECODE_LEN 4 -#define RE_HEADER_LEN 7 +#define RE_HEADER_LEN 8 static inline int is_digit(int c) { return c >= '0' && c <= '9'; @@ -115,13 +117,271 @@ static inline int is_digit(int c) { /* insert 'len' bytes at position 'pos'. Return < 0 if error. */ static int dbuf_insert(DynBuf *s, int pos, int len) { - if (dbuf_realloc(s, s->size + len)) + if (dbuf_claim(s, len)) return -1; memmove(s->buf + pos + len, s->buf + pos, s->size - pos); s->size += len; return 0; } +typedef struct REString { + struct REString *next; + uint32_t hash; + uint32_t len; + uint32_t buf[]; +} REString; + +typedef struct { + /* the string list is the union of 'char_range' and of the strings + in hash_table[]. The strings in hash_table[] have a length != + 1. */ + CharRange cr; + uint32_t n_strings; + uint32_t hash_size; + int hash_bits; + REString **hash_table; +} REStringList; + +static uint32_t re_string_hash(int len, const uint32_t *buf) +{ + int i; + uint32_t h; + h = 1; + for(i = 0; i < len; i++) + h = h * 263 + buf[i]; + return h * 0x61C88647; +} + +static void re_string_list_init(REParseState *s1, REStringList *s) +{ + cr_init(&s->cr, s1->opaque, lre_realloc); + s->n_strings = 0; + s->hash_size = 0; + s->hash_bits = 0; + s->hash_table = NULL; +} + +static void re_string_list_free(REStringList *s) +{ + REString *p, *p_next; + int i; + for(i = 0; i < s->hash_size; i++) { + for(p = s->hash_table[i]; p != NULL; p = p_next) { + p_next = p->next; + lre_realloc(s->cr.mem_opaque, p, 0); + } + } + lre_realloc(s->cr.mem_opaque, s->hash_table, 0); + + cr_free(&s->cr); +} + +static void lre_print_char(int c, BOOL is_range) +{ + if (c == '\'' || c == '\\' || + (is_range && (c == '-' || c == ']'))) { + printf("\\%c", c); + } else if (c >= ' ' && c <= 126) { + printf("%c", c); + } else { + printf("\\u{%04x}", c); + } +} + +static __maybe_unused void re_string_list_dump(const char *str, const REStringList *s) +{ + REString *p; + const CharRange *cr; + int i, j, k; + + printf("%s:\n", str); + printf(" ranges: ["); + cr = &s->cr; + for(i = 0; i < cr->len; i += 2) { + lre_print_char(cr->points[i], TRUE); + if (cr->points[i] != cr->points[i + 1] - 1) { + printf("-"); + lre_print_char(cr->points[i + 1] - 1, TRUE); + } + } + printf("]\n"); + + j = 0; + for(i = 0; i < s->hash_size; i++) { + for(p = s->hash_table[i]; p != NULL; p = p->next) { + printf(" %d/%d: '", j, s->n_strings); + for(k = 0; k < p->len; k++) { + lre_print_char(p->buf[k], FALSE); + } + printf("'\n"); + j++; + } + } +} + +static int re_string_find2(REStringList *s, int len, const uint32_t *buf, + uint32_t h0, BOOL add_flag) +{ + uint32_t h = 0; /* avoid warning */ + REString *p; + if (s->n_strings != 0) { + h = h0 >> (32 - s->hash_bits); + for(p = s->hash_table[h]; p != NULL; p = p->next) { + if (p->hash == h0 && p->len == len && + !memcmp(p->buf, buf, len * sizeof(buf[0]))) { + return 1; + } + } + } + /* not found */ + if (!add_flag) + return 0; + /* increase the size of the hash table if needed */ + if (unlikely((s->n_strings + 1) > s->hash_size)) { + REString **new_hash_table, *p_next; + int new_hash_bits, i; + uint32_t new_hash_size; + new_hash_bits = max_int(s->hash_bits + 1, 4); + new_hash_size = 1 << new_hash_bits; + new_hash_table = lre_realloc(s->cr.mem_opaque, NULL, + sizeof(new_hash_table[0]) * new_hash_size); + if (!new_hash_table) + return -1; + memset(new_hash_table, 0, sizeof(new_hash_table[0]) * new_hash_size); + for(i = 0; i < s->hash_size; i++) { + for(p = s->hash_table[i]; p != NULL; p = p_next) { + p_next = p->next; + h = p->hash >> (32 - new_hash_bits); + p->next = new_hash_table[h]; + new_hash_table[h] = p; + } + } + lre_realloc(s->cr.mem_opaque, s->hash_table, 0); + s->hash_bits = new_hash_bits; + s->hash_size = new_hash_size; + s->hash_table = new_hash_table; + h = h0 >> (32 - s->hash_bits); + } + + p = lre_realloc(s->cr.mem_opaque, NULL, sizeof(REString) + len * sizeof(buf[0])); + if (!p) + return -1; + p->next = s->hash_table[h]; + s->hash_table[h] = p; + s->n_strings++; + p->hash = h0; + p->len = len; + memcpy(p->buf, buf, sizeof(buf[0]) * len); + return 1; +} + +static int re_string_find(REStringList *s, int len, const uint32_t *buf, + BOOL add_flag) +{ + uint32_t h0; + h0 = re_string_hash(len, buf); + return re_string_find2(s, len, buf, h0, add_flag); +} + +/* return -1 if memory error, 0 if OK */ +static int re_string_add(REStringList *s, int len, const uint32_t *buf) +{ + if (len == 1) { + return cr_union_interval(&s->cr, buf[0], buf[0]); + } + if (re_string_find(s, len, buf, TRUE) < 0) + return -1; + return 0; +} + +/* a = a op b */ +static int re_string_list_op(REStringList *a, REStringList *b, int op) +{ + int i, ret; + REString *p, **pp; + + if (cr_op1(&a->cr, b->cr.points, b->cr.len, op)) + return -1; + + switch(op) { + case CR_OP_UNION: + if (b->n_strings != 0) { + for(i = 0; i < b->hash_size; i++) { + for(p = b->hash_table[i]; p != NULL; p = p->next) { + if (re_string_find2(a, p->len, p->buf, p->hash, TRUE) < 0) + return -1; + } + } + } + break; + case CR_OP_INTER: + case CR_OP_SUB: + for(i = 0; i < a->hash_size; i++) { + pp = &a->hash_table[i]; + for(;;) { + p = *pp; + if (p == NULL) + break; + ret = re_string_find2(b, p->len, p->buf, p->hash, FALSE); + if (op == CR_OP_SUB) + ret = !ret; + if (!ret) { + /* remove it */ + *pp = p->next; + a->n_strings--; + lre_realloc(a->cr.mem_opaque, p, 0); + } else { + /* keep it */ + pp = &p->next; + } + } + } + break; + default: + abort(); + } + return 0; +} + +static int re_string_list_canonicalize(REParseState *s1, + REStringList *s, BOOL is_unicode) +{ + if (cr_regexp_canonicalize(&s->cr, is_unicode)) + return -1; + if (s->n_strings != 0) { + REStringList a_s, *a = &a_s; + int i, j; + REString *p; + + /* XXX: simplify */ + re_string_list_init(s1, a); + + a->n_strings = s->n_strings; + a->hash_size = s->hash_size; + a->hash_bits = s->hash_bits; + a->hash_table = s->hash_table; + + s->n_strings = 0; + s->hash_size = 0; + s->hash_bits = 0; + s->hash_table = NULL; + + for(i = 0; i < a->hash_size; i++) { + for(p = a->hash_table[i]; p != NULL; p = p->next) { + for(j = 0; j < p->len; j++) { + p->buf[j] = lre_canonicalize(p->buf[j], is_unicode); + } + if (re_string_add(s, p->len, p->buf)) { + re_string_list_free(a); + return -1; + } + } + } + re_string_list_free(a); + } + return 0; +} + static const uint16_t char_range_d[] = { 1, 0x0030, 0x0039 + 1, @@ -170,7 +430,7 @@ static const uint16_t * const char_range_table[] = { char_range_w, }; -static int cr_init_char_range(REParseState *s, CharRange *cr, uint32_t c) +static int cr_init_char_range(REParseState *s, REStringList *cr, uint32_t c) { BOOL invert; const uint16_t *c_pt; @@ -179,18 +439,18 @@ static int cr_init_char_range(REParseState *s, CharRange *cr, uint32_t c) invert = c & 1; c_pt = char_range_table[c >> 1]; len = *c_pt++; - cr_init(cr, s->opaque, lre_realloc); + re_string_list_init(s, cr); for(i = 0; i < len * 2; i++) { - if (cr_add_point(cr, c_pt[i])) + if (cr_add_point(&cr->cr, c_pt[i])) goto fail; } if (invert) { - if (cr_invert(cr)) + if (cr_invert(&cr->cr)) goto fail; } return 0; fail: - cr_free(cr); + re_string_list_free(cr); return -1; } @@ -240,6 +500,7 @@ static __maybe_unused void lre_dump_bytecode(const uint8_t *buf, printf("%s", reopcode_info[opcode].name); switch(opcode) { case REOP_char: + case REOP_char_i: val = get_u16(buf + pos + 1); if (val >= ' ' && val <= 126) printf(" '%c'", val); @@ -247,6 +508,7 @@ static __maybe_unused void lre_dump_bytecode(const uint8_t *buf, printf(" 0x%04x", val); break; case REOP_char32: + case REOP_char32_i: val = get_u32(buf + pos + 1); if (val >= ' ' && val <= 126) printf(" '%c'", val); @@ -273,7 +535,9 @@ static __maybe_unused void lre_dump_bytecode(const uint8_t *buf, case REOP_save_start: case REOP_save_end: case REOP_back_reference: + case REOP_back_reference_i: case REOP_backward_back_reference: + case REOP_backward_back_reference_i: printf(" %u", buf[pos + 1]); break; case REOP_save_reset: @@ -284,6 +548,7 @@ static __maybe_unused void lre_dump_bytecode(const uint8_t *buf, printf(" %d", val); break; case REOP_range: + case REOP_range_i: { int n, i; n = get_u16(buf + pos + 1); @@ -295,6 +560,7 @@ static __maybe_unused void lre_dump_bytecode(const uint8_t *buf, } break; case REOP_range32: + case REOP_range32_i: { int n, i; n = get_u16(buf + pos + 1); @@ -533,8 +799,16 @@ static BOOL is_unicode_char(int c) (c == '_')); } -static int parse_unicode_property(REParseState *s, CharRange *cr, - const uint8_t **pp, BOOL is_inv) +/* XXX: memory error test */ +static void seq_prop_cb(void *opaque, const uint32_t *seq, int seq_len) +{ + REStringList *sl = opaque; + re_string_add(sl, seq_len, seq); +} + +static int parse_unicode_property(REParseState *s, REStringList *cr, + const uint8_t **pp, BOOL is_inv, + BOOL allow_sequence_prop) { const uint8_t *p; char name[64], value[64]; @@ -574,51 +848,76 @@ static int parse_unicode_property(REParseState *s, CharRange *cr, } else if (!strcmp(name, "Script_Extensions") || !strcmp(name, "scx")) { script_ext = TRUE; do_script: - cr_init(cr, s->opaque, lre_realloc); - ret = unicode_script(cr, value, script_ext); + re_string_list_init(s, cr); + ret = unicode_script(&cr->cr, value, script_ext); if (ret) { - cr_free(cr); + re_string_list_free(cr); if (ret == -2) return re_parse_error(s, "unknown unicode script"); else goto out_of_memory; } } else if (!strcmp(name, "General_Category") || !strcmp(name, "gc")) { - cr_init(cr, s->opaque, lre_realloc); - ret = unicode_general_category(cr, value); + re_string_list_init(s, cr); + ret = unicode_general_category(&cr->cr, value); if (ret) { - cr_free(cr); + re_string_list_free(cr); if (ret == -2) return re_parse_error(s, "unknown unicode general category"); else goto out_of_memory; } } else if (value[0] == '\0') { - cr_init(cr, s->opaque, lre_realloc); - ret = unicode_general_category(cr, name); + re_string_list_init(s, cr); + ret = unicode_general_category(&cr->cr, name); if (ret == -1) { - cr_free(cr); + re_string_list_free(cr); goto out_of_memory; } if (ret < 0) { - ret = unicode_prop(cr, name); - if (ret) { - cr_free(cr); - if (ret == -2) - goto unknown_property_name; - else - goto out_of_memory; + ret = unicode_prop(&cr->cr, name); + if (ret == -1) { + re_string_list_free(cr); + goto out_of_memory; + } + } + if (ret < 0 && !is_inv && allow_sequence_prop) { + CharRange cr_tmp; + cr_init(&cr_tmp, s->opaque, lre_realloc); + ret = unicode_sequence_prop(name, seq_prop_cb, cr, &cr_tmp); + cr_free(&cr_tmp); + if (ret == -1) { + re_string_list_free(cr); + goto out_of_memory; } } + if (ret < 0) + goto unknown_property_name; } else { unknown_property_name: return re_parse_error(s, "unknown unicode property name"); } + /* the ordering of case folding and inversion differs with + unicode_sets. 'unicode_sets' ordering is more consistent */ + /* XXX: the spec seems incorrect, we do it as the other engines + seem to do it. */ + if (s->ignore_case && s->unicode_sets) { + if (re_string_list_canonicalize(s, cr, s->is_unicode)) { + re_string_list_free(cr); + goto out_of_memory; + } + } if (is_inv) { - if (cr_invert(cr)) { - cr_free(cr); - return -1; + if (cr_invert(&cr->cr)) { + re_string_list_free(cr); + goto out_of_memory; + } + } + if (s->ignore_case && !s->unicode_sets) { + if (re_string_list_canonicalize(s, cr, s->is_unicode)) { + re_string_list_free(cr); + goto out_of_memory; } } *pp = p; @@ -628,10 +927,61 @@ static int parse_unicode_property(REParseState *s, CharRange *cr, } #endif /* CONFIG_ALL_UNICODE */ +static int get_class_atom(REParseState *s, REStringList *cr, + const uint8_t **pp, BOOL inclass); + +static int parse_class_string_disjunction(REParseState *s, REStringList *cr, + const uint8_t **pp) +{ + const uint8_t *p; + DynBuf str; + int c; + + p = *pp; + if (*p != '{') + return re_parse_error(s, "expecting '{' after \\q"); + + dbuf_init2(&str, s->opaque, lre_realloc); + re_string_list_init(s, cr); + + p++; + for(;;) { + str.size = 0; + while (*p != '}' && *p != '|') { + c = get_class_atom(s, NULL, &p, FALSE); + if (c < 0) + goto fail; + if (dbuf_put_u32(&str, c)) { + re_parse_out_of_memory(s); + goto fail; + } + } + if (re_string_add(cr, str.size / 4, (uint32_t *)str.buf)) { + re_parse_out_of_memory(s); + goto fail; + } + if (*p == '}') + break; + p++; + } + if (s->ignore_case) { + if (re_string_list_canonicalize(s, cr, TRUE)) + goto fail; + } + p++; /* skip the '}' */ + dbuf_free(&str); + *pp = p; + return 0; + fail: + dbuf_free(&str); + re_string_list_free(cr); + return -1; +} + /* return -1 if error otherwise the character or a class range - (CLASS_RANGE_BASE). In case of class range, 'cr' is + (CLASS_RANGE_BASE) if cr != NULL. In case of class range, 'cr' is initialized. Otherwise, it is ignored. */ -static int get_class_atom(REParseState *s, CharRange *cr, +static int get_class_atom(REParseState *s, REStringList *cr, const uint8_t **pp, BOOL inclass) { const uint8_t *p; @@ -666,6 +1016,8 @@ static int get_class_atom(REParseState *s, CharRange *cr, case 'W': c = CHAR_RANGE_W; class_range: + if (!cr) + goto default_escape; if (cr_init_char_range(s, cr, c)) return -1; c = CLASS_RANGE_BASE; @@ -690,27 +1042,50 @@ static int get_class_atom(REParseState *s, CharRange *cr, if (!inclass && s->is_unicode) goto invalid_escape; break; + case '^': + case '$': + case '\\': + case '.': + case '*': + case '+': + case '?': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '|': + case '/': + /* always valid to escape these characters */ + break; #ifdef CONFIG_ALL_UNICODE case 'p': case 'P': - if (s->is_unicode) { - if (parse_unicode_property(s, cr, &p, (c == 'P'))) + if (s->is_unicode && cr) { + if (parse_unicode_property(s, cr, &p, (c == 'P'), s->unicode_sets)) return -1; c = CLASS_RANGE_BASE; break; } - /* fall thru */ + goto default_escape; #endif + case 'q': + if (s->unicode_sets && cr && inclass) { + if (parse_class_string_disjunction(s, cr, &p)) + return -1; + c = CLASS_RANGE_BASE; + break; + } + goto default_escape; default: + default_escape: p--; ret = lre_parse_escape(&p, s->is_unicode * 2); if (ret >= 0) { c = ret; } else { - if (ret == -2 && *p != '\0' && strchr("^$\\.*+?()[]{}|/", *p)) { - /* always valid to escape these characters */ - goto normal_char; - } else if (s->is_unicode) { + if (s->is_unicode) { invalid_escape: return re_parse_error(s, "invalid escape sequence in regular expression"); } else { @@ -727,6 +1102,48 @@ static int get_class_atom(REParseState *s, CharRange *cr, return re_parse_error(s, "unexpected end"); } /* fall thru */ + goto normal_char; + + case '&': + case '!': + case '#': + case '$': + case '%': + case '*': + case '+': + case ',': + case '.': + case ':': + case ';': + case '<': + case '=': + case '>': + case '?': + case '@': + case '^': + case '`': + case '~': + if (s->unicode_sets && p[1] == c) { + /* forbidden double characters */ + return re_parse_error(s, "invalid class set operation in regular expression"); + } + goto normal_char; + + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '/': + case '-': + case '|': + if (s->unicode_sets) { + /* invalid characters in unicode sets */ + return re_parse_error(s, "invalid character in class in regular expression"); + } + goto normal_char; + default: normal_char: /* normal char */ @@ -754,8 +1171,6 @@ static int re_emit_range(REParseState *s, const CharRange *cr) if (len >= 65535) return re_parse_error(s, "too many ranges"); if (len == 0) { - /* not sure it can really happen. Emit a match that is always - false */ re_emit_op_u32(s, REOP_char32, -1); } else { high = cr->points[cr->len - 1]; @@ -764,7 +1179,7 @@ static int re_emit_range(REParseState *s, const CharRange *cr) if (high <= 0xffff) { /* can use 16 bit ranges with the conversion that 0xffff = infinity */ - re_emit_op_u16(s, REOP_range, len); + re_emit_op_u16(s, s->ignore_case ? REOP_range_i : REOP_range, len); for(i = 0; i < cr->len; i += 2) { dbuf_put_u16(&s->byte_code, cr->points[i]); high = cr->points[i + 1] - 1; @@ -773,7 +1188,7 @@ static int re_emit_range(REParseState *s, const CharRange *cr) dbuf_put_u16(&s->byte_code, high); } } else { - re_emit_op_u16(s, REOP_range32, len); + re_emit_op_u16(s, s->ignore_case ? REOP_range32_i : REOP_range32, len); for(i = 0; i < cr->len; i += 2) { dbuf_put_u32(&s->byte_code, cr->points[i]); dbuf_put_u32(&s->byte_code, cr->points[i + 1] - 1); @@ -783,15 +1198,139 @@ static int re_emit_range(REParseState *s, const CharRange *cr) return 0; } -static int re_parse_char_class(REParseState *s, const uint8_t **pp) +static int re_string_cmp_len(const void *a, const void *b, void *arg) +{ + REString *p1 = *(REString **)a; + REString *p2 = *(REString **)b; + return (p1->len < p2->len) - (p1->len > p2->len); +} + +static void re_emit_char(REParseState *s, int c) +{ + if (c <= 0xffff) + re_emit_op_u16(s, s->ignore_case ? REOP_char_i : REOP_char, c); + else + re_emit_op_u32(s, s->ignore_case ? REOP_char32_i : REOP_char32, c); +} + +static int re_emit_string_list(REParseState *s, const REStringList *sl) +{ + REString **tab, *p; + int i, j, split_pos, last_match_pos, n; + BOOL has_empty_string, is_last; + + // re_string_list_dump("sl", sl); + if (sl->n_strings == 0) { + /* simple case: only characters */ + if (re_emit_range(s, &sl->cr)) + return -1; + } else { + /* at least one string list is present : match the longest ones first */ + /* XXX: add a new op_switch opcode to compile as a trie */ + tab = lre_realloc(s->opaque, NULL, sizeof(tab[0]) * sl->n_strings); + if (!tab) { + re_parse_out_of_memory(s); + return -1; + } + has_empty_string = FALSE; + n = 0; + for(i = 0; i < sl->hash_size; i++) { + for(p = sl->hash_table[i]; p != NULL; p = p->next) { + if (p->len == 0) { + has_empty_string = TRUE; + } else { + tab[n++] = p; + } + } + } + assert(n <= sl->n_strings); + + rqsort(tab, n, sizeof(tab[0]), re_string_cmp_len, NULL); + + last_match_pos = -1; + for(i = 0; i < n; i++) { + p = tab[i]; + is_last = !has_empty_string && sl->cr.len == 0 && i == (n - 1); + if (!is_last) + split_pos = re_emit_op_u32(s, REOP_split_next_first, 0); + else + split_pos = 0; + for(j = 0; j < p->len; j++) { + re_emit_char(s, p->buf[j]); + } + if (!is_last) { + last_match_pos = re_emit_op_u32(s, REOP_goto, last_match_pos); + put_u32(s->byte_code.buf + split_pos, s->byte_code.size - (split_pos + 4)); + } + } + + if (sl->cr.len != 0) { + /* char range */ + is_last = !has_empty_string; + if (!is_last) + split_pos = re_emit_op_u32(s, REOP_split_next_first, 0); + else + split_pos = 0; /* not used */ + if (re_emit_range(s, &sl->cr)) { + lre_realloc(s->opaque, tab, 0); + return -1; + } + if (!is_last) + put_u32(s->byte_code.buf + split_pos, s->byte_code.size - (split_pos + 4)); + } + + /* patch the 'goto match' */ + while (last_match_pos != -1) { + int next_pos = get_u32(s->byte_code.buf + last_match_pos); + put_u32(s->byte_code.buf + last_match_pos, s->byte_code.size - (last_match_pos + 4)); + last_match_pos = next_pos; + } + + lre_realloc(s->opaque, tab, 0); + } + return 0; +} + +static int re_parse_nested_class(REParseState *s, REStringList *cr, const uint8_t **pp); + +static int re_parse_class_set_operand(REParseState *s, REStringList *cr, const uint8_t **pp) +{ + int c1; + const uint8_t *p = *pp; + + if (*p == '[') { + if (re_parse_nested_class(s, cr, pp)) + return -1; + } else { + c1 = get_class_atom(s, cr, pp, TRUE); + if (c1 < 0) + return -1; + if (c1 < CLASS_RANGE_BASE) { + /* create a range with a single character */ + re_string_list_init(s, cr); + if (s->ignore_case) + c1 = lre_canonicalize(c1, s->is_unicode); + if (cr_union_interval(&cr->cr, c1, c1)) { + re_string_list_free(cr); + return -1; + } + } + } + return 0; +} + +static int re_parse_nested_class(REParseState *s, REStringList *cr, const uint8_t **pp) { const uint8_t *p; uint32_t c1, c2; - CharRange cr_s, *cr = &cr_s; - CharRange cr1_s, *cr1 = &cr1_s; - BOOL invert; + int ret; + REStringList cr1_s, *cr1 = &cr1_s; + BOOL invert, is_first; - cr_init(cr, s->opaque, lre_realloc); + if (lre_check_stack_overflow(s->opaque, 0)) + return re_parse_error(s, "stack overflow"); + + re_string_list_init(s, cr); p = *pp; p++; /* skip '[' */ @@ -800,74 +1339,155 @@ static int re_parse_char_class(REParseState *s, const uint8_t **pp) p++; invert = TRUE; } - + + /* handle unions */ + is_first = TRUE; for(;;) { if (*p == ']') break; - c1 = get_class_atom(s, cr1, &p, TRUE); - if ((int)c1 < 0) - goto fail; - if (*p == '-' && p[1] != ']') { - const uint8_t *p0 = p + 1; - if (c1 >= CLASS_RANGE_BASE) { - if (s->is_unicode) { - cr_free(cr1); - goto invalid_class_range; - } - /* Annex B: match '-' character */ - goto class_atom; - } - c2 = get_class_atom(s, cr1, &p0, TRUE); - if ((int)c2 < 0) - goto fail; - if (c2 >= CLASS_RANGE_BASE) { - cr_free(cr1); - if (s->is_unicode) { - goto invalid_class_range; - } - /* Annex B: match '-' character */ - goto class_atom; - } - p = p0; - if (c2 < c1) { - invalid_class_range: - re_parse_error(s, "invalid class range"); + if (*p == '[' && s->unicode_sets) { + if (re_parse_nested_class(s, cr1, &p)) goto fail; - } - if (cr_union_interval(cr, c1, c2)) - goto memory_error; + goto class_union; } else { - class_atom: - if (c1 >= CLASS_RANGE_BASE) { - int ret; - ret = cr_union1(cr, cr1->points, cr1->len); - cr_free(cr1); - if (ret) - goto memory_error; + c1 = get_class_atom(s, cr1, &p, TRUE); + if ((int)c1 < 0) + goto fail; + if (*p == '-' && p[1] != ']') { + const uint8_t *p0 = p + 1; + if (p[1] == '-' && s->unicode_sets && is_first) + goto class_atom; /* first character class followed by '--' */ + if (c1 >= CLASS_RANGE_BASE) { + if (s->is_unicode) { + re_string_list_free(cr1); + goto invalid_class_range; + } + /* Annex B: match '-' character */ + goto class_atom; + } + c2 = get_class_atom(s, cr1, &p0, TRUE); + if ((int)c2 < 0) + goto fail; + if (c2 >= CLASS_RANGE_BASE) { + re_string_list_free(cr1); + if (s->is_unicode) { + goto invalid_class_range; + } + /* Annex B: match '-' character */ + goto class_atom; + } + p = p0; + if (c2 < c1) { + invalid_class_range: + re_parse_error(s, "invalid class range"); + goto fail; + } + if (s->ignore_case) { + CharRange cr2_s, *cr2 = &cr2_s; + cr_init(cr2, s->opaque, lre_realloc); + if (cr_add_interval(cr2, c1, c2 + 1) || + cr_regexp_canonicalize(cr2, s->is_unicode) || + cr_op1(&cr->cr, cr2->points, cr2->len, CR_OP_UNION)) { + cr_free(cr2); + goto memory_error; + } + cr_free(cr2); + } else { + if (cr_union_interval(&cr->cr, c1, c2)) + goto memory_error; + } + is_first = FALSE; /* union operation */ } else { - if (cr_union_interval(cr, c1, c1)) - goto memory_error; + class_atom: + if (c1 >= CLASS_RANGE_BASE) { + class_union: + ret = re_string_list_op(cr, cr1, CR_OP_UNION); + re_string_list_free(cr1); + if (ret) + goto memory_error; + } else { + if (s->ignore_case) + c1 = lre_canonicalize(c1, s->is_unicode); + if (cr_union_interval(&cr->cr, c1, c1)) + goto memory_error; + } } } + if (s->unicode_sets && is_first) { + if (*p == '&' && p[1] == '&' && p[2] != '&') { + /* handle '&&' */ + for(;;) { + if (*p == ']') { + break; + } else if (*p == '&' && p[1] == '&' && p[2] != '&') { + p += 2; + } else { + goto invalid_operation; + } + if (re_parse_class_set_operand(s, cr1, &p)) + goto fail; + ret = re_string_list_op(cr, cr1, CR_OP_INTER); + re_string_list_free(cr1); + if (ret) + goto memory_error; + } + } else if (*p == '-' && p[1] == '-') { + /* handle '--' */ + for(;;) { + if (*p == ']') { + break; + } else if (*p == '-' && p[1] == '-') { + p += 2; + } else { + invalid_operation: + re_parse_error(s, "invalid operation in regular expression"); + goto fail; + } + if (re_parse_class_set_operand(s, cr1, &p)) + goto fail; + ret = re_string_list_op(cr, cr1, CR_OP_SUB); + re_string_list_free(cr1); + if (ret) + goto memory_error; + } + } + } + is_first = FALSE; } - if (s->ignore_case) { - if (cr_regexp_canonicalize(cr, s->is_unicode)) - goto memory_error; - } + + p++; /* skip ']' */ + *pp = p; if (invert) { - if (cr_invert(cr)) + /* XXX: add may_contain_string syntax check to be fully + compliant. The test here accepts more input than the + spec. */ + if (cr->n_strings != 0) { + re_parse_error(s, "negated character class with strings in regular expression debugger eval code"); + goto fail; + } + if (cr_invert(&cr->cr)) goto memory_error; } - if (re_emit_range(s, cr)) - goto fail; - cr_free(cr); - p++; /* skip ']' */ - *pp = p; return 0; memory_error: re_parse_out_of_memory(s); fail: - cr_free(cr); + re_string_list_free(cr); + return -1; +} + +static int re_parse_char_class(REParseState *s, const uint8_t **pp) +{ + REStringList cr_s, *cr = &cr_s; + + if (re_parse_nested_class(s, cr, pp)) + return -1; + if (re_emit_string_list(s, cr)) + goto fail; + re_string_list_free(cr); + return 0; + fail: + re_string_list_free(cr); return -1; } @@ -888,27 +1508,35 @@ static BOOL re_need_check_advance(const uint8_t *bc_buf, int bc_buf_len) len = reopcode_info[opcode].size; switch(opcode) { case REOP_range: + case REOP_range_i: val = get_u16(bc_buf + pos + 1); len += val * 4; goto simple_char; case REOP_range32: + case REOP_range32_i: val = get_u16(bc_buf + pos + 1); len += val * 8; goto simple_char; case REOP_char: + case REOP_char_i: case REOP_char32: + case REOP_char32_i: case REOP_dot: case REOP_any: simple_char: ret = FALSE; break; case REOP_line_start: + case REOP_line_start_m: case REOP_line_end: + case REOP_line_end_m: case REOP_push_i32: case REOP_push_char_pos: case REOP_drop: case REOP_word_boundary: + case REOP_word_boundary_i: case REOP_not_word_boundary: + case REOP_not_word_boundary_i: case REOP_prev: /* no effect */ break; @@ -916,7 +1544,9 @@ static BOOL re_need_check_advance(const uint8_t *bc_buf, int bc_buf_len) case REOP_save_end: case REOP_save_reset: case REOP_back_reference: + case REOP_back_reference_i: case REOP_backward_back_reference: + case REOP_backward_back_reference_i: break; default: /* safe behavior: we cannot predict the outcome */ @@ -941,24 +1571,32 @@ static int re_is_simple_quantifier(const uint8_t *bc_buf, int bc_buf_len) len = reopcode_info[opcode].size; switch(opcode) { case REOP_range: + case REOP_range_i: val = get_u16(bc_buf + pos + 1); len += val * 4; goto simple_char; case REOP_range32: + case REOP_range32_i: val = get_u16(bc_buf + pos + 1); len += val * 8; goto simple_char; case REOP_char: + case REOP_char_i: case REOP_char32: + case REOP_char32_i: case REOP_dot: case REOP_any: simple_char: count++; break; case REOP_line_start: + case REOP_line_start_m: case REOP_line_end: + case REOP_line_end_m: case REOP_word_boundary: + case REOP_word_boundary_i: case REOP_not_word_boundary: + case REOP_not_word_boundary_i: break; default: return -1; @@ -1116,12 +1754,47 @@ static int find_group_name(REParseState *s, const char *name) static int re_parse_disjunction(REParseState *s, BOOL is_backward_dir); +static int re_parse_modifiers(REParseState *s, const uint8_t **pp) +{ + const uint8_t *p = *pp; + int mask = 0; + int val; + + for(;;) { + if (*p == 'i') { + val = LRE_FLAG_IGNORECASE; + } else if (*p == 'm') { + val = LRE_FLAG_MULTILINE; + } else if (*p == 's') { + val = LRE_FLAG_DOTALL; + } else { + break; + } + if (mask & val) + return re_parse_error(s, "duplicate modifier: '%c'", *p); + mask |= val; + p++; + } + *pp = p; + return mask; +} + +static BOOL update_modifier(BOOL val, int add_mask, int remove_mask, + int mask) +{ + if (add_mask & mask) + val = TRUE; + if (remove_mask & mask) + val = FALSE; + return val; +} + static int re_parse_term(REParseState *s, BOOL is_backward_dir) { const uint8_t *p; int c, last_atom_start, quant_min, quant_max, last_capture_count; BOOL greedy, add_zero_advance_check, is_neg, is_backward_lookahead; - CharRange cr_s, *cr = &cr_s; + REStringList cr_s, *cr = &cr_s; last_atom_start = -1; last_capture_count = 0; @@ -1130,11 +1803,11 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) switch(c) { case '^': p++; - re_emit_op(s, REOP_line_start); + re_emit_op(s, s->multi_line ? REOP_line_start_m : REOP_line_start); break; case '$': p++; - re_emit_op(s, REOP_line_end); + re_emit_op(s, s->multi_line ? REOP_line_end_m : REOP_line_end); break; case '.': p++; @@ -1184,6 +1857,44 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) p = s->buf_ptr; if (re_parse_expect(s, &p, ')')) return -1; + } else if (p[2] == 'i' || p[2] == 'm' || p[2] == 's' || p[2] == '-') { + BOOL saved_ignore_case, saved_multi_line, saved_dotall; + int add_mask, remove_mask; + p += 2; + remove_mask = 0; + add_mask = re_parse_modifiers(s, &p); + if (add_mask < 0) + return -1; + if (*p == '-') { + p++; + remove_mask = re_parse_modifiers(s, &p); + if (remove_mask < 0) + return -1; + } + if ((add_mask == 0 && remove_mask == 0) || + (add_mask & remove_mask) != 0) { + return re_parse_error(s, "invalid modifiers"); + } + if (re_parse_expect(s, &p, ':')) + return -1; + saved_ignore_case = s->ignore_case; + saved_multi_line = s->multi_line; + saved_dotall = s->dotall; + s->ignore_case = update_modifier(s->ignore_case, add_mask, remove_mask, LRE_FLAG_IGNORECASE); + s->multi_line = update_modifier(s->multi_line, add_mask, remove_mask, LRE_FLAG_MULTILINE); + s->dotall = update_modifier(s->dotall, add_mask, remove_mask, LRE_FLAG_DOTALL); + + last_atom_start = s->byte_code.size; + last_capture_count = s->capture_count; + s->buf_ptr = p; + if (re_parse_disjunction(s, is_backward_dir)) + return -1; + p = s->buf_ptr; + if (re_parse_expect(s, &p, ')')) + return -1; + s->ignore_case = saved_ignore_case; + s->multi_line = saved_multi_line; + s->dotall = saved_dotall; } else if ((p[2] == '=' || p[2] == '!')) { is_neg = (p[2] == '!'); is_backward_lookahead = FALSE; @@ -1262,7 +1973,11 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) switch(p[1]) { case 'b': case 'B': - re_emit_op(s, REOP_word_boundary + (p[1] != 'b')); + if (p[1] != 'b') { + re_emit_op(s, s->ignore_case ? REOP_not_word_boundary_i : REOP_not_word_boundary); + } else { + re_emit_op(s, s->ignore_case ? REOP_word_boundary_i : REOP_word_boundary); + } p += 2; break; case 'k': @@ -1351,7 +2066,8 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) emit_back_reference: last_atom_start = s->byte_code.size; last_capture_count = s->capture_count; - re_emit_op_u8(s, REOP_back_reference + is_backward_dir, c); + + re_emit_op_u8(s, REOP_back_reference + 2 * is_backward_dir + s->ignore_case, c); } break; default: @@ -1385,18 +2101,14 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) re_emit_op(s, REOP_prev); if (c >= CLASS_RANGE_BASE) { int ret; - /* Note: canonicalization is not needed */ - ret = re_emit_range(s, cr); - cr_free(cr); + ret = re_emit_string_list(s, cr); + re_string_list_free(cr); if (ret) return -1; } else { if (s->ignore_case) c = lre_canonicalize(c, s->is_unicode); - if (c <= 0xffff) - re_emit_op_u16(s, REOP_char, c); - else - re_emit_op_u32(s, REOP_char32, c); + re_emit_char(s, c); } if (is_backward_dir) re_emit_op(s, REOP_prev); @@ -1628,7 +2340,7 @@ static int re_parse_alternative(REParseState *s, BOOL is_backward_dir) speed is not really critical here) */ end = s->byte_code.size; term_size = end - term_start; - if (dbuf_realloc(&s->byte_code, end + term_size)) + if (dbuf_claim(&s->byte_code, term_size)) return -1; memmove(s->byte_code.buf + start + term_size, s->byte_code.buf + start, @@ -1706,10 +2418,12 @@ static int compute_stack_size(const uint8_t *bc_buf, int bc_buf_len) stack_size--; break; case REOP_range: + case REOP_range_i: val = get_u16(bc_buf + pos + 1); len += val * 4; break; case REOP_range32: + case REOP_range32_i: val = get_u16(bc_buf + pos + 1); len += val * 8; break; @@ -1719,6 +2433,17 @@ static int compute_stack_size(const uint8_t *bc_buf, int bc_buf_len) return stack_size_max; } +static void *lre_bytecode_realloc(void *opaque, void *ptr, size_t size) +{ + if (size > (INT32_MAX / 2)) { + /* the bytecode cannot be larger than 2G. Leave some slack to + avoid some overflows. */ + return NULL; + } else { + return lre_realloc(opaque, ptr, size); + } +} + /* 'buf' must be a zero terminated UTF-8 string of length buf_len. Return NULL if error and allocate an error message in *perror_msg, otherwise the compiled bytecode and its length in plen. @@ -1737,18 +2462,20 @@ uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size, s->buf_end = s->buf_ptr + buf_len; s->buf_start = s->buf_ptr; s->re_flags = re_flags; - s->is_unicode = ((re_flags & LRE_FLAG_UNICODE) != 0); + s->is_unicode = ((re_flags & (LRE_FLAG_UNICODE | LRE_FLAG_UNICODE_SETS)) != 0); is_sticky = ((re_flags & LRE_FLAG_STICKY) != 0); s->ignore_case = ((re_flags & LRE_FLAG_IGNORECASE) != 0); + s->multi_line = ((re_flags & LRE_FLAG_MULTILINE) != 0); s->dotall = ((re_flags & LRE_FLAG_DOTALL) != 0); + s->unicode_sets = ((re_flags & LRE_FLAG_UNICODE_SETS) != 0); s->capture_count = 1; s->total_capture_count = -1; s->has_named_captures = -1; - dbuf_init2(&s->byte_code, opaque, lre_realloc); + dbuf_init2(&s->byte_code, opaque, lre_bytecode_realloc); dbuf_init2(&s->group_names, opaque, lre_realloc); - dbuf_putc(&s->byte_code, re_flags); /* first element is the flags */ + dbuf_put_u16(&s->byte_code, re_flags); /* first element is the flags */ dbuf_putc(&s->byte_code, 0); /* second element is the number of captures */ dbuf_putc(&s->byte_code, 0); /* stack size */ dbuf_put_u32(&s->byte_code, 0); /* bytecode length */ @@ -1801,7 +2528,8 @@ uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size, /* add the named groups if needed */ if (s->group_names.size > (s->capture_count - 1)) { dbuf_put(&s->byte_code, s->group_names.buf, s->group_names.size); - s->byte_code.buf[RE_HEADER_FLAGS] |= LRE_FLAG_NAMED_GROUPS; + put_u16(s->byte_code.buf + RE_HEADER_FLAGS, + lre_get_flags(s->byte_code.buf) | LRE_FLAG_NAMED_GROUPS); } dbuf_free(&s->group_names); @@ -1935,8 +2663,6 @@ typedef struct { int cbuf_type; int capture_count; int stack_size_max; - BOOL multi_line; - BOOL ignore_case; BOOL is_unicode; int interrupt_counter; void *opaque; /* used for stack overflow check */ @@ -2085,17 +2811,19 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, } break; case REOP_char32: + case REOP_char32_i: val = get_u32(pc); pc += 4; goto test_char; case REOP_char: + case REOP_char_i: val = get_u16(pc); pc += 2; test_char: if (cptr >= cbuf_end) goto no_match; GET_CHAR(c, cptr, cbuf_end, cbuf_type); - if (s->ignore_case) { + if (opcode == REOP_char_i || opcode == REOP_char32_i) { c = lre_canonicalize(c, s->is_unicode); } if (val != c) @@ -2139,18 +2867,20 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, return LRE_RET_TIMEOUT; break; case REOP_line_start: + case REOP_line_start_m: if (cptr == s->cbuf) break; - if (!s->multi_line) + if (opcode == REOP_line_start) goto no_match; PEEK_PREV_CHAR(c, cptr, s->cbuf, cbuf_type); if (!is_line_terminator(c)) goto no_match; break; case REOP_line_end: + case REOP_line_end_m: if (cptr == cbuf_end) break; - if (!s->multi_line) + if (opcode == REOP_line_end) goto no_match; PEEK_CHAR(c, cptr, cbuf_end, cbuf_type); if (!is_line_terminator(c)) @@ -2213,14 +2943,20 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, goto no_match; break; case REOP_word_boundary: + case REOP_word_boundary_i: case REOP_not_word_boundary: + case REOP_not_word_boundary_i: { BOOL v1, v2; + int ignore_case = (opcode == REOP_word_boundary_i || opcode == REOP_not_word_boundary_i); + BOOL is_boundary = (opcode == REOP_word_boundary || opcode == REOP_word_boundary_i); /* char before */ if (cptr == s->cbuf) { v1 = FALSE; } else { PEEK_PREV_CHAR(c, cptr, s->cbuf, cbuf_type); + if (ignore_case) + c = lre_canonicalize(c, s->is_unicode); v1 = is_word_char(c); } /* current char */ @@ -2228,14 +2964,18 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, v2 = FALSE; } else { PEEK_CHAR(c, cptr, cbuf_end, cbuf_type); + if (ignore_case) + c = lre_canonicalize(c, s->is_unicode); v2 = is_word_char(c); } - if (v1 ^ v2 ^ (REOP_not_word_boundary - opcode)) + if (v1 ^ v2 ^ is_boundary) goto no_match; } break; case REOP_back_reference: + case REOP_back_reference_i: case REOP_backward_back_reference: + case REOP_backward_back_reference_i: { const uint8_t *cptr1, *cptr1_end, *cptr1_start; uint32_t c1, c2; @@ -2247,14 +2987,15 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, cptr1_end = capture[2 * val + 1]; if (!cptr1_start || !cptr1_end) break; - if (opcode == REOP_back_reference) { + if (opcode == REOP_back_reference || + opcode == REOP_back_reference_i) { cptr1 = cptr1_start; while (cptr1 < cptr1_end) { if (cptr >= cbuf_end) goto no_match; GET_CHAR(c1, cptr1, cptr1_end, cbuf_type); GET_CHAR(c2, cptr, cbuf_end, cbuf_type); - if (s->ignore_case) { + if (opcode == REOP_back_reference_i) { c1 = lre_canonicalize(c1, s->is_unicode); c2 = lre_canonicalize(c2, s->is_unicode); } @@ -2268,7 +3009,7 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, goto no_match; GET_PREV_CHAR(c1, cptr1, cptr1_start, cbuf_type); GET_PREV_CHAR(c2, cptr, s->cbuf, cbuf_type); - if (s->ignore_case) { + if (opcode == REOP_backward_back_reference_i) { c1 = lre_canonicalize(c1, s->is_unicode); c2 = lre_canonicalize(c2, s->is_unicode); } @@ -2279,6 +3020,7 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, } break; case REOP_range: + case REOP_range_i: { int n; uint32_t low, high, idx_min, idx_max, idx; @@ -2288,7 +3030,7 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, if (cptr >= cbuf_end) goto no_match; GET_CHAR(c, cptr, cbuf_end, cbuf_type); - if (s->ignore_case) { + if (opcode == REOP_range_i) { c = lre_canonicalize(c, s->is_unicode); } idx_min = 0; @@ -2319,6 +3061,7 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, } break; case REOP_range32: + case REOP_range32_i: { int n; uint32_t low, high, idx_min, idx_max, idx; @@ -2328,7 +3071,7 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, if (cptr >= cbuf_end) goto no_match; GET_CHAR(c, cptr, cbuf_end, cbuf_type); - if (s->ignore_case) { + if (opcode == REOP_range32_i) { c = lre_canonicalize(c, s->is_unicode); } idx_min = 0; @@ -2420,11 +3163,10 @@ int lre_exec(uint8_t **capture, REExecContext s_s, *s = &s_s; int re_flags, i, alloca_size, ret; StackInt *stack_buf; + const uint8_t *cptr; re_flags = lre_get_flags(bc_buf); - s->multi_line = (re_flags & LRE_FLAG_MULTILINE) != 0; - s->ignore_case = (re_flags & LRE_FLAG_IGNORECASE) != 0; - s->is_unicode = (re_flags & LRE_FLAG_UNICODE) != 0; + s->is_unicode = (re_flags & (LRE_FLAG_UNICODE | LRE_FLAG_UNICODE_SETS)) != 0; s->capture_count = bc_buf[RE_HEADER_CAPTURE_COUNT]; s->stack_size_max = bc_buf[RE_HEADER_STACK_SIZE]; s->cbuf = cbuf; @@ -2446,8 +3188,17 @@ int lre_exec(uint8_t **capture, capture[i] = NULL; alloca_size = s->stack_size_max * sizeof(stack_buf[0]); stack_buf = alloca(alloca_size); + + cptr = cbuf + (cindex << cbuf_type); + if (0 < cindex && cindex < clen && s->cbuf_type == 2) { + const uint16_t *p = (const uint16_t *)cptr; + if (is_lo_surrogate(*p) && is_hi_surrogate(p[-1])) { + cptr = (const uint8_t *)(p - 1); + } + } + ret = lre_exec_backtrack(s, capture, stack_buf, 0, bc_buf + RE_HEADER_LEN, - cbuf + (cindex << cbuf_type), FALSE); + cptr, FALSE); lre_realloc(s->opaque, s->state_stack, 0); return ret; } @@ -2459,7 +3210,7 @@ int lre_get_capture_count(const uint8_t *bc_buf) int lre_get_flags(const uint8_t *bc_buf) { - return bc_buf[RE_HEADER_FLAGS]; + return get_u16(bc_buf + RE_HEADER_FLAGS); } /* Return NULL if no group names. Otherwise, return a pointer to diff --git a/src/couch_quickjs/quickjs/libregexp.h b/src/couch_quickjs/quickjs/libregexp.h index 7475bbea95..da76e4cef6 100644 --- a/src/couch_quickjs/quickjs/libregexp.h +++ b/src/couch_quickjs/quickjs/libregexp.h @@ -35,6 +35,7 @@ #define LRE_FLAG_STICKY (1 << 5) #define LRE_FLAG_INDICES (1 << 6) /* Unused by libregexp, just recorded. */ #define LRE_FLAG_NAMED_GROUPS (1 << 7) /* named groups are present in the regexp */ +#define LRE_FLAG_UNICODE_SETS (1 << 8) #define LRE_RET_MEMORY_ERROR (-1) #define LRE_RET_TIMEOUT (-2) diff --git a/src/couch_quickjs/quickjs/libunicode-table.h b/src/couch_quickjs/quickjs/libunicode-table.h index dc46f16df6..67df6b3a3c 100644 --- a/src/couch_quickjs/quickjs/libunicode-table.h +++ b/src/couch_quickjs/quickjs/libunicode-table.h @@ -3130,6 +3130,7 @@ typedef enum { } UnicodeScriptEnum; static const char unicode_script_name_table[] = + "Unknown,Zzzz" "\0" "Adlam,Adlm" "\0" "Ahom,Ahom" "\0" "Anatolian_Hieroglyphs,Hluw" "\0" @@ -4054,6 +4055,89 @@ static const uint8_t unicode_prop_Changes_When_NFKC_Casefolded1_table[450] = { 0x4f, 0xff, }; +static const uint8_t unicode_prop_Basic_Emoji1_table[143] = { + 0x60, 0x23, 0x19, 0x81, 0x40, 0xcc, 0x1a, 0x01, + 0x80, 0x42, 0x08, 0x81, 0x94, 0x81, 0xb1, 0x8b, + 0xaa, 0x80, 0x92, 0x80, 0x8c, 0x07, 0x81, 0x90, + 0x0c, 0x0f, 0x04, 0x80, 0x94, 0x06, 0x08, 0x03, + 0x01, 0x06, 0x03, 0x81, 0x9b, 0x80, 0xa2, 0x00, + 0x03, 0x10, 0x80, 0xbc, 0x82, 0x97, 0x80, 0x8d, + 0x80, 0x43, 0x5a, 0x81, 0xb2, 0x03, 0x80, 0x61, + 0xc4, 0xad, 0x80, 0x40, 0xc9, 0x80, 0x40, 0xbd, + 0x01, 0x89, 0xe5, 0x80, 0x97, 0x80, 0x93, 0x01, + 0x20, 0x82, 0x94, 0x81, 0x40, 0xad, 0xa0, 0x8b, + 0x88, 0x80, 0xc5, 0x80, 0x95, 0x8b, 0xaa, 0x1c, + 0x8b, 0x90, 0x10, 0x82, 0xc6, 0x00, 0x80, 0x40, + 0xba, 0x81, 0xbe, 0x8c, 0x18, 0x97, 0x91, 0x80, + 0x99, 0x81, 0x8c, 0x80, 0xd5, 0xd4, 0xaf, 0xc5, + 0x28, 0x12, 0x0a, 0x1b, 0x8a, 0x0e, 0x88, 0x40, + 0xe2, 0x8b, 0x18, 0x41, 0x1a, 0xae, 0x80, 0x89, + 0x80, 0x40, 0xb8, 0xef, 0x8c, 0x82, 0x89, 0x84, + 0xb7, 0x86, 0x8e, 0x81, 0x8a, 0x85, 0x88, +}; + +static const uint8_t unicode_prop_Basic_Emoji2_table[183] = { + 0x40, 0xa8, 0x03, 0x80, 0x5f, 0x8c, 0x80, 0x8b, + 0x80, 0x40, 0xd7, 0x80, 0x95, 0x80, 0xd9, 0x85, + 0x8e, 0x81, 0x41, 0x7c, 0x80, 0x40, 0xa5, 0x80, + 0x9c, 0x10, 0x0c, 0x82, 0x40, 0xc6, 0x80, 0x40, + 0xe6, 0x81, 0x89, 0x80, 0x88, 0x80, 0xb9, 0x0a, + 0x84, 0x88, 0x01, 0x05, 0x03, 0x01, 0x00, 0x09, + 0x02, 0x02, 0x0f, 0x14, 0x00, 0x80, 0x9b, 0x09, + 0x00, 0x08, 0x80, 0x91, 0x01, 0x80, 0x92, 0x00, + 0x18, 0x00, 0x0a, 0x05, 0x07, 0x81, 0x95, 0x05, + 0x00, 0x00, 0x80, 0x94, 0x05, 0x09, 0x01, 0x17, + 0x04, 0x09, 0x08, 0x01, 0x00, 0x00, 0x05, 0x02, + 0x80, 0x90, 0x81, 0x8e, 0x01, 0x80, 0x9a, 0x81, + 0xbb, 0x80, 0x41, 0x91, 0x81, 0x41, 0xce, 0x82, + 0x45, 0x27, 0x80, 0x8b, 0x80, 0x42, 0x58, 0x00, + 0x80, 0x61, 0xbe, 0xd5, 0x81, 0x8b, 0x81, 0x40, + 0x81, 0x80, 0xb3, 0x80, 0x40, 0xe8, 0x01, 0x88, + 0x88, 0x80, 0xc5, 0x80, 0x97, 0x08, 0x11, 0x81, + 0xaa, 0x1c, 0x8b, 0x92, 0x00, 0x00, 0x80, 0xc6, + 0x00, 0x80, 0x40, 0xba, 0x80, 0xca, 0x81, 0xa3, + 0x09, 0x86, 0x8c, 0x01, 0x19, 0x80, 0x93, 0x01, + 0x07, 0x81, 0x88, 0x04, 0x82, 0x8b, 0x17, 0x11, + 0x00, 0x03, 0x05, 0x02, 0x05, 0x80, 0x40, 0xcf, + 0x00, 0x82, 0x8f, 0x2a, 0x05, 0x01, 0x80, +}; + +static const uint8_t unicode_prop_RGI_Emoji_Modifier_Sequence_table[73] = { + 0x60, 0x26, 0x1c, 0x80, 0x40, 0xda, 0x80, 0x8f, + 0x83, 0x61, 0xcc, 0x76, 0x80, 0xbb, 0x11, 0x01, + 0x82, 0xf4, 0x09, 0x8a, 0x94, 0x18, 0x18, 0x88, + 0x10, 0x1a, 0x02, 0x30, 0x00, 0x97, 0x80, 0x40, + 0xc8, 0x0b, 0x80, 0x94, 0x03, 0x81, 0x40, 0xad, + 0x12, 0x84, 0xd2, 0x80, 0x8f, 0x82, 0x88, 0x80, + 0x8a, 0x80, 0x42, 0x3e, 0x01, 0x07, 0x3d, 0x80, + 0x88, 0x89, 0x11, 0xb7, 0x80, 0xbc, 0x08, 0x08, + 0x80, 0x90, 0x10, 0x8c, 0x40, 0xe4, 0x82, 0xa9, + 0x88, +}; + +static const uint8_t unicode_prop_RGI_Emoji_Flag_Sequence_table[128] = { + 0x0c, 0x00, 0x09, 0x00, 0x04, 0x01, 0x02, 0x06, + 0x03, 0x03, 0x01, 0x02, 0x01, 0x03, 0x07, 0x0d, + 0x18, 0x00, 0x09, 0x00, 0x00, 0x89, 0x08, 0x00, + 0x00, 0x81, 0x88, 0x83, 0x8c, 0x10, 0x00, 0x01, + 0x07, 0x08, 0x29, 0x10, 0x28, 0x00, 0x80, 0x8a, + 0x00, 0x0a, 0x00, 0x0e, 0x15, 0x18, 0x83, 0x89, + 0x06, 0x00, 0x81, 0x8d, 0x00, 0x12, 0x08, 0x00, + 0x03, 0x00, 0x24, 0x00, 0x05, 0x21, 0x00, 0x00, + 0x29, 0x90, 0x00, 0x02, 0x00, 0x08, 0x09, 0x00, + 0x08, 0x18, 0x8b, 0x80, 0x8c, 0x02, 0x19, 0x1a, + 0x11, 0x00, 0x00, 0x80, 0x9c, 0x80, 0x88, 0x02, + 0x00, 0x00, 0x02, 0x20, 0x88, 0x0a, 0x00, 0x03, + 0x01, 0x02, 0x05, 0x08, 0x00, 0x01, 0x09, 0x20, + 0x21, 0x18, 0x22, 0x00, 0x00, 0x00, 0x00, 0x18, + 0x28, 0x89, 0x80, 0x8b, 0x80, 0x90, 0x80, 0x92, + 0x80, 0x8d, 0x05, 0x80, 0x8a, 0x80, 0x88, 0x80, +}; + +static const uint8_t unicode_prop_Emoji_Keycap_Sequence_table[4] = { + 0xa2, 0x05, 0x04, 0x89, +}; + static const uint8_t unicode_prop_ASCII_Hex_Digit_table[5] = { 0xaf, 0x89, 0x35, 0x99, 0x85, }; @@ -4493,6 +4577,11 @@ typedef enum { UNICODE_PROP_Changes_When_Titlecased1, UNICODE_PROP_Changes_When_Casefolded1, UNICODE_PROP_Changes_When_NFKC_Casefolded1, + UNICODE_PROP_Basic_Emoji1, + UNICODE_PROP_Basic_Emoji2, + UNICODE_PROP_RGI_Emoji_Modifier_Sequence, + UNICODE_PROP_RGI_Emoji_Flag_Sequence, + UNICODE_PROP_Emoji_Keycap_Sequence, UNICODE_PROP_ASCII_Hex_Digit, UNICODE_PROP_Bidi_Control, UNICODE_PROP_Dash, @@ -4633,6 +4722,11 @@ static const uint8_t * const unicode_prop_table[] = { unicode_prop_Changes_When_Titlecased1_table, unicode_prop_Changes_When_Casefolded1_table, unicode_prop_Changes_When_NFKC_Casefolded1_table, + unicode_prop_Basic_Emoji1_table, + unicode_prop_Basic_Emoji2_table, + unicode_prop_RGI_Emoji_Modifier_Sequence_table, + unicode_prop_RGI_Emoji_Flag_Sequence_table, + unicode_prop_Emoji_Keycap_Sequence_table, unicode_prop_ASCII_Hex_Digit_table, unicode_prop_Bidi_Control_table, unicode_prop_Dash_table, @@ -4688,6 +4782,11 @@ static const uint16_t unicode_prop_len_table[] = { countof(unicode_prop_Changes_When_Titlecased1_table), countof(unicode_prop_Changes_When_Casefolded1_table), countof(unicode_prop_Changes_When_NFKC_Casefolded1_table), + countof(unicode_prop_Basic_Emoji1_table), + countof(unicode_prop_Basic_Emoji2_table), + countof(unicode_prop_RGI_Emoji_Modifier_Sequence_table), + countof(unicode_prop_RGI_Emoji_Flag_Sequence_table), + countof(unicode_prop_Emoji_Keycap_Sequence_table), countof(unicode_prop_ASCII_Hex_Digit_table), countof(unicode_prop_Bidi_Control_table), countof(unicode_prop_Dash_table), @@ -4726,5 +4825,325 @@ static const uint16_t unicode_prop_len_table[] = { countof(unicode_prop_Case_Ignorable_table), }; +typedef enum { + UNICODE_SEQUENCE_PROP_Basic_Emoji, + UNICODE_SEQUENCE_PROP_Emoji_Keycap_Sequence, + UNICODE_SEQUENCE_PROP_RGI_Emoji_Modifier_Sequence, + UNICODE_SEQUENCE_PROP_RGI_Emoji_Flag_Sequence, + UNICODE_SEQUENCE_PROP_RGI_Emoji_Tag_Sequence, + UNICODE_SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence, + UNICODE_SEQUENCE_PROP_RGI_Emoji, + UNICODE_SEQUENCE_PROP_COUNT, +} UnicodeSequencePropertyEnum; + +static const char unicode_sequence_prop_name_table[] = + "Basic_Emoji" "\0" + "Emoji_Keycap_Sequence" "\0" + "RGI_Emoji_Modifier_Sequence" "\0" + "RGI_Emoji_Flag_Sequence" "\0" + "RGI_Emoji_Tag_Sequence" "\0" + "RGI_Emoji_ZWJ_Sequence" "\0" + "RGI_Emoji" "\0" +; + +static const uint8_t unicode_rgi_emoji_tag_sequence[18] = { + 0x67, 0x62, 0x65, 0x6e, 0x67, 0x00, 0x67, 0x62, + 0x73, 0x63, 0x74, 0x00, 0x67, 0x62, 0x77, 0x6c, + 0x73, 0x00, +}; + +static const uint8_t unicode_rgi_emoji_zwj_sequence[2320] = { + 0x02, 0xb8, 0x19, 0x40, 0x86, 0x02, 0xd1, 0x39, + 0xb0, 0x19, 0x02, 0x26, 0x39, 0x42, 0x86, 0x02, + 0xb4, 0x36, 0x42, 0x86, 0x03, 0x68, 0x54, 0x64, + 0x87, 0x68, 0x54, 0x02, 0xdc, 0x39, 0x42, 0x86, + 0x02, 0xd1, 0x39, 0x73, 0x13, 0x02, 0x39, 0x39, + 0x40, 0x86, 0x02, 0x69, 0x34, 0xbd, 0x19, 0x03, + 0xb6, 0x36, 0x40, 0x86, 0xa1, 0x87, 0x03, 0x68, + 0x74, 0x1d, 0x19, 0x68, 0x74, 0x03, 0x68, 0x34, + 0xbd, 0x19, 0xa1, 0x87, 0x02, 0xf1, 0x7a, 0xf2, + 0x7a, 0x02, 0xca, 0x33, 0x42, 0x86, 0x02, 0x69, + 0x34, 0xb0, 0x19, 0x04, 0x68, 0x14, 0x68, 0x14, + 0x67, 0x14, 0x66, 0x14, 0x02, 0xf9, 0x26, 0x42, + 0x86, 0x03, 0x69, 0x74, 0x1d, 0x19, 0x69, 0x74, + 0x03, 0xd1, 0x19, 0xbc, 0x19, 0xa1, 0x87, 0x02, + 0x3c, 0x19, 0x40, 0x86, 0x02, 0x68, 0x34, 0xeb, + 0x13, 0x02, 0xc3, 0x33, 0xa1, 0x87, 0x02, 0x70, + 0x34, 0x40, 0x86, 0x02, 0xd4, 0x39, 0x42, 0x86, + 0x02, 0xcf, 0x39, 0x42, 0x86, 0x02, 0x47, 0x36, + 0x40, 0x86, 0x02, 0x39, 0x39, 0x42, 0x86, 0x04, + 0xd1, 0x79, 0x64, 0x87, 0x8b, 0x14, 0xd1, 0x79, + 0x02, 0xd1, 0x39, 0x95, 0x86, 0x02, 0x68, 0x34, + 0x93, 0x13, 0x02, 0x69, 0x34, 0xed, 0x13, 0x02, + 0xda, 0x39, 0x40, 0x86, 0x03, 0x69, 0x34, 0xaf, + 0x19, 0xa1, 0x87, 0x02, 0xd1, 0x39, 0x93, 0x13, + 0x03, 0xce, 0x39, 0x42, 0x86, 0xa1, 0x87, 0x03, + 0xd1, 0x79, 0x64, 0x87, 0xd1, 0x79, 0x03, 0xc3, + 0x33, 0x42, 0x86, 0xa1, 0x87, 0x03, 0x69, 0x74, + 0x1d, 0x19, 0x68, 0x74, 0x02, 0x69, 0x34, 0x92, + 0x16, 0x02, 0xd1, 0x39, 0x96, 0x86, 0x04, 0x69, + 0x14, 0x64, 0x87, 0x8b, 0x14, 0x68, 0x14, 0x02, + 0x68, 0x34, 0x7c, 0x13, 0x02, 0x47, 0x36, 0x42, + 0x86, 0x02, 0x86, 0x34, 0x42, 0x86, 0x02, 0xd1, + 0x39, 0x7c, 0x13, 0x02, 0x69, 0x14, 0xa4, 0x13, + 0x02, 0xda, 0x39, 0x42, 0x86, 0x02, 0x37, 0x39, + 0x40, 0x86, 0x02, 0xd1, 0x39, 0x08, 0x87, 0x04, + 0x68, 0x54, 0x64, 0x87, 0x8b, 0x14, 0x68, 0x54, + 0x02, 0x4d, 0x36, 0x40, 0x86, 0x02, 0x68, 0x34, + 0x2c, 0x15, 0x02, 0x69, 0x34, 0xaf, 0x19, 0x02, + 0x6e, 0x34, 0x40, 0x86, 0x02, 0xcd, 0x39, 0x42, + 0x86, 0x02, 0xd1, 0x39, 0x2c, 0x15, 0x02, 0x6f, + 0x14, 0x40, 0x86, 0x03, 0xd1, 0x39, 0xbc, 0x19, + 0xa1, 0x87, 0x02, 0x68, 0x34, 0xa8, 0x13, 0x02, + 0x69, 0x34, 0x73, 0x13, 0x04, 0x69, 0x54, 0x64, + 0x87, 0x8b, 0x14, 0x68, 0x54, 0x02, 0x71, 0x34, + 0x42, 0x86, 0x02, 0xd1, 0x39, 0xa8, 0x13, 0x02, + 0x45, 0x36, 0x40, 0x86, 0x03, 0x69, 0x54, 0x64, + 0x87, 0x68, 0x54, 0x03, 0x69, 0x54, 0x64, 0x87, + 0x69, 0x54, 0x03, 0xce, 0x39, 0x40, 0x86, 0xa1, + 0x87, 0x02, 0xd8, 0x39, 0x40, 0x86, 0x03, 0xc3, + 0x33, 0x40, 0x86, 0xa1, 0x87, 0x02, 0x4d, 0x36, + 0x42, 0x86, 0x02, 0xd1, 0x19, 0x92, 0x16, 0x02, + 0xd1, 0x39, 0xeb, 0x13, 0x02, 0x68, 0x34, 0xbc, + 0x14, 0x02, 0xd1, 0x39, 0xbc, 0x14, 0x02, 0x3d, + 0x39, 0x40, 0x86, 0x02, 0xb8, 0x39, 0x42, 0x86, + 0x02, 0xa3, 0x36, 0x40, 0x86, 0x02, 0x75, 0x35, + 0x40, 0x86, 0x02, 0xd8, 0x39, 0x42, 0x86, 0x02, + 0x69, 0x34, 0x93, 0x13, 0x02, 0x35, 0x39, 0x40, + 0x86, 0x02, 0x4b, 0x36, 0x40, 0x86, 0x02, 0x3d, + 0x39, 0x42, 0x86, 0x02, 0x38, 0x39, 0x42, 0x86, + 0x02, 0xa3, 0x36, 0x42, 0x86, 0x03, 0x69, 0x14, + 0x67, 0x14, 0x67, 0x14, 0x02, 0xb6, 0x36, 0x40, + 0x86, 0x02, 0x69, 0x34, 0x7c, 0x13, 0x02, 0x75, + 0x35, 0x42, 0x86, 0x02, 0xcc, 0x93, 0x40, 0x86, + 0x02, 0xcc, 0x33, 0x40, 0x86, 0x03, 0xd1, 0x39, + 0xbd, 0x19, 0xa1, 0x87, 0x02, 0x82, 0x34, 0x40, + 0x86, 0x02, 0x87, 0x34, 0x40, 0x86, 0x02, 0x69, + 0x14, 0x3e, 0x13, 0x02, 0xd6, 0x39, 0x40, 0x86, + 0x02, 0x68, 0x14, 0xbd, 0x19, 0x02, 0x46, 0x36, + 0x42, 0x86, 0x02, 0x4b, 0x36, 0x42, 0x86, 0x02, + 0x69, 0x34, 0x2c, 0x15, 0x03, 0xb6, 0x36, 0x42, + 0x86, 0xa1, 0x87, 0x02, 0xc4, 0x33, 0x40, 0x86, + 0x02, 0x26, 0x19, 0x40, 0x86, 0x02, 0x69, 0x14, + 0xb0, 0x19, 0x02, 0xde, 0x19, 0x42, 0x86, 0x02, + 0x69, 0x34, 0xa8, 0x13, 0x02, 0xcc, 0x33, 0x42, + 0x86, 0x02, 0x82, 0x34, 0x42, 0x86, 0x02, 0xd1, + 0x19, 0x93, 0x13, 0x02, 0x81, 0x14, 0x42, 0x86, + 0x02, 0x69, 0x34, 0x95, 0x86, 0x02, 0x68, 0x34, + 0xbb, 0x14, 0x02, 0xd1, 0x39, 0xbb, 0x14, 0x02, + 0x69, 0x34, 0xeb, 0x13, 0x02, 0xd1, 0x39, 0x84, + 0x13, 0x02, 0x69, 0x34, 0xbc, 0x14, 0x04, 0x69, + 0x54, 0x64, 0x87, 0x8b, 0x14, 0x69, 0x54, 0x02, + 0x26, 0x39, 0x40, 0x86, 0x02, 0xb4, 0x36, 0x40, + 0x86, 0x02, 0x47, 0x16, 0x42, 0x86, 0x02, 0xdc, + 0x39, 0x40, 0x86, 0x02, 0xca, 0x33, 0x40, 0x86, + 0x02, 0xf9, 0x26, 0x40, 0x86, 0x02, 0x69, 0x34, + 0x08, 0x87, 0x03, 0x69, 0x14, 0x69, 0x14, 0x66, + 0x14, 0x03, 0xd1, 0x59, 0x1d, 0x19, 0xd1, 0x59, + 0x02, 0xd4, 0x39, 0x40, 0x86, 0x02, 0xcf, 0x39, + 0x40, 0x86, 0x02, 0x68, 0x34, 0xa4, 0x13, 0x02, + 0xd1, 0x39, 0xa4, 0x13, 0x02, 0xd1, 0x19, 0xa8, + 0x13, 0x02, 0xd7, 0x39, 0x42, 0x86, 0x03, 0x69, + 0x34, 0xbc, 0x19, 0xa1, 0x87, 0x02, 0x68, 0x14, + 0xb0, 0x19, 0x02, 0x68, 0x14, 0x73, 0x13, 0x04, + 0x69, 0x14, 0x69, 0x14, 0x66, 0x14, 0x66, 0x14, + 0x03, 0x68, 0x34, 0xaf, 0x19, 0xa1, 0x87, 0x02, + 0x68, 0x34, 0x80, 0x16, 0x02, 0x73, 0x34, 0x42, + 0x86, 0x02, 0xd1, 0x39, 0x80, 0x16, 0x02, 0x68, + 0x34, 0xb0, 0x19, 0x02, 0x86, 0x34, 0x40, 0x86, + 0x02, 0x38, 0x19, 0x42, 0x86, 0x02, 0x69, 0x34, + 0xbb, 0x14, 0x02, 0xb5, 0x36, 0x42, 0x86, 0x02, + 0xcd, 0x39, 0x40, 0x86, 0x02, 0x68, 0x34, 0x95, + 0x86, 0x02, 0x68, 0x34, 0x27, 0x15, 0x03, 0x68, + 0x14, 0x68, 0x14, 0x66, 0x14, 0x02, 0x71, 0x34, + 0x40, 0x86, 0x02, 0xd1, 0x39, 0x27, 0x15, 0x02, + 0x2e, 0x16, 0xa8, 0x14, 0x02, 0xc3, 0x33, 0x42, + 0x86, 0x02, 0x69, 0x14, 0x66, 0x14, 0x02, 0x68, + 0x34, 0x96, 0x86, 0x02, 0x69, 0x34, 0xa4, 0x13, + 0x03, 0x69, 0x14, 0x64, 0x87, 0x68, 0x14, 0x02, + 0xb8, 0x39, 0x40, 0x86, 0x02, 0x68, 0x34, 0x3e, + 0x13, 0x03, 0xd1, 0x19, 0xaf, 0x19, 0xa1, 0x87, + 0x02, 0xd1, 0x39, 0x3e, 0x13, 0x02, 0x68, 0x34, + 0xbd, 0x19, 0x02, 0xd1, 0x19, 0xbb, 0x14, 0x02, + 0xd1, 0x19, 0x95, 0x86, 0x02, 0xdb, 0x39, 0x42, + 0x86, 0x02, 0x38, 0x39, 0x40, 0x86, 0x02, 0x69, + 0x34, 0x80, 0x16, 0x02, 0x69, 0x14, 0xeb, 0x13, + 0x04, 0x68, 0x14, 0x69, 0x14, 0x67, 0x14, 0x67, + 0x14, 0x02, 0x77, 0x34, 0x42, 0x86, 0x02, 0x46, + 0x36, 0x40, 0x86, 0x02, 0x68, 0x34, 0x92, 0x16, + 0x02, 0x4e, 0x36, 0x42, 0x86, 0x03, 0x69, 0x14, + 0xbd, 0x19, 0xa1, 0x87, 0x02, 0xde, 0x19, 0x40, + 0x86, 0x02, 0x69, 0x34, 0x27, 0x15, 0x03, 0xc3, + 0x13, 0x40, 0x86, 0xa1, 0x87, 0x02, 0x81, 0x14, + 0x40, 0x86, 0x03, 0xd1, 0x39, 0xaf, 0x19, 0xa1, + 0x87, 0x02, 0x68, 0x34, 0xbc, 0x19, 0x02, 0xd1, + 0x19, 0x80, 0x16, 0x02, 0xd9, 0x39, 0x42, 0x86, + 0x02, 0xd1, 0x39, 0xbc, 0x19, 0x02, 0xdc, 0x19, + 0x42, 0x86, 0x02, 0x68, 0x34, 0x73, 0x13, 0x02, + 0x69, 0x34, 0x3e, 0x13, 0x02, 0x47, 0x16, 0x40, + 0x86, 0x02, 0xd1, 0x39, 0xbd, 0x19, 0x02, 0x3e, + 0x39, 0x42, 0x86, 0x02, 0x69, 0x14, 0x95, 0x86, + 0x02, 0x68, 0x14, 0x96, 0x86, 0x03, 0x69, 0x34, + 0xbd, 0x19, 0xa1, 0x87, 0x02, 0xd7, 0x39, 0x40, + 0x86, 0x02, 0x45, 0x16, 0x42, 0x86, 0x02, 0x68, + 0x34, 0xed, 0x13, 0x03, 0x68, 0x34, 0xbc, 0x19, + 0xa1, 0x87, 0x02, 0xd1, 0x39, 0xed, 0x13, 0x02, + 0xd1, 0x39, 0x92, 0x16, 0x02, 0x73, 0x34, 0x40, + 0x86, 0x02, 0x38, 0x19, 0x40, 0x86, 0x02, 0xb5, + 0x36, 0x40, 0x86, 0x02, 0x68, 0x34, 0xaf, 0x19, + 0x02, 0xd1, 0x39, 0xaf, 0x19, 0x02, 0x69, 0x34, + 0xbc, 0x19, 0x02, 0xb6, 0x16, 0x42, 0x86, 0x02, + 0x26, 0x14, 0x25, 0x15, 0x02, 0xc3, 0x33, 0x40, + 0x86, 0x02, 0xdd, 0x39, 0x42, 0x86, 0x02, 0xcb, + 0x93, 0x42, 0x86, 0x02, 0xcb, 0x33, 0x42, 0x86, + 0x02, 0x81, 0x34, 0x42, 0x86, 0x02, 0xce, 0x39, + 0xa1, 0x87, 0x02, 0xdb, 0x39, 0x40, 0x86, 0x02, + 0x68, 0x34, 0x08, 0x87, 0x02, 0xd1, 0x19, 0xb0, + 0x19, 0x02, 0x77, 0x34, 0x40, 0x86, 0x02, 0x4e, + 0x36, 0x40, 0x86, 0x02, 0xce, 0x39, 0x42, 0x86, + 0x02, 0x4e, 0x16, 0x42, 0x86, 0x02, 0xd9, 0x39, + 0x40, 0x86, 0x02, 0xdc, 0x19, 0x40, 0x86, 0x02, + 0x3e, 0x39, 0x40, 0x86, 0x02, 0xb9, 0x39, 0x42, + 0x86, 0x02, 0xda, 0x19, 0x42, 0x86, 0x02, 0x42, + 0x16, 0x94, 0x81, 0x02, 0x45, 0x16, 0x40, 0x86, + 0x02, 0x69, 0x14, 0xbd, 0x19, 0x02, 0x70, 0x34, + 0x42, 0x86, 0x02, 0xce, 0x19, 0xa1, 0x87, 0x02, + 0xc3, 0x13, 0x42, 0x86, 0x02, 0x68, 0x14, 0x08, + 0x87, 0x02, 0xd1, 0x19, 0x7c, 0x13, 0x02, 0x68, + 0x14, 0x92, 0x16, 0x02, 0xb6, 0x16, 0x40, 0x86, + 0x02, 0x37, 0x39, 0x42, 0x86, 0x03, 0xce, 0x19, + 0x42, 0x86, 0xa1, 0x87, 0x03, 0x68, 0x14, 0x67, + 0x14, 0x67, 0x14, 0x02, 0xdd, 0x39, 0x40, 0x86, + 0x02, 0xcf, 0x19, 0x42, 0x86, 0x02, 0xd1, 0x19, + 0x2c, 0x15, 0x02, 0x4b, 0x13, 0xe9, 0x17, 0x02, + 0x68, 0x14, 0x67, 0x14, 0x02, 0xcb, 0x93, 0x40, + 0x86, 0x02, 0x6e, 0x34, 0x42, 0x86, 0x02, 0xcb, + 0x33, 0x40, 0x86, 0x02, 0x81, 0x34, 0x40, 0x86, + 0x02, 0xb6, 0x36, 0xa1, 0x87, 0x02, 0x45, 0x36, + 0x42, 0x86, 0x02, 0xb4, 0x16, 0x42, 0x86, 0x02, + 0x69, 0x14, 0x73, 0x13, 0x04, 0x69, 0x14, 0x69, + 0x14, 0x67, 0x14, 0x66, 0x14, 0x02, 0x35, 0x39, + 0x42, 0x86, 0x02, 0x68, 0x14, 0x93, 0x13, 0x02, + 0xb6, 0x36, 0x42, 0x86, 0x03, 0x68, 0x14, 0x69, + 0x14, 0x66, 0x14, 0x02, 0xce, 0x39, 0x40, 0x86, + 0x02, 0x4e, 0x16, 0x40, 0x86, 0x02, 0x87, 0x34, + 0x42, 0x86, 0x02, 0x86, 0x14, 0x42, 0x86, 0x02, + 0xd6, 0x39, 0x42, 0x86, 0x02, 0xc4, 0x33, 0x42, + 0x86, 0x02, 0x69, 0x34, 0x96, 0x86, 0x02, 0xb9, + 0x39, 0x40, 0x86, 0x02, 0x68, 0x14, 0xa8, 0x13, + 0x02, 0xd1, 0x19, 0x84, 0x13, 0x02, 0xda, 0x19, + 0x40, 0x86, 0x02, 0xd8, 0x19, 0x42, 0x86, 0x02, + 0xc3, 0x13, 0x40, 0x86, 0x02, 0xb9, 0x19, 0x42, + 0x86, 0x02, 0x3d, 0x19, 0x42, 0x86, 0x02, 0xcf, + 0x19, 0x40, 0x86, 0x04, 0x68, 0x14, 0x68, 0x14, + 0x67, 0x14, 0x67, 0x14, 0x03, 0xd1, 0x19, 0xd1, + 0x19, 0xd2, 0x19, 0x02, 0x68, 0x14, 0xbb, 0x14, + 0x02, 0x3b, 0x14, 0x44, 0x87, 0x02, 0xd1, 0x19, + 0x27, 0x15, 0x02, 0xb4, 0x16, 0x40, 0x86, 0x02, + 0xcd, 0x19, 0x42, 0x86, 0x02, 0xd3, 0x86, 0xa5, + 0x14, 0x02, 0x70, 0x14, 0x42, 0x86, 0x03, 0xb6, + 0x16, 0x42, 0x86, 0xa1, 0x87, 0x04, 0x69, 0x14, + 0x64, 0x87, 0x8b, 0x14, 0x69, 0x14, 0x02, 0x36, + 0x16, 0x2b, 0x93, 0x02, 0x68, 0x14, 0x80, 0x16, + 0x02, 0x86, 0x14, 0x40, 0x86, 0x02, 0x08, 0x14, + 0x1b, 0x0b, 0x02, 0xd1, 0x19, 0xbc, 0x19, 0x02, + 0xca, 0x13, 0x42, 0x86, 0x02, 0x41, 0x94, 0xe8, + 0x95, 0x02, 0xd8, 0x19, 0x40, 0x86, 0x02, 0xb9, + 0x19, 0x40, 0x86, 0x02, 0xd1, 0x19, 0xed, 0x13, + 0x02, 0xf9, 0x86, 0x42, 0x86, 0x03, 0xd1, 0x19, + 0xbd, 0x19, 0xa1, 0x87, 0x02, 0x3d, 0x19, 0x40, + 0x86, 0x02, 0xd6, 0x19, 0x42, 0x86, 0x03, 0x69, + 0x14, 0x66, 0x14, 0x66, 0x14, 0x02, 0xd1, 0x19, + 0xaf, 0x19, 0x03, 0x69, 0x14, 0x69, 0x14, 0x67, + 0x14, 0x02, 0xcd, 0x19, 0x40, 0x86, 0x02, 0x70, + 0x14, 0x40, 0x86, 0x03, 0x68, 0x14, 0xbc, 0x19, + 0xa1, 0x87, 0x02, 0x6e, 0x14, 0x42, 0x86, 0x02, + 0x69, 0x14, 0x92, 0x16, 0x03, 0x68, 0x14, 0x68, + 0x14, 0x67, 0x14, 0x02, 0x69, 0x14, 0x67, 0x14, + 0x02, 0x75, 0x95, 0x42, 0x86, 0x03, 0x69, 0x14, + 0x64, 0x87, 0x69, 0x14, 0x02, 0xd1, 0x19, 0xbc, + 0x14, 0x02, 0xdf, 0x19, 0x42, 0x86, 0x02, 0xca, + 0x13, 0x40, 0x86, 0x02, 0x82, 0x14, 0x42, 0x86, + 0x02, 0x69, 0x14, 0x93, 0x13, 0x02, 0x68, 0x14, + 0x7c, 0x13, 0x02, 0xf9, 0x86, 0x40, 0x86, 0x02, + 0xd6, 0x19, 0x40, 0x86, 0x02, 0x68, 0x14, 0x2c, + 0x15, 0x02, 0x69, 0x14, 0xa8, 0x13, 0x02, 0xd4, + 0x19, 0x42, 0x86, 0x04, 0x68, 0x14, 0x69, 0x14, + 0x66, 0x14, 0x66, 0x14, 0x02, 0x77, 0x14, 0x42, + 0x86, 0x02, 0x39, 0x19, 0x42, 0x86, 0x02, 0xd1, + 0x19, 0xa4, 0x13, 0x02, 0x6e, 0x14, 0x40, 0x86, + 0x03, 0xd1, 0x19, 0xd2, 0x19, 0xd2, 0x19, 0x02, + 0x69, 0x14, 0xbb, 0x14, 0x02, 0xd1, 0x19, 0x96, + 0x86, 0x02, 0x75, 0x95, 0x40, 0x86, 0x04, 0x68, + 0x14, 0x64, 0x87, 0x8b, 0x14, 0x68, 0x14, 0x02, + 0xd1, 0x19, 0x3e, 0x13, 0x02, 0xdf, 0x19, 0x40, + 0x86, 0x02, 0x82, 0x14, 0x40, 0x86, 0x02, 0x44, + 0x13, 0xeb, 0x17, 0x02, 0xdd, 0x19, 0x42, 0x86, + 0x02, 0x69, 0x14, 0x80, 0x16, 0x03, 0x68, 0x14, + 0xaf, 0x19, 0xa1, 0x87, 0x02, 0xa3, 0x16, 0x42, + 0x86, 0x02, 0x69, 0x14, 0x96, 0x86, 0x02, 0x46, + 0x16, 0x42, 0x86, 0x02, 0xb6, 0x16, 0xa1, 0x87, + 0x02, 0x68, 0x14, 0x27, 0x15, 0x02, 0x26, 0x14, + 0x1b, 0x0b, 0x02, 0xd4, 0x19, 0x40, 0x86, 0x02, + 0x77, 0x14, 0x40, 0x86, 0x02, 0x39, 0x19, 0x40, + 0x86, 0x02, 0x37, 0x19, 0x42, 0x86, 0x03, 0x69, + 0x14, 0x67, 0x14, 0x66, 0x14, 0x03, 0xc3, 0x13, + 0x42, 0x86, 0xa1, 0x87, 0x02, 0x68, 0x14, 0xbc, + 0x19, 0x02, 0xd1, 0x19, 0xeb, 0x13, 0x04, 0x69, + 0x14, 0x69, 0x14, 0x67, 0x14, 0x67, 0x14, 0x02, + 0xd1, 0x19, 0x08, 0x87, 0x02, 0x68, 0x14, 0xed, + 0x13, 0x03, 0x69, 0x14, 0xbc, 0x19, 0xa1, 0x87, + 0x02, 0xdd, 0x19, 0x40, 0x86, 0x02, 0xc3, 0x13, + 0xa1, 0x87, 0x03, 0x68, 0x14, 0x66, 0x14, 0x66, + 0x14, 0x03, 0x68, 0x14, 0x69, 0x14, 0x67, 0x14, + 0x02, 0xa3, 0x16, 0x40, 0x86, 0x02, 0xdb, 0x19, + 0x42, 0x86, 0x02, 0x68, 0x14, 0xaf, 0x19, 0x02, + 0x46, 0x16, 0x40, 0x86, 0x02, 0x35, 0x16, 0xab, + 0x14, 0x02, 0x68, 0x14, 0x95, 0x86, 0x02, 0x42, + 0x16, 0x95, 0x81, 0x02, 0xc4, 0x13, 0x42, 0x86, + 0x02, 0x15, 0x14, 0xba, 0x19, 0x02, 0x69, 0x14, + 0x08, 0x87, 0x03, 0xd1, 0x19, 0x1d, 0x19, 0xd1, + 0x19, 0x02, 0x69, 0x14, 0x7c, 0x13, 0x02, 0x37, + 0x19, 0x40, 0x86, 0x02, 0x73, 0x14, 0x42, 0x86, + 0x02, 0x69, 0x14, 0x2c, 0x15, 0x02, 0xb5, 0x16, + 0x42, 0x86, 0x02, 0x35, 0x19, 0x42, 0x86, 0x04, + 0x68, 0x14, 0x69, 0x14, 0x67, 0x14, 0x66, 0x14, + 0x02, 0x64, 0x87, 0x25, 0x15, 0x02, 0x64, 0x87, + 0x79, 0x1a, 0x02, 0x68, 0x14, 0xbc, 0x14, 0x03, + 0xce, 0x19, 0x40, 0x86, 0xa1, 0x87, 0x02, 0x87, + 0x14, 0x42, 0x86, 0x02, 0x4d, 0x16, 0x42, 0x86, + 0x04, 0x68, 0x14, 0x68, 0x14, 0x66, 0x14, 0x66, + 0x14, 0x02, 0xdb, 0x19, 0x40, 0x86, 0x02, 0xd9, + 0x19, 0x42, 0x86, 0x02, 0xc4, 0x13, 0x40, 0x86, + 0x02, 0xd1, 0x19, 0xbd, 0x19, 0x02, 0x68, 0x14, + 0xa4, 0x13, 0x02, 0x3e, 0x19, 0x42, 0x86, 0x02, + 0xf3, 0x93, 0xa7, 0x86, 0x03, 0x69, 0x14, 0xaf, + 0x19, 0xa1, 0x87, 0x02, 0xf3, 0x93, 0x08, 0x13, + 0x02, 0xd1, 0x19, 0xd2, 0x19, 0x02, 0x73, 0x14, + 0x40, 0x86, 0x02, 0xb5, 0x16, 0x40, 0x86, 0x02, + 0x35, 0x19, 0x40, 0x86, 0x02, 0x69, 0x14, 0x27, + 0x15, 0x02, 0xce, 0x19, 0x42, 0x86, 0x02, 0x71, + 0x14, 0x42, 0x86, 0x02, 0xd1, 0x19, 0x73, 0x13, + 0x02, 0x68, 0x14, 0x3e, 0x13, 0x02, 0xf4, 0x13, + 0x20, 0x86, 0x02, 0x87, 0x14, 0x40, 0x86, 0x03, + 0xb6, 0x16, 0x40, 0x86, 0xa1, 0x87, 0x02, 0x4d, + 0x16, 0x40, 0x86, 0x02, 0x69, 0x14, 0xbc, 0x19, + 0x02, 0x4b, 0x16, 0x42, 0x86, 0x02, 0xd9, 0x19, + 0x40, 0x86, 0x02, 0x3e, 0x19, 0x40, 0x86, 0x02, + 0x69, 0x14, 0xed, 0x13, 0x02, 0xd7, 0x19, 0x42, + 0x86, 0x02, 0xb8, 0x19, 0x42, 0x86, 0x03, 0x68, + 0x14, 0x67, 0x14, 0x66, 0x14, 0x02, 0x3c, 0x19, + 0x42, 0x86, 0x02, 0x68, 0x14, 0x66, 0x14, 0x03, + 0x68, 0x14, 0x64, 0x87, 0x68, 0x14, 0x02, 0x69, + 0x14, 0xaf, 0x19, 0x02, 0xce, 0x19, 0x40, 0x86, + 0x02, 0x71, 0x14, 0x40, 0x86, 0x02, 0x68, 0x14, + 0xeb, 0x13, 0x03, 0x68, 0x14, 0xbd, 0x19, 0xa1, + 0x87, 0x02, 0x6f, 0x14, 0x42, 0x86, 0x04, 0xd1, + 0x19, 0xd1, 0x19, 0xd2, 0x19, 0xd2, 0x19, 0x02, + 0x69, 0x14, 0xbc, 0x14, 0x02, 0xcc, 0x93, 0x42, + 0x86, 0x02, 0x4b, 0x16, 0x40, 0x86, 0x02, 0x26, + 0x19, 0x42, 0x86, 0x02, 0xd7, 0x19, 0x40, 0x86, +}; + #endif /* CONFIG_ALL_UNICODE */ -/* 64 tables / 33442 bytes, 5 index / 351 bytes */ +/* 71 tables / 36311 bytes, 5 index / 351 bytes */ diff --git a/src/couch_quickjs/quickjs/libunicode.c b/src/couch_quickjs/quickjs/libunicode.c index d1bf1e91c1..0c510ccb15 100644 --- a/src/couch_quickjs/quickjs/libunicode.c +++ b/src/couch_quickjs/quickjs/libunicode.c @@ -499,6 +499,9 @@ int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len, case CR_OP_XOR: is_in = (a_idx & 1) ^ (b_idx & 1); break; + case CR_OP_SUB: + is_in = (a_idx & 1) & ((b_idx & 1) ^ 1); + break; default: abort(); } @@ -511,14 +514,14 @@ int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len, return 0; } -int cr_union1(CharRange *cr, const uint32_t *b_pt, int b_len) +int cr_op1(CharRange *cr, const uint32_t *b_pt, int b_len, int op) { CharRange a = *cr; int ret; cr->len = 0; cr->size = 0; cr->points = NULL; - ret = cr_op(cr, a.points, a.len, b_pt, b_len, CR_OP_UNION); + ret = cr_op(cr, a.points, a.len, b_pt, b_len, op); cr_free(&a); return ret; } @@ -1176,7 +1179,7 @@ int unicode_normalize(uint32_t **pdst, const uint32_t *src, int src_len, is_compat = n_type >> 1; dbuf_init2(dbuf, opaque, realloc_func); - if (dbuf_realloc(dbuf, sizeof(int) * src_len)) + if (dbuf_claim(dbuf, sizeof(int) * src_len)) goto fail; /* common case: latin1 is unaffected by NFC */ @@ -1282,8 +1285,6 @@ int unicode_script(CharRange *cr, script_idx = unicode_find_name(unicode_script_name_table, script_name); if (script_idx < 0) return -2; - /* Note: we remove the "Unknown" Script */ - script_idx += UNICODE_SCRIPT_Unknown + 1; is_common = (script_idx == UNICODE_SCRIPT_Common || script_idx == UNICODE_SCRIPT_Inherited); @@ -1313,17 +1314,21 @@ int unicode_script(CharRange *cr, n |= *p++; n += 96 + (1 << 12); } - if (type == 0) - v = 0; - else - v = *p++; c1 = c + n + 1; - if (v == script_idx) { - if (cr_add_interval(cr1, c, c1)) - goto fail; + if (type != 0) { + v = *p++; + if (v == script_idx || script_idx == UNICODE_SCRIPT_Unknown) { + if (cr_add_interval(cr1, c, c1)) + goto fail; + } } c = c1; } + if (script_idx == UNICODE_SCRIPT_Unknown) { + /* Unknown is all the characters outside scripts */ + if (cr_invert(cr1)) + goto fail; + } if (is_ext) { /* add the script extensions */ @@ -1554,6 +1559,7 @@ static int unicode_prop_ops(CharRange *cr, ...) cr2 = &stack[stack_len - 1]; cr3 = &stack[stack_len++]; cr_init(cr3, cr->mem_opaque, cr->realloc_func); + /* CR_OP_XOR may be used here */ if (cr_op(cr3, cr1->points, cr1->len, cr2->points, cr2->len, op - POP_UNION + CR_OP_UNION)) goto fail; @@ -1908,3 +1914,210 @@ BOOL lre_is_space_non_ascii(uint32_t c) } return FALSE; } + +#define SEQ_MAX_LEN 16 + +static int unicode_sequence_prop1(int seq_prop_idx, UnicodeSequencePropCB *cb, void *opaque, + CharRange *cr) +{ + int i, c, j; + uint32_t seq[SEQ_MAX_LEN]; + + switch(seq_prop_idx) { + case UNICODE_SEQUENCE_PROP_Basic_Emoji: + if (unicode_prop1(cr, UNICODE_PROP_Basic_Emoji1) < 0) + return -1; + for(i = 0; i < cr->len; i += 2) { + for(c = cr->points[i]; c < cr->points[i + 1]; c++) { + seq[0] = c; + cb(opaque, seq, 1); + } + } + + cr->len = 0; + + if (unicode_prop1(cr, UNICODE_PROP_Basic_Emoji2) < 0) + return -1; + for(i = 0; i < cr->len; i += 2) { + for(c = cr->points[i]; c < cr->points[i + 1]; c++) { + seq[0] = c; + seq[1] = 0xfe0f; + cb(opaque, seq, 2); + } + } + + break; + case UNICODE_SEQUENCE_PROP_RGI_Emoji_Modifier_Sequence: + if (unicode_prop1(cr, UNICODE_PROP_Emoji_Modifier_Base) < 0) + return -1; + for(i = 0; i < cr->len; i += 2) { + for(c = cr->points[i]; c < cr->points[i + 1]; c++) { + for(j = 0; j < 5; j++) { + seq[0] = c; + seq[1] = 0x1f3fb + j; + cb(opaque, seq, 2); + } + } + } + break; + case UNICODE_SEQUENCE_PROP_RGI_Emoji_Flag_Sequence: + if (unicode_prop1(cr, UNICODE_PROP_RGI_Emoji_Flag_Sequence) < 0) + return -1; + for(i = 0; i < cr->len; i += 2) { + for(c = cr->points[i]; c < cr->points[i + 1]; c++) { + int c0, c1; + c0 = c / 26; + c1 = c % 26; + seq[0] = 0x1F1E6 + c0; + seq[1] = 0x1F1E6 + c1; + cb(opaque, seq, 2); + } + } + break; + case UNICODE_SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence: + { + int len, code, pres, k, mod, mod_count, mod_pos[2], hc_pos, n_mod, n_hc, mod1; + int mod_idx, hc_idx, i0, i1; + const uint8_t *tab = unicode_rgi_emoji_zwj_sequence; + + for(i = 0; i < countof(unicode_rgi_emoji_zwj_sequence);) { + len = tab[i++]; + k = 0; + mod = 0; + mod_count = 0; + hc_pos = -1; + for(j = 0; j < len; j++) { + code = tab[i++]; + code |= tab[i++] << 8; + pres = code >> 15; + mod1 = (code >> 13) & 3; + code &= 0x1fff; + if (code < 0x1000) { + c = code + 0x2000; + } else { + c = 0x1f000 + (code - 0x1000); + } + if (c == 0x1f9b0) + hc_pos = k; + seq[k++] = c; + if (mod1 != 0) { + assert(mod_count < 2); + mod = mod1; + mod_pos[mod_count++] = k; + seq[k++] = 0; /* will be filled later */ + } + if (pres) { + seq[k++] = 0xfe0f; + } + if (j < len - 1) { + seq[k++] = 0x200d; + } + } + + /* genrate all the variants */ + switch(mod) { + case 1: + n_mod = 5; + break; + case 2: + n_mod = 25; + break; + case 3: + n_mod = 20; + break; + default: + n_mod = 1; + break; + } + if (hc_pos >= 0) + n_hc = 4; + else + n_hc = 1; + for(hc_idx = 0; hc_idx < n_hc; hc_idx++) { + for(mod_idx = 0; mod_idx < n_mod; mod_idx++) { + if (hc_pos >= 0) + seq[hc_pos] = 0x1f9b0 + hc_idx; + + switch(mod) { + case 1: + seq[mod_pos[0]] = 0x1f3fb + mod_idx; + break; + case 2: + case 3: + i0 = mod_idx / 5; + i1 = mod_idx % 5; + /* avoid identical values */ + if (mod == 3 && i0 >= i1) + i0++; + seq[mod_pos[0]] = 0x1f3fb + i0; + seq[mod_pos[1]] = 0x1f3fb + i1; + break; + default: + break; + } +#if 0 + for(j = 0; j < k; j++) + printf(" %04x", seq[j]); + printf("\n"); +#endif + cb(opaque, seq, k); + } + } + } + } + break; + case UNICODE_SEQUENCE_PROP_RGI_Emoji_Tag_Sequence: + { + for(i = 0; i < countof(unicode_rgi_emoji_tag_sequence);) { + j = 0; + seq[j++] = 0x1F3F4; + for(;;) { + c = unicode_rgi_emoji_tag_sequence[i++]; + if (c == 0x00) + break; + seq[j++] = 0xe0000 + c; + } + seq[j++] = 0xe007f; + cb(opaque, seq, j); + } + } + break; + case UNICODE_SEQUENCE_PROP_Emoji_Keycap_Sequence: + if (unicode_prop1(cr, UNICODE_PROP_Emoji_Keycap_Sequence) < 0) + return -1; + for(i = 0; i < cr->len; i += 2) { + for(c = cr->points[i]; c < cr->points[i + 1]; c++) { + seq[0] = c; + seq[1] = 0xfe0f; + seq[2] = 0x20e3; + cb(opaque, seq, 3); + } + } + break; + case UNICODE_SEQUENCE_PROP_RGI_Emoji: + /* all prevous sequences */ + for(i = UNICODE_SEQUENCE_PROP_Basic_Emoji; i <= UNICODE_SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence; i++) { + int ret; + ret = unicode_sequence_prop1(i, cb, opaque, cr); + if (ret < 0) + return ret; + cr->len = 0; + } + break; + default: + return -2; + } + return 0; +} + +/* build a unicode sequence property */ +/* return -2 if not found, -1 if other error. 'cr' is used as temporary memory. */ +int unicode_sequence_prop(const char *prop_name, UnicodeSequencePropCB *cb, void *opaque, + CharRange *cr) +{ + int seq_prop_idx; + seq_prop_idx = unicode_find_name(unicode_sequence_prop_name_table, prop_name); + if (seq_prop_idx < 0) + return -2; + return unicode_sequence_prop1(seq_prop_idx, cb, opaque, cr); +} diff --git a/src/couch_quickjs/quickjs/libunicode.h b/src/couch_quickjs/quickjs/libunicode.h index cc2f244c7f..5d964e40f7 100644 --- a/src/couch_quickjs/quickjs/libunicode.h +++ b/src/couch_quickjs/quickjs/libunicode.h @@ -45,6 +45,7 @@ typedef enum { CR_OP_UNION, CR_OP_INTER, CR_OP_XOR, + CR_OP_SUB, } CharRangeOpEnum; void cr_init(CharRange *cr, void *mem_opaque, void *(*realloc_func)(void *opaque, void *ptr, size_t size)); @@ -73,19 +74,18 @@ static inline int cr_add_interval(CharRange *cr, uint32_t c1, uint32_t c2) return 0; } -int cr_union1(CharRange *cr, const uint32_t *b_pt, int b_len); +int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len, + const uint32_t *b_pt, int b_len, int op); +int cr_op1(CharRange *cr, const uint32_t *b_pt, int b_len, int op); static inline int cr_union_interval(CharRange *cr, uint32_t c1, uint32_t c2) { uint32_t b_pt[2]; b_pt[0] = c1; b_pt[1] = c2 + 1; - return cr_union1(cr, b_pt, 2); + return cr_op1(cr, b_pt, 2, CR_OP_UNION); } -int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len, - const uint32_t *b_pt, int b_len, int op); - int cr_invert(CharRange *cr); int cr_regexp_canonicalize(CharRange *cr, int is_unicode); @@ -107,6 +107,10 @@ int unicode_script(CharRange *cr, const char *script_name, int is_ext); int unicode_general_category(CharRange *cr, const char *gc_name); int unicode_prop(CharRange *cr, const char *prop_name); +typedef void UnicodeSequencePropCB(void *opaque, const uint32_t *buf, int len); +int unicode_sequence_prop(const char *prop_name, UnicodeSequencePropCB *cb, void *opaque, + CharRange *cr); + int lre_case_conv(uint32_t *res, uint32_t c, int conv_type); int lre_canonicalize(uint32_t c, int is_unicode); diff --git a/src/couch_quickjs/quickjs/qjsc.c b/src/couch_quickjs/quickjs/qjsc.c index f9e1928a25..e55ca61ce6 100644 --- a/src/couch_quickjs/quickjs/qjsc.c +++ b/src/couch_quickjs/quickjs/qjsc.c @@ -170,14 +170,24 @@ static void dump_hex(FILE *f, const uint8_t *buf, size_t len) fprintf(f, "\n"); } +typedef enum { + CNAME_TYPE_SCRIPT, + CNAME_TYPE_MODULE, + CNAME_TYPE_JSON_MODULE, +} CNameTypeEnum; + static void output_object_code(JSContext *ctx, FILE *fo, JSValueConst obj, const char *c_name, - BOOL load_only) + CNameTypeEnum c_name_type) { uint8_t *out_buf; size_t out_buf_len; int flags; - flags = JS_WRITE_OBJ_BYTECODE; + + if (c_name_type == CNAME_TYPE_JSON_MODULE) + flags = 0; + else + flags = JS_WRITE_OBJ_BYTECODE; if (byte_swap) flags |= JS_WRITE_OBJ_BSWAP; out_buf = JS_WriteObject(ctx, &out_buf_len, obj, flags); @@ -186,7 +196,7 @@ static void output_object_code(JSContext *ctx, exit(1); } - namelist_add(&cname_list, c_name, NULL, load_only); + namelist_add(&cname_list, c_name, NULL, c_name_type); fprintf(fo, "const uint32_t %s_size = %u;\n\n", c_name, (unsigned int)out_buf_len); @@ -227,7 +237,8 @@ static void find_unique_cname(char *cname, size_t cname_size) } JSModuleDef *jsc_module_loader(JSContext *ctx, - const char *module_name, void *opaque) + const char *module_name, void *opaque, + JSValueConst attributes) { JSModuleDef *m; namelist_entry_t *e; @@ -249,9 +260,9 @@ JSModuleDef *jsc_module_loader(JSContext *ctx, } else { size_t buf_len; uint8_t *buf; - JSValue func_val; char cname[1024]; - + int res; + buf = js_load_file(ctx, &buf_len, module_name); if (!buf) { JS_ThrowReferenceError(ctx, "could not load module filename '%s'", @@ -259,21 +270,59 @@ JSModuleDef *jsc_module_loader(JSContext *ctx, return NULL; } - /* compile the module */ - func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name, - JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); - js_free(ctx, buf); - if (JS_IsException(func_val)) - return NULL; - get_c_name(cname, sizeof(cname), module_name); - if (namelist_find(&cname_list, cname)) { - find_unique_cname(cname, sizeof(cname)); - } - output_object_code(ctx, outfile, func_val, cname, TRUE); + res = js_module_test_json(ctx, attributes); + if (has_suffix(module_name, ".json") || res > 0) { + /* compile as JSON or JSON5 depending on "type" */ + JSValue val; + int flags; + + if (res == 2) + flags = JS_PARSE_JSON_EXT; + else + flags = 0; + val = JS_ParseJSON2(ctx, (char *)buf, buf_len, module_name, flags); + js_free(ctx, buf); + if (JS_IsException(val)) + return NULL; + /* create a dummy module */ + m = JS_NewCModule(ctx, module_name, js_module_dummy_init); + if (!m) { + JS_FreeValue(ctx, val); + return NULL; + } - /* the module is already referenced, so we must free it */ - m = JS_VALUE_GET_PTR(func_val); - JS_FreeValue(ctx, func_val); + get_c_name(cname, sizeof(cname), module_name); + if (namelist_find(&cname_list, cname)) { + find_unique_cname(cname, sizeof(cname)); + } + + /* output the module name */ + fprintf(outfile, "static const uint8_t %s_module_name[] = {\n", + cname); + dump_hex(outfile, (const uint8_t *)module_name, strlen(module_name) + 1); + fprintf(outfile, "};\n\n"); + + output_object_code(ctx, outfile, val, cname, CNAME_TYPE_JSON_MODULE); + JS_FreeValue(ctx, val); + } else { + JSValue func_val; + + /* compile the module */ + func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name, + JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + js_free(ctx, buf); + if (JS_IsException(func_val)) + return NULL; + get_c_name(cname, sizeof(cname), module_name); + if (namelist_find(&cname_list, cname)) { + find_unique_cname(cname, sizeof(cname)); + } + output_object_code(ctx, outfile, func_val, cname, CNAME_TYPE_MODULE); + + /* the module is already referenced, so we must free it */ + m = JS_VALUE_GET_PTR(func_val); + JS_FreeValue(ctx, func_val); + } } return m; } @@ -313,8 +362,11 @@ static void compile_file(JSContext *ctx, FILE *fo, pstrcpy(c_name, sizeof(c_name), c_name1); } else { get_c_name(c_name, sizeof(c_name), filename); + if (namelist_find(&cname_list, c_name)) { + find_unique_cname(c_name, sizeof(c_name)); + } } - output_object_code(ctx, fo, obj, c_name, FALSE); + output_object_code(ctx, fo, obj, c_name, CNAME_TYPE_SCRIPT); JS_FreeValue(ctx, obj); } @@ -709,7 +761,7 @@ int main(int argc, char **argv) JS_SetStripInfo(rt, strip_flags); /* loader for ES6 modules */ - JS_SetModuleLoaderFunc(rt, NULL, jsc_module_loader, NULL); + JS_SetModuleLoaderFunc2(rt, NULL, jsc_module_loader, NULL, NULL); fprintf(fo, "/* File generated automatically by the QuickJS compiler. */\n" "\n" @@ -732,7 +784,7 @@ int main(int argc, char **argv) } for(i = 0; i < dynamic_module_list.count; i++) { - if (!jsc_module_loader(ctx, dynamic_module_list.array[i].name, NULL)) { + if (!jsc_module_loader(ctx, dynamic_module_list.array[i].name, NULL, JS_UNDEFINED)) { fprintf(stderr, "Could not load dynamic module '%s'\n", dynamic_module_list.array[i].name); exit(1); @@ -770,9 +822,12 @@ int main(int argc, char **argv) } for(i = 0; i < cname_list.count; i++) { namelist_entry_t *e = &cname_list.array[i]; - if (e->flags) { + if (e->flags == CNAME_TYPE_MODULE) { fprintf(fo, " js_std_eval_binary(ctx, %s, %s_size, 1);\n", e->name, e->name); + } else if (e->flags == CNAME_TYPE_JSON_MODULE) { + fprintf(fo, " js_std_eval_binary_json_module(ctx, %s, %s_size, (const char *)%s_module_name);\n", + e->name, e->name, e->name); } } fprintf(fo, @@ -788,7 +843,7 @@ int main(int argc, char **argv) /* add the module loader if necessary */ if (feature_bitmap & (1 << FE_MODULE_LOADER)) { - fprintf(fo, " JS_SetModuleLoaderFunc(rt, NULL, js_module_loader, NULL);\n"); + fprintf(fo, " JS_SetModuleLoaderFunc2(rt, NULL, js_module_loader, js_module_check_attributes, NULL);\n"); } fprintf(fo, @@ -797,7 +852,7 @@ int main(int argc, char **argv) for(i = 0; i < cname_list.count; i++) { namelist_entry_t *e = &cname_list.array[i]; - if (!e->flags) { + if (e->flags == CNAME_TYPE_SCRIPT) { fprintf(fo, " js_std_eval_binary(ctx, %s, %s_size, 0);\n", e->name, e->name); } diff --git a/src/couch_quickjs/quickjs/quickjs-atom.h b/src/couch_quickjs/quickjs/quickjs-atom.h index 73766f23c7..23c2ed07a2 100644 --- a/src/couch_quickjs/quickjs/quickjs-atom.h +++ b/src/couch_quickjs/quickjs/quickjs-atom.h @@ -78,6 +78,8 @@ DEF(await, "await") /* empty string */ DEF(empty_string, "") /* identifiers */ +DEF(keys, "keys") +DEF(size, "size") DEF(length, "length") DEF(fileName, "fileName") DEF(lineNumber, "lineNumber") @@ -177,12 +179,19 @@ DEF(minus_zero, "-0") DEF(Infinity, "Infinity") DEF(minus_Infinity, "-Infinity") DEF(NaN, "NaN") +DEF(hasIndices, "hasIndices") +DEF(ignoreCase, "ignoreCase") +DEF(multiline, "multiline") +DEF(dotAll, "dotAll") +DEF(sticky, "sticky") +DEF(unicodeSets, "unicodeSets") /* the following 3 atoms are only used with CONFIG_ATOMICS */ DEF(not_equal, "not-equal") DEF(timed_out, "timed-out") DEF(ok, "ok") /* */ DEF(toJSON, "toJSON") +DEF(maxByteLength, "maxByteLength") /* class names */ DEF(Object, "Object") DEF(Array, "Array") @@ -211,6 +220,7 @@ DEF(Int32Array, "Int32Array") DEF(Uint32Array, "Uint32Array") DEF(BigInt64Array, "BigInt64Array") DEF(BigUint64Array, "BigUint64Array") +DEF(Float16Array, "Float16Array") DEF(Float32Array, "Float32Array") DEF(Float64Array, "Float64Array") DEF(DataView, "DataView") @@ -221,6 +231,9 @@ DEF(Map, "Map") DEF(Set, "Set") /* Map + 1 */ DEF(WeakMap, "WeakMap") /* Map + 2 */ DEF(WeakSet, "WeakSet") /* Map + 3 */ +DEF(Iterator, "Iterator") +DEF(IteratorHelper, "Iterator Helper") +DEF(IteratorWrap, "Iterator Wrap") DEF(Map_Iterator, "Map Iterator") DEF(Set_Iterator, "Set Iterator") DEF(Array_Iterator, "Array Iterator") @@ -243,6 +256,7 @@ DEF(SyntaxError, "SyntaxError") DEF(TypeError, "TypeError") DEF(URIError, "URIError") DEF(InternalError, "InternalError") +DEF(AggregateError, "AggregateError") /* private symbols */ DEF(Private_brand, "") /* symbols */ diff --git a/src/couch_quickjs/quickjs/quickjs-libc.c b/src/couch_quickjs/quickjs/quickjs-libc.c index b59c6ac55b..54a7a15bd5 100644 --- a/src/couch_quickjs/quickjs/quickjs-libc.c +++ b/src/couch_quickjs/quickjs/quickjs-libc.c @@ -136,11 +136,18 @@ typedef struct { JSValue on_message_func; } JSWorkerMessageHandler; +typedef struct { + struct list_head link; + JSValue promise; + JSValue reason; +} JSRejectedPromiseEntry; + typedef struct JSThreadState { struct list_head os_rw_handlers; /* list of JSOSRWHandler.link */ struct list_head os_signal_handlers; /* list JSOSSignalHandler.link */ struct list_head os_timers; /* list of JSOSTimer.link */ struct list_head port_list; /* list of JSWorkerMessageHandler.link */ + struct list_head rejected_promise_list; /* list of JSRejectedPromiseEntry.link */ int eval_script_recurse; /* only used in the main thread */ int next_timer_id; /* for setTimeout() */ /* not used in the main thread */ @@ -160,6 +167,7 @@ static BOOL my_isdigit(int c) return (c >= '0' && c <= '9'); } +/* XXX: use 'o' and 'O' for object using JS_PrintValue() ? */ static JSValue js_printf_internal(JSContext *ctx, int argc, JSValueConst *argv, FILE *fp) { @@ -583,17 +591,101 @@ int js_module_set_import_meta(JSContext *ctx, JSValueConst func_val, return 0; } -JSModuleDef *js_module_loader(JSContext *ctx, - const char *module_name, void *opaque) +static int json_module_init(JSContext *ctx, JSModuleDef *m) +{ + JSValue val; + val = JS_GetModulePrivateValue(ctx, m); + JS_SetModuleExport(ctx, m, "default", val); + return 0; +} + +static JSModuleDef *create_json_module(JSContext *ctx, const char *module_name, JSValue val) { JSModuleDef *m; + m = JS_NewCModule(ctx, module_name, json_module_init); + if (!m) { + JS_FreeValue(ctx, val); + return NULL; + } + /* only export the "default" symbol which will contain the JSON object */ + JS_AddModuleExport(ctx, m, "default"); + JS_SetModulePrivateValue(ctx, m, val); + return m; +} + +/* in order to conform with the specification, only the keys should be + tested and not the associated values. */ +int js_module_check_attributes(JSContext *ctx, void *opaque, + JSValueConst attributes) +{ + JSPropertyEnum *tab; + uint32_t i, len; + int ret; + const char *cstr; + size_t cstr_len; + + if (JS_GetOwnPropertyNames(ctx, &tab, &len, attributes, JS_GPN_ENUM_ONLY | JS_GPN_STRING_MASK)) + return -1; + ret = 0; + for(i = 0; i < len; i++) { + cstr = JS_AtomToCStringLen(ctx, &cstr_len, tab[i].atom); + if (!cstr) { + ret = -1; + break; + } + if (!(cstr_len == 4 && !memcmp(cstr, "type", cstr_len))) { + JS_ThrowTypeError(ctx, "import attribute '%s' is not supported", cstr); + ret = -1; + } + JS_FreeCString(ctx, cstr); + if (ret) + break; + } + JS_FreePropertyEnum(ctx, tab, len); + return ret; +} +/* return > 0 if the attributes indicate a JSON module */ +int js_module_test_json(JSContext *ctx, JSValueConst attributes) +{ + JSValue str; + const char *cstr; + size_t len; + BOOL res; + + if (JS_IsUndefined(attributes)) + return FALSE; + str = JS_GetPropertyStr(ctx, attributes, "type"); + if (!JS_IsString(str)) + return FALSE; + cstr = JS_ToCStringLen(ctx, &len, str); + JS_FreeValue(ctx, str); + if (!cstr) + return FALSE; + /* XXX: raise an error if unknown type ? */ + if (len == 4 && !memcmp(cstr, "json", len)) { + res = 1; + } else if (len == 5 && !memcmp(cstr, "json5", len)) { + res = 2; + } else { + res = 0; + } + JS_FreeCString(ctx, cstr); + return res; +} + +JSModuleDef *js_module_loader(JSContext *ctx, + const char *module_name, void *opaque, + JSValueConst attributes) +{ + JSModuleDef *m; + int res; + if (has_suffix(module_name, ".so")) { m = js_module_loader_so(ctx, module_name); } else { size_t buf_len; uint8_t *buf; - JSValue func_val; buf = js_load_file(ctx, &buf_len, module_name); if (!buf) { @@ -601,18 +693,36 @@ JSModuleDef *js_module_loader(JSContext *ctx, module_name); return NULL; } - - /* compile the module */ - func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name, - JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); - js_free(ctx, buf); - if (JS_IsException(func_val)) - return NULL; - /* XXX: could propagate the exception */ - js_module_set_import_meta(ctx, func_val, TRUE, FALSE); - /* the module is already referenced, so we must free it */ - m = JS_VALUE_GET_PTR(func_val); - JS_FreeValue(ctx, func_val); + res = js_module_test_json(ctx, attributes); + if (has_suffix(module_name, ".json") || res > 0) { + /* compile as JSON or JSON5 depending on "type" */ + JSValue val; + int flags; + if (res == 2) + flags = JS_PARSE_JSON_EXT; + else + flags = 0; + val = JS_ParseJSON2(ctx, (char *)buf, buf_len, module_name, flags); + js_free(ctx, buf); + if (JS_IsException(val)) + return NULL; + m = create_json_module(ctx, module_name, val); + if (!m) + return NULL; + } else { + JSValue func_val; + /* compile the module */ + func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name, + JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + js_free(ctx, buf); + if (JS_IsException(func_val)) + return NULL; + /* XXX: could propagate the exception */ + js_module_set_import_meta(ctx, func_val, TRUE, FALSE); + /* the module is already referenced, so we must free it */ + m = JS_VALUE_GET_PTR(func_val); + JS_FreeValue(ctx, func_val); + } } return m; } @@ -1083,10 +1193,16 @@ static JSValue js_std_file_printf(JSContext *ctx, JSValueConst this_val, return js_printf_internal(ctx, argc, argv, f); } +static void js_print_value_write(void *opaque, const char *buf, size_t len) +{ + FILE *fo = opaque; + fwrite(buf, 1, len, fo); +} + static JSValue js_std_file_printObject(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { - JS_PrintValue(ctx, stdout, argv[0], NULL); + JS_PrintValue(ctx, js_print_value_write, stdout, argv[0], NULL); return JS_UNDEFINED; } @@ -2924,9 +3040,7 @@ static char **build_envp(JSContext *ctx, JSValueConst obj) JS_FreeCString(ctx, str); } done: - for(i = 0; i < len; i++) - JS_FreeAtom(ctx, tab[i].atom); - js_free(ctx, tab); + JS_FreePropertyEnum(ctx, tab, len); return envp; fail: if (envp) { @@ -3466,7 +3580,7 @@ static void *worker_func(void *opaque) JS_SetStripInfo(rt, args->strip_flags); js_std_init_handlers(rt); - JS_SetModuleLoaderFunc(rt, NULL, js_module_loader, NULL); + JS_SetModuleLoaderFunc2(rt, NULL, js_module_loader, js_module_check_attributes, NULL); /* set the pipe to communicate with the parent */ ts = JS_GetRuntimeOpaque(rt); @@ -3914,7 +4028,7 @@ static JSValue js_print(JSContext *ctx, JSValueConst this_val, fwrite(str, 1, len, stdout); JS_FreeCString(ctx, str); } else { - JS_PrintValue(ctx, stdout, v, NULL); + JS_PrintValue(ctx, js_print_value_write, stdout, v, NULL); } } putchar('\n'); @@ -3979,6 +4093,7 @@ void js_std_init_handlers(JSRuntime *rt) init_list_head(&ts->os_signal_handlers); init_list_head(&ts->os_timers); init_list_head(&ts->port_list); + init_list_head(&ts->rejected_promise_list); ts->next_timer_id = 1; JS_SetRuntimeOpaque(rt, ts); @@ -4016,6 +4131,13 @@ void js_std_free_handlers(JSRuntime *rt) free_timer(rt, th); } + list_for_each_safe(el, el1, &ts->rejected_promise_list) { + JSRejectedPromiseEntry *rp = list_entry(el, JSRejectedPromiseEntry, link); + JS_FreeValueRT(rt, rp->promise); + JS_FreeValueRT(rt, rp->reason); + free(rp); + } + #ifdef USE_WORKER /* XXX: free port_list ? */ js_free_message_pipe(ts->recv_pipe); @@ -4028,7 +4150,7 @@ void js_std_free_handlers(JSRuntime *rt) static void js_std_dump_error1(JSContext *ctx, JSValueConst exception_val) { - JS_PrintValue(ctx, stderr, exception_val, NULL); + JS_PrintValue(ctx, js_print_value_write, stderr, exception_val, NULL); fputc('\n', stderr); } @@ -4041,13 +4163,66 @@ void js_std_dump_error(JSContext *ctx) JS_FreeValue(ctx, exception_val); } +static JSRejectedPromiseEntry *find_rejected_promise(JSContext *ctx, JSThreadState *ts, + JSValueConst promise) +{ + struct list_head *el; + + list_for_each(el, &ts->rejected_promise_list) { + JSRejectedPromiseEntry *rp = list_entry(el, JSRejectedPromiseEntry, link); + if (JS_SameValue(ctx, rp->promise, promise)) + return rp; + } + return NULL; +} + void js_std_promise_rejection_tracker(JSContext *ctx, JSValueConst promise, JSValueConst reason, BOOL is_handled, void *opaque) { + JSRuntime *rt = JS_GetRuntime(ctx); + JSThreadState *ts = JS_GetRuntimeOpaque(rt); + JSRejectedPromiseEntry *rp; + if (!is_handled) { - fprintf(stderr, "Possibly unhandled promise rejection: "); - js_std_dump_error1(ctx, reason); + /* add a new entry if needed */ + rp = find_rejected_promise(ctx, ts, promise); + if (!rp) { + rp = malloc(sizeof(*rp)); + if (rp) { + rp->promise = JS_DupValue(ctx, promise); + rp->reason = JS_DupValue(ctx, reason); + list_add_tail(&rp->link, &ts->rejected_promise_list); + } + } + } else { + /* the rejection is handled, so the entry can be removed if present */ + rp = find_rejected_promise(ctx, ts, promise); + if (rp) { + JS_FreeValue(ctx, rp->promise); + JS_FreeValue(ctx, rp->reason); + list_del(&rp->link); + free(rp); + } + } +} + +/* check if there are pending promise rejections. It must be done + asynchrously in case a rejected promise is handled later. Currently + we do it once the application is about to sleep. It could be done + more often if needed. */ +static void js_std_promise_rejection_check(JSContext *ctx) +{ + JSRuntime *rt = JS_GetRuntime(ctx); + JSThreadState *ts = JS_GetRuntimeOpaque(rt); + struct list_head *el; + + if (unlikely(!list_empty(&ts->rejected_promise_list))) { + list_for_each(el, &ts->rejected_promise_list) { + JSRejectedPromiseEntry *rp = list_entry(el, JSRejectedPromiseEntry, link); + fprintf(stderr, "Possibly unhandled promise rejection: "); + js_std_dump_error1(ctx, rp->reason); + } exit(1); } } @@ -4055,21 +4230,21 @@ void js_std_promise_rejection_tracker(JSContext *ctx, JSValueConst promise, /* main loop which calls the user JS callbacks */ void js_std_loop(JSContext *ctx) { - JSContext *ctx1; int err; for(;;) { /* execute the pending jobs */ for(;;) { - err = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1); + err = JS_ExecutePendingJob(JS_GetRuntime(ctx), NULL); if (err <= 0) { - if (err < 0) { - js_std_dump_error(ctx1); - } + if (err < 0) + js_std_dump_error(ctx); break; } } + js_std_promise_rejection_check(ctx); + if (!os_poll_func || os_poll_func(ctx)) break; } @@ -4094,13 +4269,14 @@ JSValue js_std_await(JSContext *ctx, JSValue obj) JS_FreeValue(ctx, obj); break; } else if (state == JS_PROMISE_PENDING) { - JSContext *ctx1; int err; - err = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1); + err = JS_ExecutePendingJob(JS_GetRuntime(ctx), NULL); if (err < 0) { - js_std_dump_error(ctx1); + js_std_dump_error(ctx); } if (err == 0) { + js_std_promise_rejection_check(ctx); + if (os_poll_func) os_poll_func(ctx); } @@ -4124,6 +4300,7 @@ void js_std_eval_binary(JSContext *ctx, const uint8_t *buf, size_t buf_len, if (JS_VALUE_GET_TAG(obj) == JS_TAG_MODULE) { js_module_set_import_meta(ctx, obj, FALSE, FALSE); } + JS_FreeValue(ctx, obj); } else { if (JS_VALUE_GET_TAG(obj) == JS_TAG_MODULE) { if (JS_ResolveModule(ctx, obj) < 0) { @@ -4144,3 +4321,22 @@ void js_std_eval_binary(JSContext *ctx, const uint8_t *buf, size_t buf_len, JS_FreeValue(ctx, val); } } + +void js_std_eval_binary_json_module(JSContext *ctx, + const uint8_t *buf, size_t buf_len, + const char *module_name) +{ + JSValue obj; + JSModuleDef *m; + + obj = JS_ReadObject(ctx, buf, buf_len, 0); + if (JS_IsException(obj)) + goto exception; + m = create_json_module(ctx, module_name, obj); + if (!m) { + exception: + js_std_dump_error(ctx); + exit(1); + } +} + diff --git a/src/couch_quickjs/quickjs/quickjs-libc.h b/src/couch_quickjs/quickjs/quickjs-libc.h index 850484f361..5c8301b717 100644 --- a/src/couch_quickjs/quickjs/quickjs-libc.h +++ b/src/couch_quickjs/quickjs/quickjs-libc.h @@ -44,10 +44,16 @@ void js_std_dump_error(JSContext *ctx); uint8_t *js_load_file(JSContext *ctx, size_t *pbuf_len, const char *filename); int js_module_set_import_meta(JSContext *ctx, JSValueConst func_val, JS_BOOL use_realpath, JS_BOOL is_main); +int js_module_test_json(JSContext *ctx, JSValueConst attributes); +int js_module_check_attributes(JSContext *ctx, void *opaque, JSValueConst attributes); JSModuleDef *js_module_loader(JSContext *ctx, - const char *module_name, void *opaque); + const char *module_name, void *opaque, + JSValueConst attributes); void js_std_eval_binary(JSContext *ctx, const uint8_t *buf, size_t buf_len, int flags); +void js_std_eval_binary_json_module(JSContext *ctx, + const uint8_t *buf, size_t buf_len, + const char *module_name); void js_std_promise_rejection_tracker(JSContext *ctx, JSValueConst promise, JSValueConst reason, JS_BOOL is_handled, void *opaque); diff --git a/src/couch_quickjs/quickjs/quickjs-opcode.h b/src/couch_quickjs/quickjs/quickjs-opcode.h index e721e17072..d93852133d 100644 --- a/src/couch_quickjs/quickjs/quickjs-opcode.h +++ b/src/couch_quickjs/quickjs/quickjs-opcode.h @@ -121,21 +121,16 @@ DEF( apply_eval, 3, 2, 1, u16) /* func array -> ret_eval */ DEF( regexp, 1, 2, 1, none) /* create a RegExp object from the pattern and a bytecode string */ DEF( get_super, 1, 1, 1, none) -DEF( import, 1, 1, 1, none) /* dynamic module import */ +DEF( import, 1, 2, 1, none) /* dynamic module import */ -DEF( check_var, 5, 0, 1, atom) /* check if a variable exists */ -DEF( get_var_undef, 5, 0, 1, atom) /* push undefined if the variable does not exist */ -DEF( get_var, 5, 0, 1, atom) /* throw an exception if the variable does not exist */ -DEF( put_var, 5, 1, 0, atom) /* must come after get_var */ -DEF( put_var_init, 5, 1, 0, atom) /* must come after put_var. Used to initialize a global lexical variable */ -DEF( put_var_strict, 5, 2, 0, atom) /* for strict mode variable write */ +DEF( get_var_undef, 3, 0, 1, var_ref) /* push undefined if the variable does not exist */ +DEF( get_var, 3, 0, 1, var_ref) /* throw an exception if the variable does not exist */ +DEF( put_var, 3, 1, 0, var_ref) /* must come after get_var */ +DEF( put_var_init, 3, 1, 0, var_ref) /* must come after put_var. Used to initialize a global lexical variable */ DEF( get_ref_value, 1, 2, 3, none) DEF( put_ref_value, 1, 3, 0, none) -DEF( define_var, 6, 0, 0, atom_u8) -DEF(check_define_var, 6, 0, 0, atom_u8) -DEF( define_func, 6, 1, 0, atom_u8) DEF( get_field, 5, 1, 1, atom) DEF( get_field2, 5, 1, 2, atom) DEF( put_field, 5, 2, 0, atom) diff --git a/src/couch_quickjs/quickjs/quickjs.c b/src/couch_quickjs/quickjs/quickjs.c index 5ff86eedf4..07ddb6aa1a 100644 --- a/src/couch_quickjs/quickjs/quickjs.c +++ b/src/couch_quickjs/quickjs/quickjs.c @@ -103,6 +103,7 @@ //#define DUMP_ATOMS /* dump atoms in JS_FreeContext */ //#define DUMP_SHAPES /* dump shapes in JS_FreeContext */ //#define DUMP_MODULE_RESOLVE +//#define DUMP_MODULE_EXEC //#define DUMP_PROMISE //#define DUMP_READ_OBJECT //#define DUMP_ROPE_REBALANCE @@ -147,6 +148,7 @@ enum { JS_CLASS_UINT32_ARRAY, /* u.array (typed_array) */ JS_CLASS_BIG_INT64_ARRAY, /* u.array (typed_array) */ JS_CLASS_BIG_UINT64_ARRAY, /* u.array (typed_array) */ + JS_CLASS_FLOAT16_ARRAY, /* u.array (typed_array) */ JS_CLASS_FLOAT32_ARRAY, /* u.array (typed_array) */ JS_CLASS_FLOAT64_ARRAY, /* u.array (typed_array) */ JS_CLASS_DATAVIEW, /* u.typed_array */ @@ -155,12 +157,16 @@ enum { JS_CLASS_SET, /* u.map_state */ JS_CLASS_WEAKMAP, /* u.map_state */ JS_CLASS_WEAKSET, /* u.map_state */ + JS_CLASS_ITERATOR, /* u.map_iterator_data */ + JS_CLASS_ITERATOR_HELPER, /* u.iterator_helper_data */ + JS_CLASS_ITERATOR_WRAP, /* u.iterator_wrap_data */ JS_CLASS_MAP_ITERATOR, /* u.map_iterator_data */ JS_CLASS_SET_ITERATOR, /* u.map_iterator_data */ JS_CLASS_ARRAY_ITERATOR, /* u.array_iterator_data */ JS_CLASS_STRING_ITERATOR, /* u.array_iterator_data */ JS_CLASS_REGEXP_STRING_ITERATOR, /* u.regexp_string_iterator_data */ JS_CLASS_GENERATOR, /* u.generator_data */ + JS_CLASS_GLOBAL_OBJECT, /* u.global_object */ JS_CLASS_PROXY, /* u.proxy_data */ JS_CLASS_PROMISE, /* u.promise_data */ JS_CLASS_PROMISE_RESOLVE_FUNCTION, /* u.promise_function_data */ @@ -279,7 +285,12 @@ struct JSRuntime { struct list_head job_list; /* list of JSJobEntry.link */ JSModuleNormalizeFunc *module_normalize_func; - JSModuleLoaderFunc *module_loader_func; + BOOL module_loader_has_attr; + union { + JSModuleLoaderFunc *module_loader_func; + JSModuleLoaderFunc2 *module_loader_func2; + } u; + JSModuleCheckSupportedImportAttributes *module_check_attrs; void *module_loader_opaque; /* timestamp for internal use in module evaluation */ int64_t module_async_evaluation_next_timestamp; @@ -334,6 +345,7 @@ typedef enum { JS_GC_OBJ_TYPE_VAR_REF, JS_GC_OBJ_TYPE_ASYNC_FUNCTION, JS_GC_OBJ_TYPE_JS_CONTEXT, + JS_GC_OBJ_TYPE_MODULE, } JSGCObjectTypeEnum; /* header for GC objects. GC objects are C data structures with a @@ -342,7 +354,8 @@ typedef enum { struct JSGCObjectHeader { int ref_count; /* must come first, 32-bit */ JSGCObjectTypeEnum gc_obj_type : 4; - uint8_t mark : 4; /* used by the GC */ + uint8_t mark : 1; /* used by the GC */ + uint8_t dummy0: 3; uint8_t dummy1; /* not used by the GC */ uint16_t dummy2; /* not used by the GC */ struct list_head link; @@ -366,6 +379,8 @@ typedef struct JSVarRef { int __gc_ref_count; /* corresponds to header.ref_count */ uint8_t __gc_mark; /* corresponds to header.mark/gc_obj_type */ uint8_t is_detached; + uint8_t is_lexical; /* only used with global variables */ + uint8_t is_const; /* only used with global variables */ }; }; JSValue *pvalue; /* pointer to the value, either on the stack or @@ -435,7 +450,14 @@ struct JSContext { uint16_t binary_object_count; int binary_object_size; - + /* TRUE if the array prototype is "normal": + - no small index properties which are get/set or non writable + - its prototype is Object.prototype + - Object.prototype has no small index properties which are get/set or non writable + - the prototype of Object.prototype is null (always true as it is immutable) + */ + uint8_t std_array_prototype; + JSShape *array_shape; /* initial shape for Array objects */ JSValue *class_proto; @@ -445,7 +467,7 @@ struct JSContext { JSValue regexp_ctor; JSValue promise_ctor; JSValue native_error_proto[JS_NATIVE_ERROR_COUNT]; - JSValue iterator_proto; + JSValue iterator_ctor; JSValue async_iterator_proto; JSValue array_proto_values; JSValue throw_type_error; @@ -523,11 +545,23 @@ typedef struct JSStringRope { JSValue right; /* might be the empty string */ } JSStringRope; +typedef enum { + JS_CLOSURE_LOCAL, /* 'var_idx' is the index of a local variable in the parent function */ + JS_CLOSURE_ARG, /* 'var_idx' is the index of a argument variable in the parent function */ + JS_CLOSURE_REF, /* 'var_idx' is the index of a closure variable in the parent function */ + JS_CLOSURE_GLOBAL_REF, /* 'var_idx' in the index of a closure + variable in the parent function + referencing a global variable */ + JS_CLOSURE_GLOBAL_DECL, /* global variable declaration (eval code only) */ + JS_CLOSURE_GLOBAL, /* global variable (eval code only) */ + JS_CLOSURE_MODULE_DECL, /* definition of a module variable (eval code only) */ + JS_CLOSURE_MODULE_IMPORT, /* definition of a module import (eval code only) */ +} JSClosureTypeEnum; + typedef struct JSClosureVar { - uint8_t is_local : 1; - uint8_t is_arg : 1; - uint8_t is_const : 1; - uint8_t is_lexical : 1; + JSClosureTypeEnum closure_type : 3; + uint8_t is_lexical : 1; /* lexical variable */ + uint8_t is_const : 1; /* const variable (is_lexical = 1 if is_const = 1 */ uint8_t var_kind : 4; /* see JSVarKindEnum */ /* 8 bits available */ uint16_t var_idx; /* is_local = TRUE: index to a normal variable of the @@ -557,6 +591,7 @@ typedef enum { JS_VAR_PRIVATE_GETTER, JS_VAR_PRIVATE_SETTER, /* must come after JS_VAR_PRIVATE_GETTER */ JS_VAR_PRIVATE_GETTER_SETTER, /* must come after JS_VAR_PRIVATE_SETTER */ + JS_VAR_GLOBAL_FUNCTION_DECL, /* global function definition, only in JSVarDef */ } JSVarKindEnum; /* XXX: could use a different structure in bytecode functions to save @@ -678,6 +713,7 @@ typedef struct JSProxyData { typedef struct JSArrayBuffer { int byte_length; /* 0 if detached */ + int max_byte_length; /* -1 if not resizable; >= byte_length otherwise */ uint8_t detached; uint8_t shared; /* if shared, the array buffer cannot be detached */ uint8_t *data; /* NULL if detached */ @@ -690,10 +726,15 @@ typedef struct JSTypedArray { struct list_head link; /* link to arraybuffer */ JSObject *obj; /* back pointer to the TypedArray/DataView object */ JSObject *buffer; /* based array buffer */ - uint32_t offset; /* offset in the array buffer */ - uint32_t length; /* length in the array buffer */ + uint32_t offset; /* byte offset in the array buffer */ + uint32_t length; /* byte length in the array buffer */ + BOOL track_rab; /* auto-track length of backing array buffer */ } JSTypedArray; +typedef struct JSGlobalObject { + JSValue uninitialized_vars; /* hidden object containing the list of uninitialized variables */ +} JSGlobalObject; + typedef struct JSAsyncFunctionState { JSGCObjectHeader header; JSValue this_val; /* 'this' argument */ @@ -755,6 +796,7 @@ typedef struct { typedef struct JSReqModuleEntry { JSAtom module_name; JSModuleDef *module; /* used using resolution */ + JSValue attributes; /* JS_UNDEFINED or an object contains the attributes as key/value */ } JSReqModuleEntry; typedef enum JSExportTypeEnum { @@ -797,7 +839,7 @@ typedef enum { } JSModuleStatus; struct JSModuleDef { - JSRefCountHeader header; /* must come first, 32-bit */ + JSGCObjectHeader header; /* must come first */ JSAtom module_name; struct list_head link; @@ -832,7 +874,8 @@ struct JSModuleDef { int async_parent_modules_count; int async_parent_modules_size; int pending_async_dependencies; - BOOL async_evaluation; + BOOL async_evaluation; /* true: async_evaluation_timestamp corresponds to [[AsyncEvaluationOrder]] + false: [[AsyncEvaluationOrder]] is UNSET or DONE */ int64_t async_evaluation_timestamp; JSModuleDef *cycle_root; JSValue promise; /* corresponds to spec field: capability */ @@ -843,11 +886,12 @@ struct JSModuleDef { BOOL eval_has_exception : 8; JSValue eval_exception; JSValue meta_obj; /* for import.meta */ + JSValue private_value; /* private value for C modules */ }; typedef struct JSJobEntry { struct list_head link; - JSContext *ctx; + JSContext *realm; JSJobFunc *job_func; int argc; JSValue argv[0]; @@ -873,7 +917,6 @@ typedef struct JSProperty { #define JS_PROP_INITIAL_SIZE 2 #define JS_PROP_INITIAL_HASH_SIZE 4 /* must be a power of two */ -#define JS_ARRAY_INITIAL_SIZE 2 typedef struct JSShapeProperty { uint32_t hash_next : 26; /* 0 if last in list */ @@ -888,10 +931,6 @@ struct JSShape { /* true if the shape is inserted in the shape hash table. If not, JSShape.hash is not valid */ uint8_t is_hashed; - /* If true, the shape may have small array index properties 'n' with 0 - <= n <= 2^31-1. If false, the shape is guaranteed not to have - small array index properties */ - uint8_t has_small_array_index; uint32_t hash; /* current hash value */ uint32_t prop_hash_mask; int prop_size; /* allocated properties */ @@ -907,7 +946,8 @@ struct JSObject { JSGCObjectHeader header; struct { int __gc_ref_count; /* corresponds to header.ref_count */ - uint8_t __gc_mark; /* corresponds to header.mark/gc_obj_type */ + uint8_t __gc_mark : 7; /* corresponds to header.mark/gc_obj_type */ + uint8_t is_prototype : 1; /* object may be used as prototype */ uint8_t extensible : 1; uint8_t free_mark : 1; /* only used when freeing objects with cycles */ @@ -938,6 +978,8 @@ struct JSObject { struct JSArrayIteratorData *array_iterator_data; /* JS_CLASS_ARRAY_ITERATOR, JS_CLASS_STRING_ITERATOR */ struct JSRegExpStringIteratorData *regexp_string_iterator_data; /* JS_CLASS_REGEXP_STRING_ITERATOR */ struct JSGeneratorData *generator_data; /* JS_CLASS_GENERATOR */ + struct JSIteratorHelperData *iterator_helper_data; /* JS_CLASS_ITERATOR_HELPER */ + struct JSIteratorWrapData *iterator_wrap_data; /* JS_CLASS_ITERATOR_WRAP */ struct JSProxyData *proxy_data; /* JS_CLASS_PROXY */ struct JSPromiseData *promise_data; /* JS_CLASS_PROMISE */ struct JSPromiseFunctionData *promise_function_data; /* JS_CLASS_PROMISE_RESOLVE_FUNCTION, JS_CLASS_PROMISE_REJECT_FUNCTION */ @@ -974,6 +1016,7 @@ struct JSObject { uint32_t *uint32_ptr; /* JS_CLASS_UINT32_ARRAY */ int64_t *int64_ptr; /* JS_CLASS_INT64_ARRAY */ uint64_t *uint64_ptr; /* JS_CLASS_UINT64_ARRAY */ + uint16_t *fp16_ptr; /* JS_CLASS_FLOAT16_ARRAY */ float *float_ptr; /* JS_CLASS_FLOAT32_ARRAY */ double *double_ptr; /* JS_CLASS_FLOAT64_ARRAY */ } u; @@ -981,6 +1024,7 @@ struct JSObject { } array; /* 12/20 bytes */ JSRegExp regexp; /* JS_CLASS_REGEXP: 8/16 bytes */ JSValue object_data; /* for JS_SetObjectData(): 8/16/16 bytes */ + JSGlobalObject global_object; } u; }; @@ -1078,15 +1122,16 @@ static __exception int JS_ToArrayLengthFree(JSContext *ctx, uint32_t *plen, static JSValue JS_EvalObject(JSContext *ctx, JSValueConst this_obj, JSValueConst val, int flags, int scope_idx); JSValue __attribute__((format(printf, 2, 3))) JS_ThrowInternalError(JSContext *ctx, const char *fmt, ...); -static void js_print_atom(JSRuntime *rt, FILE *fo, JSAtom atom); static __maybe_unused void JS_DumpAtoms(JSRuntime *rt); static __maybe_unused void JS_DumpString(JSRuntime *rt, const JSString *p); static __maybe_unused void JS_DumpObjectHeader(JSRuntime *rt); static __maybe_unused void JS_DumpObject(JSRuntime *rt, JSObject *p); static __maybe_unused void JS_DumpGCObject(JSRuntime *rt, JSGCObjectHeader *p); +static __maybe_unused void JS_DumpAtom(JSContext *ctx, const char *str, JSAtom atom); static __maybe_unused void JS_DumpValueRT(JSRuntime *rt, const char *str, JSValueConst val); static __maybe_unused void JS_DumpValue(JSContext *ctx, const char *str, JSValueConst val); static __maybe_unused void JS_DumpShapes(JSRuntime *rt); +static void js_dump_value_write(void *opaque, const char *buf, size_t len); static JSValue js_function_apply(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, int magic); static void js_array_finalizer(JSRuntime *rt, JSValue val); @@ -1121,12 +1166,21 @@ static void js_map_iterator_mark(JSRuntime *rt, JSValueConst val, static void js_array_iterator_finalizer(JSRuntime *rt, JSValue val); static void js_array_iterator_mark(JSRuntime *rt, JSValueConst val, JS_MarkFunc *mark_func); +static void js_iterator_helper_finalizer(JSRuntime *rt, JSValue val); +static void js_iterator_helper_mark(JSRuntime *rt, JSValueConst val, + JS_MarkFunc *mark_func); +static void js_iterator_wrap_finalizer(JSRuntime *rt, JSValue val); +static void js_iterator_wrap_mark(JSRuntime *rt, JSValueConst val, + JS_MarkFunc *mark_func); static void js_regexp_string_iterator_finalizer(JSRuntime *rt, JSValue val); static void js_regexp_string_iterator_mark(JSRuntime *rt, JSValueConst val, JS_MarkFunc *mark_func); static void js_generator_finalizer(JSRuntime *rt, JSValue obj); static void js_generator_mark(JSRuntime *rt, JSValueConst val, JS_MarkFunc *mark_func); +static void js_global_object_finalizer(JSRuntime *rt, JSValue obj); +static void js_global_object_mark(JSRuntime *rt, JSValueConst val, + JS_MarkFunc *mark_func); static void js_promise_finalizer(JSRuntime *rt, JSValue val); static void js_promise_mark(JSRuntime *rt, JSValueConst val, JS_MarkFunc *mark_func); @@ -1147,8 +1201,8 @@ static int JS_ToUint8ClampFree(JSContext *ctx, int32_t *pres, JSValue val); static JSValue js_new_string8_len(JSContext *ctx, const char *buf, int len); static JSValue js_compile_regexp(JSContext *ctx, JSValueConst pattern, JSValueConst flags); -static JSValue js_regexp_constructor_internal(JSContext *ctx, JSValueConst ctor, - JSValue pattern, JSValue bc); +static JSValue js_regexp_set_internal(JSContext *ctx, JSValue obj, + JSValue pattern, JSValue bc); static void gc_decref(JSRuntime *rt); static int JS_NewClass1(JSRuntime *rt, JSClassID class_id, const JSClassDef *class_def, JSAtom name); @@ -1181,11 +1235,14 @@ static int js_string_memcmp(const JSString *p1, int pos1, const JSString *p2, int pos2, int len); static JSValue js_array_buffer_constructor3(JSContext *ctx, JSValueConst new_target, - uint64_t len, JSClassID class_id, + uint64_t len, uint64_t *max_len, + JSClassID class_id, uint8_t *buf, JSFreeArrayBufferDataFunc *free_func, void *opaque, BOOL alloc_flag); +static void js_array_buffer_free(JSRuntime *rt, void *opaque, void *ptr); static JSArrayBuffer *js_get_array_buffer(JSContext *ctx, JSValueConst obj); +static BOOL array_buffer_is_resizable(const JSArrayBuffer *abuf); static JSValue js_typed_array_constructor(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, @@ -1193,10 +1250,12 @@ static JSValue js_typed_array_constructor(JSContext *ctx, static JSValue js_typed_array_constructor_ta(JSContext *ctx, JSValueConst new_target, JSValueConst src_obj, - int classid); -static BOOL typed_array_is_detached(JSContext *ctx, JSObject *p); -static uint32_t typed_array_get_length(JSContext *ctx, JSObject *p); + int classid, uint32_t len); +static BOOL typed_array_is_oob(JSObject *p); +static int js_typed_array_get_length_unsafe(JSContext *ctx, JSValueConst obj); static JSValue JS_ThrowTypeErrorDetachedArrayBuffer(JSContext *ctx); +static JSValue JS_ThrowTypeErrorArrayBufferOOB(JSContext *ctx); +static JSVarRef *js_create_var_ref(JSContext *ctx, BOOL is_lexical); static JSVarRef *get_var_ref(JSContext *ctx, JSStackFrame *sf, int var_idx, BOOL is_arg); static void __async_func_free(JSRuntime *rt, JSAsyncFunctionState *s); @@ -1211,11 +1270,11 @@ static void js_async_function_resolve_mark(JSRuntime *rt, JSValueConst val, static JSValue JS_EvalInternal(JSContext *ctx, JSValueConst this_obj, const char *input, size_t input_len, const char *filename, int flags, int scope_idx); -static void js_free_module_def(JSContext *ctx, JSModuleDef *m); +static void js_free_module_def(JSRuntime *rt, JSModuleDef *m); static void js_mark_module_def(JSRuntime *rt, JSModuleDef *m, JS_MarkFunc *mark_func); static JSValue js_import_meta(JSContext *ctx); -static JSValue js_dynamic_import(JSContext *ctx, JSValueConst specifier); +static JSValue js_dynamic_import(JSContext *ctx, JSValueConst specifier, JSValueConst options); static void free_var_ref(JSRuntime *rt, JSVarRef *var_ref); static JSValue js_new_promise_capability(JSContext *ctx, JSValue *resolving_funcs, @@ -1228,6 +1287,8 @@ static JSValue js_promise_resolve(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, int magic); static JSValue js_promise_then(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv); +static BOOL js_string_eq(JSContext *ctx, + const JSString *p1, const JSString *p2); static int js_string_compare(JSContext *ctx, const JSString *p1, const JSString *p2); static JSValue JS_ToNumber(JSContext *ctx, JSValueConst val); @@ -1239,7 +1300,7 @@ static JSValue JS_ToNumberFree(JSContext *ctx, JSValue val); static int JS_GetOwnPropertyInternal(JSContext *ctx, JSPropertyDescriptor *desc, JSObject *p, JSAtom prop); static void js_free_desc(JSContext *ctx, JSPropertyDescriptor *desc); -static void JS_AddIntrinsicBasicObjects(JSContext *ctx); +static int JS_AddIntrinsicBasicObjects(JSContext *ctx); static void js_free_shape(JSRuntime *rt, JSShape *sh); static void js_free_shape_null(JSRuntime *rt, JSShape *sh); static int js_shape_prepare_update(JSContext *ctx, JSObject *p, @@ -1286,6 +1347,8 @@ static JSValue get_date_string(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, int magic); static JSValue js_error_toString(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv); +static JSVarRef *js_global_object_find_uninitialized_var(JSContext *ctx, JSObject *p, + JSAtom atom, BOOL is_lexical); static const JSClassExoticMethods js_arguments_exotic_methods; static const JSClassExoticMethods js_string_exotic_methods; @@ -1458,6 +1521,23 @@ static inline void js_dbuf_init(JSContext *ctx, DynBuf *s) dbuf_init2(s, ctx->rt, (DynBufReallocFunc *)js_realloc_rt); } +static void *js_realloc_bytecode_rt(void *opaque, void *ptr, size_t size) +{ + JSRuntime *rt = opaque; + if (size > (INT32_MAX / 2)) { + /* the bytecode cannot be larger than 2G. Leave some slack to + avoid some overflows. */ + return NULL; + } else { + return rt->mf.js_realloc(&rt->malloc_state, ptr, size); + } +} + +static inline void js_dbuf_bytecode_init(JSContext *ctx, DynBuf *s) +{ + dbuf_init2(s, ctx->rt, js_realloc_bytecode_rt); +} + static inline int is_digit(int c) { return c >= '0' && c <= '9'; } @@ -1502,6 +1582,7 @@ static JSClassShortDef const js_std_class_def[] = { { JS_ATOM_Uint32Array, js_typed_array_finalizer, js_typed_array_mark }, /* JS_CLASS_UINT32_ARRAY */ { JS_ATOM_BigInt64Array, js_typed_array_finalizer, js_typed_array_mark }, /* JS_CLASS_BIG_INT64_ARRAY */ { JS_ATOM_BigUint64Array, js_typed_array_finalizer, js_typed_array_mark }, /* JS_CLASS_BIG_UINT64_ARRAY */ + { JS_ATOM_Float16Array, js_typed_array_finalizer, js_typed_array_mark }, /* JS_CLASS_FLOAT16_ARRAY */ { JS_ATOM_Float32Array, js_typed_array_finalizer, js_typed_array_mark }, /* JS_CLASS_FLOAT32_ARRAY */ { JS_ATOM_Float64Array, js_typed_array_finalizer, js_typed_array_mark }, /* JS_CLASS_FLOAT64_ARRAY */ { JS_ATOM_DataView, js_typed_array_finalizer, js_typed_array_mark }, /* JS_CLASS_DATAVIEW */ @@ -1510,12 +1591,16 @@ static JSClassShortDef const js_std_class_def[] = { { JS_ATOM_Set, js_map_finalizer, js_map_mark }, /* JS_CLASS_SET */ { JS_ATOM_WeakMap, js_map_finalizer, js_map_mark }, /* JS_CLASS_WEAKMAP */ { JS_ATOM_WeakSet, js_map_finalizer, js_map_mark }, /* JS_CLASS_WEAKSET */ + { JS_ATOM_Iterator, NULL, NULL }, /* JS_CLASS_ITERATOR */ + { JS_ATOM_IteratorHelper, js_iterator_helper_finalizer, js_iterator_helper_mark }, /* JS_CLASS_ITERATOR_HELPER */ + { JS_ATOM_IteratorWrap, js_iterator_wrap_finalizer, js_iterator_wrap_mark }, /* JS_CLASS_ITERATOR_WRAP */ { JS_ATOM_Map_Iterator, js_map_iterator_finalizer, js_map_iterator_mark }, /* JS_CLASS_MAP_ITERATOR */ { JS_ATOM_Set_Iterator, js_map_iterator_finalizer, js_map_iterator_mark }, /* JS_CLASS_SET_ITERATOR */ { JS_ATOM_Array_Iterator, js_array_iterator_finalizer, js_array_iterator_mark }, /* JS_CLASS_ARRAY_ITERATOR */ { JS_ATOM_String_Iterator, js_array_iterator_finalizer, js_array_iterator_mark }, /* JS_CLASS_STRING_ITERATOR */ { JS_ATOM_RegExp_String_Iterator, js_regexp_string_iterator_finalizer, js_regexp_string_iterator_mark }, /* JS_CLASS_REGEXP_STRING_ITERATOR */ { JS_ATOM_Generator, js_generator_finalizer, js_generator_mark }, /* JS_CLASS_GENERATOR */ + { JS_ATOM_Object, js_global_object_finalizer, js_global_object_mark }, /* JS_CLASS_GLOBAL_OBJECT */ }; static int init_class_range(JSRuntime *rt, JSClassShortDef const *tab, @@ -1770,7 +1855,7 @@ int JS_EnqueueJob(JSContext *ctx, JSJobFunc *job_func, e = js_malloc(ctx, sizeof(*e) + argc * sizeof(JSValue)); if (!e) return -1; - e->ctx = ctx; + e->realm = JS_DupContext(ctx); e->job_func = job_func; e->argc = argc; for(i = 0; i < argc; i++) { @@ -1786,7 +1871,10 @@ BOOL JS_IsJobPending(JSRuntime *rt) } /* return < 0 if exception, 0 if no job pending, 1 if a job was - executed successfully. the context of the job is stored in '*pctx' */ + executed successfully. The context of the job is stored in '*pctx' + if pctx != NULL. It may be NULL if the context was already + destroyed or if no job was pending. The 'pctx' parameter is now + absolete. */ int JS_ExecutePendingJob(JSRuntime *rt, JSContext **pctx) { JSContext *ctx; @@ -1795,15 +1883,16 @@ int JS_ExecutePendingJob(JSRuntime *rt, JSContext **pctx) int i, ret; if (list_empty(&rt->job_list)) { - *pctx = NULL; + if (pctx) + *pctx = NULL; return 0; } /* get the first pending job and execute it */ e = list_entry(rt->job_list.next, JSJobEntry, link); list_del(&e->link); - ctx = e->ctx; - res = e->job_func(e->ctx, e->argc, (JSValueConst *)e->argv); + ctx = e->realm; + res = e->job_func(ctx, e->argc, (JSValueConst *)e->argv); for(i = 0; i < e->argc; i++) JS_FreeValue(ctx, e->argv[i]); if (JS_IsException(res)) @@ -1812,7 +1901,13 @@ int JS_ExecutePendingJob(JSRuntime *rt, JSContext **pctx) ret = 1; JS_FreeValue(ctx, res); js_free(ctx, e); - *pctx = ctx; + if (pctx) { + if (ctx->header.ref_count > 1) + *pctx = ctx; + else + *pctx = NULL; + } + JS_FreeContext(ctx); return ret; } @@ -1893,6 +1988,7 @@ void JS_FreeRuntime(JSRuntime *rt) JSJobEntry *e = list_entry(el, JSJobEntry, link); for(i = 0; i < e->argc; i++) JS_FreeValueRT(rt, e->argv[i]); + JS_FreeContext(e->realm); js_free_rt(rt, e); } init_list_head(&rt->job_list); @@ -2091,11 +2187,15 @@ JSContext *JS_NewContextRaw(JSRuntime *rt) for(i = 0; i < rt->class_count; i++) ctx->class_proto[i] = JS_NULL; ctx->array_ctor = JS_NULL; + ctx->iterator_ctor = JS_NULL; ctx->regexp_ctor = JS_NULL; ctx->promise_ctor = JS_NULL; init_list_head(&ctx->loaded_modules); - JS_AddIntrinsicBasicObjects(ctx); + if (JS_AddIntrinsicBasicObjects(ctx)) { + JS_FreeContext(ctx); + return NULL; + } return ctx; } @@ -2107,17 +2207,20 @@ JSContext *JS_NewContext(JSRuntime *rt) if (!ctx) return NULL; - JS_AddIntrinsicBaseObjects(ctx); - JS_AddIntrinsicDate(ctx); - JS_AddIntrinsicEval(ctx); - JS_AddIntrinsicStringNormalize(ctx); - JS_AddIntrinsicRegExp(ctx); - JS_AddIntrinsicJSON(ctx); - JS_AddIntrinsicProxy(ctx); - JS_AddIntrinsicMapSet(ctx); - JS_AddIntrinsicTypedArrays(ctx); - JS_AddIntrinsicPromise(ctx); - JS_AddIntrinsicWeakRef(ctx); + if (JS_AddIntrinsicBaseObjects(ctx) || + JS_AddIntrinsicDate(ctx) || + JS_AddIntrinsicEval(ctx) || + JS_AddIntrinsicStringNormalize(ctx) || + JS_AddIntrinsicRegExp(ctx) || + JS_AddIntrinsicJSON(ctx) || + JS_AddIntrinsicProxy(ctx) || + JS_AddIntrinsicMapSet(ctx) || + JS_AddIntrinsicTypedArrays(ctx) || + JS_AddIntrinsicPromise(ctx) || + JS_AddIntrinsicWeakRef(ctx)) { + JS_FreeContext(ctx); + return NULL; + } return ctx; } @@ -2168,7 +2271,13 @@ static void js_free_modules(JSContext *ctx, JSFreeModuleEnum flag) JSModuleDef *m = list_entry(el, JSModuleDef, link); if (flag == JS_FREE_MODULE_ALL || (flag == JS_FREE_MODULE_NOT_RESOLVED && !m->resolved)) { - js_free_module_def(ctx, m); + /* warning: the module may be referenced elsewhere. It + could be simpler to use an array instead of a list for + 'ctx->loaded_modules' */ + list_del(&m->link); + m->link.prev = NULL; + m->link.next = NULL; + JS_FreeValue(ctx, JS_MKPTR(JS_TAG_MODULE, m)); } } } @@ -2186,11 +2295,9 @@ static void JS_MarkContext(JSRuntime *rt, JSContext *ctx, int i; struct list_head *el; - /* modules are not seen by the GC, so we directly mark the objects - referenced by each module */ list_for_each(el, &ctx->loaded_modules) { JSModuleDef *m = list_entry(el, JSModuleDef, link); - js_mark_module_def(rt, m, mark_func); + JS_MarkValue(rt, JS_MKPTR(JS_TAG_MODULE, m), mark_func); } JS_MarkValue(rt, ctx->global_obj, mark_func); @@ -2206,7 +2313,7 @@ static void JS_MarkContext(JSRuntime *rt, JSContext *ctx, for(i = 0; i < rt->class_count; i++) { JS_MarkValue(rt, ctx->class_proto[i], mark_func); } - JS_MarkValue(rt, ctx->iterator_proto, mark_func); + JS_MarkValue(rt, ctx->iterator_ctor, mark_func); JS_MarkValue(rt, ctx->async_iterator_proto, mark_func); JS_MarkValue(rt, ctx->promise_ctor, mark_func); JS_MarkValue(rt, ctx->array_ctor, mark_func); @@ -2270,7 +2377,7 @@ void JS_FreeContext(JSContext *ctx) JS_FreeValue(ctx, ctx->class_proto[i]); } js_free_rt(rt, ctx->class_proto); - JS_FreeValue(ctx, ctx->iterator_proto); + JS_FreeValue(ctx, ctx->iterator_ctor); JS_FreeValue(ctx, ctx->async_iterator_proto); JS_FreeValue(ctx, ctx->promise_ctor); JS_FreeValue(ctx, ctx->array_ctor); @@ -2537,7 +2644,7 @@ static int JS_InitAtoms(JSRuntime *rt) rt->atom_count = 0; rt->atom_size = 0; rt->atom_free_index = 0; - if (JS_ResizeAtomHash(rt, 256)) /* there are at least 195 predefined atoms */ + if (JS_ResizeAtomHash(rt, 512)) /* there are at least 504 predefined atoms */ return -1; p = js_atom_init; @@ -2682,9 +2789,9 @@ static JSAtom __JS_NewAtom(JSRuntime *rt, JSString *str, int atom_type) /* alloc new with size progression 3/2: 4 6 9 13 19 28 42 63 94 141 211 316 474 711 1066 1599 2398 3597 5395 8092 - preallocating space for predefined atoms (at least 195). + preallocating space for predefined atoms (at least 504). */ - new_size = max_int(211, rt->atom_size * 3 / 2); + new_size = max_int(711, rt->atom_size * 3 / 2); if (new_size > JS_ATOM_MAX) goto fail; /* XXX: should use realloc2 to use slack space */ @@ -3150,9 +3257,9 @@ static JSValue JS_AtomIsNumericIndex1(JSContext *ctx, JSAtom atom) JS_FreeValue(ctx, num); return str; } - ret = js_string_compare(ctx, p, JS_VALUE_GET_STRING(str)); + ret = js_string_eq(ctx, p, JS_VALUE_GET_STRING(str)); JS_FreeValue(ctx, str); - if (ret == 0) { + if (ret) { return num; } else { JS_FreeValue(ctx, num); @@ -3201,21 +3308,19 @@ static BOOL JS_AtomSymbolHasDescription(JSContext *ctx, JSAtom v) !(p->len == 0 && p->is_wide_char != 0)); } -static __maybe_unused void print_atom(JSContext *ctx, JSAtom atom) -{ - js_print_atom(ctx->rt, stdout, atom); -} - /* free with JS_FreeCString() */ -const char *JS_AtomToCString(JSContext *ctx, JSAtom atom) +const char *JS_AtomToCStringLen(JSContext *ctx, size_t *plen, JSAtom atom) { JSValue str; const char *cstr; str = JS_AtomToString(ctx, atom); - if (JS_IsException(str)) + if (JS_IsException(str)) { + if (plen) + *plen = 0; return NULL; - cstr = JS_ToCString(ctx, str); + } + cstr = JS_ToCStringLen(ctx, plen, str); JS_FreeValue(ctx, str); return cstr; } @@ -3546,7 +3651,7 @@ static no_inline int string_buffer_realloc(StringBuffer *s, int new_len, int c) return 0; } -static no_inline int string_buffer_putc_slow(StringBuffer *s, uint32_t c) +static no_inline int string_buffer_putc16_slow(StringBuffer *s, uint32_t c) { if (unlikely(s->len >= s->size)) { if (string_buffer_realloc(s, s->len + 1, c)) @@ -3591,11 +3696,10 @@ static int string_buffer_putc16(StringBuffer *s, uint32_t c) return 0; } } - return string_buffer_putc_slow(s, c); + return string_buffer_putc16_slow(s, c); } -/* 0 <= c <= 0x10ffff */ -static int string_buffer_putc(StringBuffer *s, uint32_t c) +static int string_buffer_putc_slow(StringBuffer *s, uint32_t c) { if (unlikely(c >= 0x10000)) { /* surrogate pair */ @@ -3606,6 +3710,27 @@ static int string_buffer_putc(StringBuffer *s, uint32_t c) return string_buffer_putc16(s, c); } +/* 0 <= c <= 0x10ffff */ +static inline int string_buffer_putc(StringBuffer *s, uint32_t c) +{ + if (likely(s->len < s->size)) { + if (s->is_wide_char) { + if (c < 0x10000) { + s->str->u.str16[s->len++] = c; + return 0; + } else if (likely((s->len + 1) < s->size)) { + s->str->u.str16[s->len++] = get_hi_surrogate(c); + s->str->u.str16[s->len++] = get_lo_surrogate(c); + return 0; + } + } else if (c < 0x100) { + s->str->u.str8[s->len++] = c; + return 0; + } + } + return string_buffer_putc_slow(s, c); +} + static int string_getc(const JSString *p, int *pidx) { int idx, c, c1; @@ -4036,6 +4161,16 @@ static int js_string_memcmp(const JSString *p1, int pos1, const JSString *p2, return res; } +static BOOL js_string_eq(JSContext *ctx, + const JSString *p1, const JSString *p2) +{ + if (p1->len != p2->len) + return FALSE; + if (p1 == p2) + return TRUE; + return js_string_memcmp(p1, 0, p2, 0, p1->len) == 0; +} + /* return < 0, 0 or > 0 */ static int js_string_compare(JSContext *ctx, const JSString *p1, const JSString *p2) @@ -4649,19 +4784,14 @@ static void js_shape_hash_unlink(JSRuntime *rt, JSShape *sh) rt->shape_hash_count--; } -/* create a new empty shape with prototype 'proto' */ -static no_inline JSShape *js_new_shape2(JSContext *ctx, JSObject *proto, - int hash_size, int prop_size) +/* create a new empty shape with prototype 'proto'. It is not hashed */ +static inline JSShape *js_new_shape_nohash(JSContext *ctx, JSObject *proto, + int hash_size, int prop_size) { JSRuntime *rt = ctx->rt; void *sh_alloc; JSShape *sh; - /* resize the shape hash table if necessary */ - if (2 * (rt->shape_hash_count + 1) > rt->shape_hash_size) { - resize_shape_hash(rt, rt->shape_hash_bits + 1); - } - sh_alloc = js_malloc(ctx, get_shape_size(hash_size, prop_size)); if (!sh_alloc) return NULL; @@ -4677,11 +4807,29 @@ static no_inline JSShape *js_new_shape2(JSContext *ctx, JSObject *proto, sh->prop_size = prop_size; sh->prop_count = 0; sh->deleted_prop_count = 0; + sh->is_hashed = FALSE; + return sh; +} +/* create a new empty shape with prototype 'proto' */ +static no_inline JSShape *js_new_shape2(JSContext *ctx, JSObject *proto, + int hash_size, int prop_size) +{ + JSRuntime *rt = ctx->rt; + JSShape *sh; + + /* resize the shape hash table if necessary */ + if (2 * (rt->shape_hash_count + 1) > rt->shape_hash_size) { + resize_shape_hash(rt, rt->shape_hash_bits + 1); + } + + sh = js_new_shape_nohash(ctx, proto, hash_size, prop_size); + if (!sh) + return NULL; + /* insert in the hash table */ sh->hash = shape_initial_hash(proto); sh->is_hashed = TRUE; - sh->has_small_array_index = FALSE; js_shape_hash_link(ctx->rt, sh); return sh; } @@ -4926,7 +5074,6 @@ static int add_shape_property(JSContext *ctx, JSShape **psh, pr = &prop[sh->prop_count++]; pr->atom = JS_DupAtom(ctx, atom); pr->flags = prop_flags; - sh->has_small_array_index |= __JS_AtomIsTaggedInt(atom); /* add in hash table */ hash_mask = sh->prop_hash_mask; h = atom & hash_mask; @@ -5041,6 +5188,7 @@ static JSValue JS_NewObjectFromShape(JSContext *ctx, JSShape *sh, JSClassID clas if (unlikely(!p)) goto fail; p->class_id = class_id; + p->is_prototype = 0; p->extensible = TRUE; p->free_mark = 0; p->is_exotic = 0; @@ -5096,6 +5244,7 @@ static JSValue JS_NewObjectFromShape(JSContext *ctx, JSShape *sh, JSClassID clas case JS_CLASS_UINT32_ARRAY: case JS_CLASS_BIG_INT64_ARRAY: case JS_CLASS_BIG_UINT64_ARRAY: + case JS_CLASS_FLOAT16_ARRAY: case JS_CLASS_FLOAT32_ARRAY: case JS_CLASS_FLOAT64_ARRAY: p->is_exotic = 1; @@ -5118,7 +5267,10 @@ static JSValue JS_NewObjectFromShape(JSContext *ctx, JSShape *sh, JSClassID clas case JS_CLASS_REGEXP: p->u.regexp.pattern = NULL; p->u.regexp.bytecode = NULL; - goto set_exotic; + break; + case JS_CLASS_GLOBAL_OBJECT: + p->u.global_object.uninitialized_vars = JS_UNDEFINED; + break; default: set_exotic: if (ctx->rt->class_array[class_id].exotic) { @@ -5158,6 +5310,29 @@ JSValue JS_NewObjectProtoClass(JSContext *ctx, JSValueConst proto_val, return JS_NewObjectFromShape(ctx, sh, class_id); } +/* WARNING: the shape is not hashed. It is used for objects where + factorizing the shape is not relevant (prototypes, constructors) */ +static JSValue JS_NewObjectProtoClassAlloc(JSContext *ctx, JSValueConst proto_val, + JSClassID class_id, int n_alloc_props) +{ + JSShape *sh; + JSObject *proto; + int hash_size, hash_bits; + + if (n_alloc_props <= JS_PROP_INITIAL_SIZE) { + n_alloc_props = JS_PROP_INITIAL_SIZE; + hash_size = JS_PROP_INITIAL_HASH_SIZE; + } else { + hash_bits = 32 - clz32(n_alloc_props - 1); /* ceil(log2(radix)) */ + hash_size = 1 << hash_bits; + } + proto = get_proto_obj(proto_val); + sh = js_new_shape_nohash(ctx, proto, hash_size, n_alloc_props); + if (!sh) + return JS_EXCEPTION; + return JS_NewObjectFromShape(ctx, sh, class_id); +} + #if 0 static JSValue JS_GetObjectData(JSContext *ctx, JSValueConst obj) { @@ -5321,13 +5496,17 @@ static int js_method_set_properties(JSContext *ctx, JSValueConst func_obj, static JSValue JS_NewCFunction3(JSContext *ctx, JSCFunction *func, const char *name, int length, JSCFunctionEnum cproto, int magic, - JSValueConst proto_val) + JSValueConst proto_val, int n_fields) { JSValue func_obj; JSObject *p; JSAtom name_atom; - func_obj = JS_NewObjectProtoClass(ctx, proto_val, JS_CLASS_C_FUNCTION); + if (n_fields > 0) { + func_obj = JS_NewObjectProtoClassAlloc(ctx, proto_val, JS_CLASS_C_FUNCTION, n_fields); + } else { + func_obj = JS_NewObjectProtoClass(ctx, proto_val, JS_CLASS_C_FUNCTION); + } if (JS_IsException(func_obj)) return func_obj; p = JS_VALUE_GET_OBJ(func_obj); @@ -5343,6 +5522,10 @@ static JSValue JS_NewCFunction3(JSContext *ctx, JSCFunction *func, if (!name) name = ""; name_atom = JS_NewAtom(ctx, name); + if (name_atom == JS_ATOM_NULL) { + JS_FreeValue(ctx, func_obj); + return JS_EXCEPTION; + } js_function_set_properties(ctx, func_obj, name_atom, length); JS_FreeAtom(ctx, name_atom); return func_obj; @@ -5354,7 +5537,7 @@ JSValue JS_NewCFunction2(JSContext *ctx, JSCFunction *func, int length, JSCFunctionEnum cproto, int magic) { return JS_NewCFunction3(ctx, func, name, length, cproto, magic, - ctx->function_proto); + ctx->function_proto, 0); } typedef struct JSCFunctionDataRecord { @@ -5768,6 +5951,9 @@ static void free_gc_object(JSRuntime *rt, JSGCObjectHeader *gp) case JS_GC_OBJ_TYPE_ASYNC_FUNCTION: __async_func_free(rt, (JSAsyncFunctionState *)gp); break; + case JS_GC_OBJ_TYPE_MODULE: + js_free_module_def(rt, (JSModuleDef *)gp); + break; default: abort(); } @@ -5832,6 +6018,7 @@ void __JS_FreeValueRT(JSRuntime *rt, JSValue v) break; case JS_TAG_OBJECT: case JS_TAG_FUNCTION_BYTECODE: + case JS_TAG_MODULE: { JSGCObjectHeader *p = JS_VALUE_GET_PTR(v); if (rt->gc_phase != JS_GC_PHASE_REMOVE_CYCLES) { @@ -5844,9 +6031,6 @@ void __JS_FreeValueRT(JSRuntime *rt, JSValue v) } } break; - case JS_TAG_MODULE: - abort(); /* never freed here */ - break; case JS_TAG_BIG_INT: { JSBigInt *p = JS_VALUE_GET_PTR(v); @@ -5920,6 +6104,7 @@ void JS_MarkValue(JSRuntime *rt, JSValueConst val, JS_MarkFunc *mark_func) switch(JS_VALUE_GET_TAG(val)) { case JS_TAG_OBJECT: case JS_TAG_FUNCTION_BYTECODE: + case JS_TAG_MODULE: mark_func(rt, JS_VALUE_GET_PTR(val)); break; default: @@ -6032,6 +6217,12 @@ static void mark_children(JSRuntime *rt, JSGCObjectHeader *gp, JS_MarkContext(rt, ctx, mark_func); } break; + case JS_GC_OBJ_TYPE_MODULE: + { + JSModuleDef *m = (JSModuleDef *)gp; + js_mark_module_def(rt, m, mark_func); + } + break; default: abort(); } @@ -6128,6 +6319,7 @@ static void gc_free_cycles(JSRuntime *rt) case JS_GC_OBJ_TYPE_JS_OBJECT: case JS_GC_OBJ_TYPE_FUNCTION_BYTECODE: case JS_GC_OBJ_TYPE_ASYNC_FUNCTION: + case JS_GC_OBJ_TYPE_MODULE: #ifdef DUMP_GC_FREE if (!header_done) { printf("Freeing cycles:\n"); @@ -6150,7 +6342,8 @@ static void gc_free_cycles(JSRuntime *rt) p = list_entry(el, JSGCObjectHeader, link); assert(p->gc_obj_type == JS_GC_OBJ_TYPE_JS_OBJECT || p->gc_obj_type == JS_GC_OBJ_TYPE_FUNCTION_BYTECODE || - p->gc_obj_type == JS_GC_OBJ_TYPE_ASYNC_FUNCTION); + p->gc_obj_type == JS_GC_OBJ_TYPE_ASYNC_FUNCTION || + p->gc_obj_type == JS_GC_OBJ_TYPE_MODULE); if (p->gc_obj_type == JS_GC_OBJ_TYPE_JS_OBJECT && ((JSObject *)p)->weakref_count != 0) { /* keep the object because there are weak references to it */ @@ -6487,6 +6680,7 @@ void JS_ComputeMemoryUsage(JSRuntime *rt, JSMemoryUsage *s) case JS_CLASS_UINT32_ARRAY: /* u.typed_array / u.array */ case JS_CLASS_BIG_INT64_ARRAY: /* u.typed_array / u.array */ case JS_CLASS_BIG_UINT64_ARRAY: /* u.typed_array / u.array */ + case JS_CLASS_FLOAT16_ARRAY: /* u.typed_array / u.array */ case JS_CLASS_FLOAT32_ARRAY: /* u.typed_array / u.array */ case JS_CLASS_FLOAT64_ARRAY: /* u.typed_array / u.array */ case JS_CLASS_DATAVIEW: /* u.typed_array */ @@ -6835,20 +7029,30 @@ static int find_line_num(JSContext *ctx, JSFunctionBytecode *b, return 0; } -/* in order to avoid executing arbitrary code during the stack trace - generation, we only look at simple 'name' properties containing a - string. */ -static const char *get_func_name(JSContext *ctx, JSValueConst func) +/* return a string property without executing arbitrary JS code (used + when dumping the stack trace or in debug print). */ +static const char *get_prop_string(JSContext *ctx, JSValueConst obj, JSAtom prop) { + JSObject *p; JSProperty *pr; JSShapeProperty *prs; JSValueConst val; - if (JS_VALUE_GET_TAG(func) != JS_TAG_OBJECT) - return NULL; - prs = find_own_property(&pr, JS_VALUE_GET_OBJ(func), JS_ATOM_name); - if (!prs) + if (JS_VALUE_GET_TAG(obj) != JS_TAG_OBJECT) return NULL; + p = JS_VALUE_GET_OBJ(obj); + prs = find_own_property(&pr, p, prop); + if (!prs) { + /* we look at one level in the prototype to handle the 'name' + field of the Error objects */ + p = p->shape->proto; + if (!p) + return NULL; + prs = find_own_property(&pr, p, prop); + if (!prs) + return NULL; + } + if ((prs->flags & JS_PROP_TMASK) != JS_PROP_NORMAL) return NULL; val = pr->u.value; @@ -6872,6 +7076,9 @@ static void build_backtrace(JSContext *ctx, JSValueConst error_obj, const char *str1; JSObject *p; + if (!JS_IsObject(error_obj)) + return; /* protection in the out of memory case */ + js_dbuf_init(ctx, &dbuf); if (filename) { dbuf_printf(&dbuf, " at %s", filename); @@ -6879,13 +7086,17 @@ static void build_backtrace(JSContext *ctx, JSValueConst error_obj, dbuf_printf(&dbuf, ":%d:%d", line_num, col_num); dbuf_putc(&dbuf, '\n'); str = JS_NewString(ctx, filename); + if (JS_IsException(str)) + return; /* Note: SpiderMonkey does that, could update once there is a standard */ - JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_fileName, str, - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_lineNumber, JS_NewInt32(ctx, line_num), - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_columnNumber, JS_NewInt32(ctx, col_num), - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); + if (JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_fileName, str, + JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE) < 0 || + JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_lineNumber, JS_NewInt32(ctx, line_num), + JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE) < 0 || + JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_columnNumber, JS_NewInt32(ctx, col_num), + JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE) < 0) { + return; + } } for(sf = ctx->rt->current_stack_frame; sf != NULL; sf = sf->prev_frame) { if (sf->js_mode & JS_MODE_BACKTRACE_BARRIER) @@ -6894,7 +7105,7 @@ static void build_backtrace(JSContext *ctx, JSValueConst error_obj, backtrace_flags &= ~JS_BACKTRACE_FLAG_SKIP_FIRST_LEVEL; continue; } - func_name_str = get_func_name(ctx, sf->cur_func); + func_name_str = get_prop_string(ctx, sf->cur_func, JS_ATOM_name); if (!func_name_str || func_name_str[0] == '\0') str1 = ""; else @@ -6970,9 +7181,9 @@ static JSValue JS_ThrowError2(JSContext *ctx, JSErrorEnum error_num, JS_DefinePropertyValue(ctx, obj, JS_ATOM_message, JS_NewString(ctx, buf), JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - } - if (add_backtrace) { - build_backtrace(ctx, obj, NULL, 0, 0, 0); + if (add_backtrace) { + build_backtrace(ctx, obj, NULL, 0, 0, 0); + } } ret = JS_Throw(ctx, obj); return ret; @@ -7115,6 +7326,22 @@ static JSValue JS_ThrowTypeErrorNotAnObject(JSContext *ctx) return JS_ThrowTypeError(ctx, "not an object"); } +static JSValue JS_ThrowTypeErrorNotAConstructor(JSContext *ctx, + JSValueConst func_obj) +{ + const char *name; + if (!JS_IsFunction(ctx, func_obj)) + goto fail; + name = get_prop_string(ctx, func_obj, JS_ATOM_name); + if (!name) { + fail: + return JS_ThrowTypeError(ctx, "not a constructor"); + } + JS_ThrowTypeError(ctx, "%s is not a constructor", name); + JS_FreeCString(ctx, name); + return JS_EXCEPTION; +} + static JSValue JS_ThrowTypeErrorNotASymbol(JSContext *ctx) { return JS_ThrowTypeError(ctx, "not a symbol"); @@ -7284,6 +7511,14 @@ static int JS_SetPrototypeInternal(JSContext *ctx, JSValueConst obj, if (sh->proto) JS_FreeValue(ctx, JS_MKPTR(JS_TAG_OBJECT, sh->proto)); sh->proto = proto; + if (proto) + proto->is_prototype = TRUE; + if (p->is_prototype) { + /* track modification of Array.prototype */ + if (unlikely(p == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]))) { + ctx->std_array_prototype = FALSE; + } + } return TRUE; } @@ -7493,6 +7728,16 @@ static int JS_AutoInitProperty(JSContext *ctx, JSObject *p, JSAtom prop, prs->flags |= JS_PROP_VARREF; pr->u.var_ref = JS_VALUE_GET_PTR(val); pr->u.var_ref->header.ref_count++; + } else if (p->class_id == JS_CLASS_GLOBAL_OBJECT) { + JSVarRef *var_ref; + /* in the global object we use references */ + var_ref = js_create_var_ref(ctx, FALSE); + if (!var_ref) + return -1; + prs->flags |= JS_PROP_VARREF; + pr->u.var_ref = var_ref; + var_ref->value = val; + var_ref->is_const = !(prs->flags & JS_PROP_WRITABLE); } else { pr->u.value = val; } @@ -7875,7 +8120,7 @@ static int num_keys_cmp(const void *p1, const void *p2, void *opaque) return 1; } -static void js_free_prop_enum(JSContext *ctx, JSPropertyEnum *tab, uint32_t len) +void JS_FreePropertyEnum(JSContext *ctx, JSPropertyEnum *tab, uint32_t len) { uint32_t i; if (tab) { @@ -7969,7 +8214,7 @@ static int __exception JS_GetOwnPropertyNamesInternal(JSContext *ctx, /* set the "is_enumerable" field if necessary */ res = JS_GetOwnPropertyInternal(ctx, &desc, p, atom); if (res < 0) { - js_free_prop_enum(ctx, tab_exotic, exotic_count); + JS_FreePropertyEnum(ctx, tab_exotic, exotic_count); return -1; } if (res) { @@ -8000,7 +8245,7 @@ static int __exception JS_GetOwnPropertyNamesInternal(JSContext *ctx, if (atom_count < exotic_keys_count || atom_count > INT32_MAX) { add_overflow: JS_ThrowOutOfMemory(ctx); - js_free_prop_enum(ctx, tab_exotic, exotic_count); + JS_FreePropertyEnum(ctx, tab_exotic, exotic_count); return -1; } /* XXX: need generic way to test for js_malloc(ctx, a * b) overflow */ @@ -8008,7 +8253,7 @@ static int __exception JS_GetOwnPropertyNamesInternal(JSContext *ctx, /* avoid allocating 0 bytes */ tab_atom = js_malloc(ctx, sizeof(tab_atom[0]) * max_int(atom_count, 1)); if (!tab_atom) { - js_free_prop_enum(ctx, tab_exotic, exotic_count); + JS_FreePropertyEnum(ctx, tab_exotic, exotic_count); return -1; } @@ -8053,7 +8298,7 @@ static int __exception JS_GetOwnPropertyNamesInternal(JSContext *ctx, for(i = 0; i < len; i++) { tab_atom[num_index].atom = __JS_AtomFromUInt32(i); if (tab_atom[num_index].atom == JS_ATOM_NULL) { - js_free_prop_enum(ctx, tab_atom, num_index); + JS_FreePropertyEnum(ctx, tab_atom, num_index); return -1; } tab_atom[num_index].is_enumerable = TRUE; @@ -8220,9 +8465,21 @@ int JS_PreventExtensions(JSContext *ctx, JSValueConst obj) return FALSE; p = JS_VALUE_GET_OBJ(obj); if (unlikely(p->is_exotic)) { - const JSClassExoticMethods *em = ctx->rt->class_array[p->class_id].exotic; - if (em && em->prevent_extensions) { - return em->prevent_extensions(ctx, obj); + if (p->class_id >= JS_CLASS_UINT8C_ARRAY && + p->class_id <= JS_CLASS_FLOAT64_ARRAY) { + JSTypedArray *ta; + JSArrayBuffer *abuf; + /* resizable type arrays return FALSE */ + ta = p->u.typed_array; + abuf = ta->buffer->u.array_buffer; + if (ta->track_rab || + (array_buffer_is_resizable(abuf) && !abuf->shared)) + return FALSE; + } else { + const JSClassExoticMethods *em = ctx->rt->class_array[p->class_id].exotic; + if (em && em->prevent_extensions) { + return em->prevent_extensions(ctx, obj); + } } } p->extensible = FALSE; @@ -8349,6 +8606,9 @@ static JSValue JS_GetPropertyValue(JSContext *ctx, JSValueConst this_obj, case JS_CLASS_BIG_UINT64_ARRAY: if (unlikely(idx >= p->u.array.count)) goto slow_path; return JS_NewBigUint64(ctx, p->u.array.u.uint64_ptr[idx]); + case JS_CLASS_FLOAT16_ARRAY: + if (unlikely(idx >= p->u.array.count)) goto slow_path; + return __JS_NewFloat64(ctx, fromfp16(p->u.array.u.fp16_ptr[idx])); case JS_CLASS_FLOAT32_ARRAY: if (unlikely(idx >= p->u.array.count)) goto slow_path; return __JS_NewFloat64(ctx, p->u.array.u.float_ptr[idx]); @@ -8441,6 +8701,8 @@ JSValue JS_GetPropertyStr(JSContext *ctx, JSValueConst this_obj, JSAtom atom; JSValue ret; atom = JS_NewAtom(ctx, prop); + if (atom == JS_ATOM_NULL) + return JS_EXCEPTION; ret = JS_GetProperty(ctx, this_obj, atom); JS_FreeAtom(ctx, atom); return ret; @@ -8453,6 +8715,14 @@ static JSProperty *add_property(JSContext *ctx, { JSShape *sh, *new_sh; + if (unlikely(p->is_prototype)) { + /* track addition of small integer properties to Array.prototype and Object.prototype */ + if (unlikely((p == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]) || + p == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_OBJECT])) && + __JS_AtomIsTaggedInt(prop))) { + ctx->std_array_prototype = FALSE; + } + } sh = p->shape; if (sh->is_hashed) { /* try to find an existing shape */ @@ -8522,6 +8792,34 @@ static no_inline __exception int convert_fast_array_to_array(JSContext *ctx, p->u.array.u.values = NULL; /* fail safe */ p->u.array.u1.size = 0; p->fast_array = 0; + + /* track modification of Array.prototype */ + if (unlikely(p == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]))) { + ctx->std_array_prototype = FALSE; + } + return 0; +} + +static int remove_global_object_property(JSContext *ctx, JSObject *p, + JSShapeProperty *prs, JSProperty *pr) +{ + JSVarRef *var_ref; + JSObject *p1; + JSProperty *pr1; + + var_ref = pr->u.var_ref; + if (var_ref->header.ref_count == 1) + return 0; + p1 = JS_VALUE_GET_OBJ(p->u.global_object.uninitialized_vars); + pr1 = add_property(ctx, p1, prs->atom, JS_PROP_C_W_E | JS_PROP_VARREF); + if (!pr1) + return -1; + pr1->u.var_ref = var_ref; + var_ref->header.ref_count++; + JS_FreeValue(ctx, var_ref->value); + var_ref->is_lexical = FALSE; + var_ref->is_const = FALSE; + var_ref->value = JS_UNINITIALIZED; return 0; } @@ -8562,6 +8860,11 @@ static int delete_property(JSContext *ctx, JSObject *p, JSAtom atom) sh->deleted_prop_count++; /* free the entry */ pr1 = &p->prop[h - 1]; + if (unlikely(p->class_id == JS_CLASS_GLOBAL_OBJECT)) { + if ((pr->flags & JS_PROP_TMASK) == JS_PROP_VARREF) + if (remove_global_object_property(ctx, p, pr, pr1)) + return -1; + } free_property(ctx->rt, pr1, pr->flags); JS_FreeAtom(ctx, pr->atom); /* put default values */ @@ -8746,8 +9049,8 @@ static int expand_fast_array(JSContext *ctx, JSObject *p, uint32_t new_len) /* Preconditions: 'p' must be of class JS_CLASS_ARRAY, p->fast_array = TRUE and p->extensible = TRUE */ -static int add_fast_array_element(JSContext *ctx, JSObject *p, - JSValue val, int flags) +static inline int add_fast_array_element(JSContext *ctx, JSObject *p, + JSValue val, int flags) { uint32_t new_len, array_len; /* extend the array by one */ @@ -8800,6 +9103,57 @@ static JSValue js_allocate_fast_array(JSContext *ctx, int64_t len) return arr; } +static JSValue js_create_array(JSContext *ctx, int len, JSValueConst *tab) +{ + JSValue obj; + JSObject *p; + int i; + + obj = JS_NewArray(ctx); + if (JS_IsException(obj)) + return JS_EXCEPTION; + if (len > 0) { + p = JS_VALUE_GET_OBJ(obj); + if (expand_fast_array(ctx, p, len) < 0) { + JS_FreeValue(ctx, obj); + return JS_EXCEPTION; + } + p->u.array.count = len; + for(i = 0; i < len; i++) + p->u.array.u.values[i] = JS_DupValue(ctx, tab[i]); + /* update the 'length' field */ + set_value(ctx, &p->prop[0].u.value, JS_NewInt32(ctx, len)); + } + return obj; +} + +static JSValue js_create_array_free(JSContext *ctx, int len, JSValue *tab) +{ + JSValue obj; + JSObject *p; + int i; + + obj = JS_NewArray(ctx); + if (JS_IsException(obj)) + goto fail; + if (len > 0) { + p = JS_VALUE_GET_OBJ(obj); + if (expand_fast_array(ctx, p, len) < 0) { + JS_FreeValue(ctx, obj); + fail: + for(i = 0; i < len; i++) + JS_FreeValue(ctx, tab[i]); + return JS_EXCEPTION; + } + p->u.array.count = len; + for(i = 0; i < len; i++) + p->u.array.u.values[i] = tab[i]; + /* update the 'length' field */ + set_value(ctx, &p->prop[0].u.value, JS_NewInt32(ctx, len)); + } + return obj; +} + static void js_free_desc(JSContext *ctx, JSPropertyDescriptor *desc) { JS_FreeValue(ctx, desc->getter); @@ -8808,11 +9162,9 @@ static void js_free_desc(JSContext *ctx, JSPropertyDescriptor *desc) } /* return -1 in case of exception or TRUE or FALSE. Warning: 'val' is - freed by the function. 'flags' is a bitmask of JS_PROP_NO_ADD, - JS_PROP_THROW or JS_PROP_THROW_STRICT. If JS_PROP_NO_ADD is set, - the new property is not added and an error is raised. 'this_obj' is - the receiver. If obj != this_obj, then obj must be an object - (Reflect.set case). */ + freed by the function. 'flags' is a bitmask of JS_PROP_THROW and + JS_PROP_THROW_STRICT. 'this_obj' is the receiver. If obj != + this_obj, then obj must be an object (Reflect.set case). */ int JS_SetPropertyInternal(JSContext *ctx, JSValueConst obj, JSAtom prop, JSValue val, JSValueConst this_obj, int flags) { @@ -8871,10 +9223,9 @@ int JS_SetPropertyInternal(JSContext *ctx, JSValueConst obj, } else if ((prs->flags & JS_PROP_TMASK) == JS_PROP_GETSET) { return call_setter(ctx, pr->u.getset.setter, this_obj, val, flags); } else if ((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF) { - /* JS_PROP_WRITABLE is always true for variable - references, but they are write protected in module name - spaces. */ - if (p->class_id == JS_CLASS_MODULE_NS) + /* XXX: already use var_ref->is_const. Cannot simplify use the + writable flag for JS_CLASS_MODULE_NS. */ + if (p->class_id == JS_CLASS_MODULE_NS || pr->u.var_ref->is_const) goto read_only_prop; set_value(ctx, pr->u.var_ref->pvalue, val); return TRUE; @@ -9008,12 +9359,6 @@ int JS_SetPropertyInternal(JSContext *ctx, JSValueConst obj, } } - if (unlikely(flags & JS_PROP_NO_ADD)) { - JS_FreeValue(ctx, val); - JS_ThrowReferenceErrorNotDefined(ctx, prop); - return -1; - } - if (unlikely(!p)) { JS_FreeValue(ctx, val); return JS_ThrowTypeErrorOrFalse(ctx, flags, "not an object"); @@ -9039,6 +9384,8 @@ int JS_SetPropertyInternal(JSContext *ctx, JSValueConst obj, goto generic_create_prop; } } else { + if (unlikely(p->class_id == JS_CLASS_GLOBAL_OBJECT)) + goto generic_create_prop; pr = add_property(ctx, p, prop, JS_PROP_C_W_E); if (unlikely(!pr)) { JS_FreeValue(ctx, val); @@ -9106,27 +9453,13 @@ static int JS_SetPropertyValue(JSContext *ctx, JSValueConst this_obj, switch(p->class_id) { case JS_CLASS_ARRAY: if (unlikely(idx >= (uint32_t)p->u.array.count)) { - JSObject *p1; - JSShape *sh1; - /* fast path to add an element to the array */ - if (idx != (uint32_t)p->u.array.count || - !p->fast_array || !p->extensible) + if (unlikely(idx != (uint32_t)p->u.array.count || + !p->fast_array || + !p->extensible || + p->shape->proto != JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]) || + !ctx->std_array_prototype)) { goto slow_path; - /* check if prototype chain has a numeric property */ - p1 = p->shape->proto; - while (p1 != NULL) { - sh1 = p1->shape; - if (p1->class_id == JS_CLASS_ARRAY) { - if (unlikely(!p1->fast_array)) - goto slow_path; - } else if (p1->class_id == JS_CLASS_OBJECT) { - if (unlikely(sh1->has_small_array_index)) - goto slow_path; - } else { - goto slow_path; - } - p1 = sh1->proto; } /* add element */ return add_fast_array_element(ctx, p, val, flags); @@ -9183,6 +9516,13 @@ static int JS_SetPropertyValue(JSContext *ctx, JSValueConst this_obj, p->u.array.u.uint64_ptr[idx] = v; } break; + case JS_CLASS_FLOAT16_ARRAY: + if (JS_ToFloat64Free(ctx, &d, val)) + return -1; + if (unlikely(idx >= (uint32_t)p->u.array.count)) + goto ta_out_of_bound; + p->u.array.u.fp16_ptr[idx] = tofp16(d); + break; case JS_CLASS_FLOAT32_ARRAY: if (JS_ToFloat64Free(ctx, &d, val)) return -1; @@ -9253,6 +9593,10 @@ int JS_SetPropertyStr(JSContext *ctx, JSValueConst this_obj, JSAtom atom; int ret; atom = JS_NewAtom(ctx, prop); + if (atom == JS_ATOM_NULL) { + JS_FreeValue(ctx, val); + return -1; + } ret = JS_SetPropertyInternal(ctx, this_obj, atom, val, this_obj, JS_PROP_THROW); JS_FreeAtom(ctx, atom); return ret; @@ -9276,7 +9620,9 @@ static int JS_CreateProperty(JSContext *ctx, JSObject *p, { JSProperty *pr; int ret, prop_flags; - + JSVarRef *var_ref; + JSObject *delete_obj; + /* add a new property or modify an existing exotic one */ if (p->is_exotic) { if (p->class_id == JS_CLASS_ARRAY) { @@ -9351,15 +9697,37 @@ static int JS_CreateProperty(JSContext *ctx, JSObject *p, return JS_ThrowTypeErrorOrFalse(ctx, flags, "object is not extensible"); } + var_ref = NULL; + delete_obj = NULL; if (flags & (JS_PROP_HAS_GET | JS_PROP_HAS_SET)) { prop_flags = (flags & (JS_PROP_CONFIGURABLE | JS_PROP_ENUMERABLE)) | JS_PROP_GETSET; } else { prop_flags = flags & JS_PROP_C_W_E; + if (p->class_id == JS_CLASS_GLOBAL_OBJECT) { + JSObject *p1 = JS_VALUE_GET_OBJ(p->u.global_object.uninitialized_vars); + JSShapeProperty *prs1; + JSProperty *pr1; + prs1 = find_own_property(&pr1, p1, prop); + if (prs1) { + delete_obj = p1; + var_ref = pr1->u.var_ref; + var_ref->header.ref_count++; + } else { + var_ref = js_create_var_ref(ctx, FALSE); + if (!var_ref) + return -1; + } + var_ref->is_const = !(prop_flags & JS_PROP_WRITABLE); + prop_flags |= JS_PROP_VARREF; + } } pr = add_property(ctx, p, prop, prop_flags); - if (unlikely(!pr)) + if (unlikely(!pr)) { + if (var_ref) + free_var_ref(ctx->rt, var_ref); return -1; + } if (flags & (JS_PROP_HAS_GET | JS_PROP_HAS_SET)) { pr->u.getset.getter = NULL; if ((flags & JS_PROP_HAS_GET) && JS_IsFunction(ctx, getter)) { @@ -9371,6 +9739,15 @@ static int JS_CreateProperty(JSContext *ctx, JSObject *p, pr->u.getset.setter = JS_VALUE_GET_OBJ(JS_DupValue(ctx, setter)); } + } else if (p->class_id == JS_CLASS_GLOBAL_OBJECT) { + if (delete_obj) + delete_property(ctx, delete_obj, prop); + pr->u.var_ref = var_ref; + if (flags & JS_PROP_HAS_VALUE) { + *var_ref->pvalue = JS_DupValue(ctx, val); + } else { + *var_ref->pvalue = JS_UNDEFINED; + } } else { if (flags & JS_PROP_HAS_VALUE) { pr->u.value = JS_DupValue(ctx, val); @@ -9525,6 +9902,10 @@ int JS_DefineProperty(JSContext *ctx, JSValueConst this_obj, return -1; /* convert to getset */ if ((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF) { + if (unlikely(p->class_id == JS_CLASS_GLOBAL_OBJECT)) { + if (remove_global_object_property(ctx, p, prs, pr)) + return -1; + } free_var_ref(ctx->rt, pr->u.var_ref); } else { JS_FreeValue(ctx, pr->u.value); @@ -9563,14 +9944,31 @@ int JS_DefineProperty(JSContext *ctx, JSValueConst this_obj, } else { if ((prs->flags & JS_PROP_TMASK) == JS_PROP_GETSET) { /* convert to data descriptor */ - if (js_shape_prepare_update(ctx, p, &prs)) + JSVarRef *var_ref; + if (unlikely(p->class_id == JS_CLASS_GLOBAL_OBJECT)) { + var_ref = js_global_object_find_uninitialized_var(ctx, p, prop, FALSE); + if (!var_ref) + return -1; + } else { + var_ref = NULL; + } + if (js_shape_prepare_update(ctx, p, &prs)) { + if (var_ref) + free_var_ref(ctx->rt, var_ref); return -1; + } if (pr->u.getset.getter) JS_FreeValue(ctx, JS_MKPTR(JS_TAG_OBJECT, pr->u.getset.getter)); if (pr->u.getset.setter) JS_FreeValue(ctx, JS_MKPTR(JS_TAG_OBJECT, pr->u.getset.setter)); - prs->flags &= ~(JS_PROP_TMASK | JS_PROP_WRITABLE); - pr->u.value = JS_UNDEFINED; + if (var_ref) { + prs->flags = (prs->flags & ~JS_PROP_TMASK) | + JS_PROP_VARREF | JS_PROP_WRITABLE; + pr->u.var_ref = var_ref; + } else { + prs->flags &= ~(JS_PROP_TMASK | JS_PROP_WRITABLE); + pr->u.value = JS_UNDEFINED; + } } else if ((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF) { /* Note: JS_PROP_VARREF is always writable */ } else { @@ -9597,8 +9995,6 @@ int JS_DefineProperty(JSContext *ctx, JSValueConst this_obj, JS_DupValue(ctx, val)); } } - /* if writable is set to false, no longer a - reference (for mapped arguments) */ if ((flags & (JS_PROP_HAS_WRITABLE | JS_PROP_WRITABLE)) == JS_PROP_HAS_WRITABLE) { JSValue val1; if (p->class_id == JS_CLASS_MODULE_NS) { @@ -9606,10 +10002,17 @@ int JS_DefineProperty(JSContext *ctx, JSValueConst this_obj, } if (js_shape_prepare_update(ctx, p, &prs)) return -1; - val1 = JS_DupValue(ctx, *pr->u.var_ref->pvalue); - free_var_ref(ctx->rt, pr->u.var_ref); - pr->u.value = val1; - prs->flags &= ~(JS_PROP_TMASK | JS_PROP_WRITABLE); + if (p->class_id == JS_CLASS_GLOBAL_OBJECT) { + pr->u.var_ref->is_const = TRUE; /* mark as read-only */ + prs->flags &= ~JS_PROP_WRITABLE; + } else { + /* if writable is set to false, no longer a + reference (for mapped arguments) */ + val1 = JS_DupValue(ctx, *pr->u.var_ref->pvalue); + free_var_ref(ctx->rt, pr->u.var_ref); + pr->u.value = val1; + prs->flags &= ~(JS_PROP_TMASK | JS_PROP_WRITABLE); + } } } else if (prs->flags & JS_PROP_LENGTH) { if (flags & JS_PROP_HAS_VALUE) { @@ -9809,6 +10212,10 @@ int JS_DefinePropertyValueStr(JSContext *ctx, JSValueConst this_obj, JSAtom atom; int ret; atom = JS_NewAtom(ctx, prop); + if (atom == JS_ATOM_NULL) { + JS_FreeValue(ctx, val); + return -1; + } ret = JS_DefinePropertyValue(ctx, this_obj, atom, val, flags); JS_FreeAtom(ctx, atom); return ret; @@ -9938,83 +10345,6 @@ static int JS_CheckDefineGlobalVar(JSContext *ctx, JSAtom prop, int flags) return 0; } -/* def_flags is (0, DEFINE_GLOBAL_LEX_VAR) | - JS_PROP_CONFIGURABLE | JS_PROP_WRITABLE */ -/* XXX: could support exotic global object. */ -static int JS_DefineGlobalVar(JSContext *ctx, JSAtom prop, int def_flags) -{ - JSObject *p; - JSShapeProperty *prs; - JSProperty *pr; - JSValue val; - int flags; - - if (def_flags & DEFINE_GLOBAL_LEX_VAR) { - p = JS_VALUE_GET_OBJ(ctx->global_var_obj); - flags = JS_PROP_ENUMERABLE | (def_flags & JS_PROP_WRITABLE) | - JS_PROP_CONFIGURABLE; - val = JS_UNINITIALIZED; - } else { - p = JS_VALUE_GET_OBJ(ctx->global_obj); - flags = JS_PROP_ENUMERABLE | JS_PROP_WRITABLE | - (def_flags & JS_PROP_CONFIGURABLE); - val = JS_UNDEFINED; - } - prs = find_own_property1(p, prop); - if (prs) - return 0; - if (!p->extensible) - return 0; - pr = add_property(ctx, p, prop, flags); - if (unlikely(!pr)) - return -1; - pr->u.value = val; - return 0; -} - -/* 'def_flags' is 0 or JS_PROP_CONFIGURABLE. */ -/* XXX: could support exotic global object. */ -static int JS_DefineGlobalFunction(JSContext *ctx, JSAtom prop, - JSValueConst func, int def_flags) -{ - - JSObject *p; - JSShapeProperty *prs; - int flags; - - p = JS_VALUE_GET_OBJ(ctx->global_obj); - prs = find_own_property1(p, prop); - flags = JS_PROP_HAS_VALUE | JS_PROP_THROW; - if (!prs || (prs->flags & JS_PROP_CONFIGURABLE)) { - flags |= JS_PROP_ENUMERABLE | JS_PROP_WRITABLE | def_flags | - JS_PROP_HAS_CONFIGURABLE | JS_PROP_HAS_WRITABLE | JS_PROP_HAS_ENUMERABLE; - } - if (JS_DefineProperty(ctx, ctx->global_obj, prop, func, - JS_UNDEFINED, JS_UNDEFINED, flags) < 0) - return -1; - return 0; -} - -static JSValue JS_GetGlobalVar(JSContext *ctx, JSAtom prop, - BOOL throw_ref_error) -{ - JSObject *p; - JSShapeProperty *prs; - JSProperty *pr; - - /* no exotic behavior is possible in global_var_obj */ - p = JS_VALUE_GET_OBJ(ctx->global_var_obj); - prs = find_own_property(&pr, p, prop); - if (prs) { - /* XXX: should handle JS_PROP_TMASK properties */ - if (unlikely(JS_IsUninitialized(pr->u.value))) - return JS_ThrowReferenceErrorUninitialized(ctx, prs->atom); - return JS_DupValue(ctx, pr->u.value); - } - return JS_GetPropertyInternal(ctx, ctx->global_obj, prop, - ctx->global_obj, throw_ref_error); -} - /* construct a reference to a global variable */ static int JS_GetGlobalVarRef(JSContext *ctx, JSAtom prop, JSValue *sp) { @@ -10026,10 +10356,9 @@ static int JS_GetGlobalVarRef(JSContext *ctx, JSAtom prop, JSValue *sp) p = JS_VALUE_GET_OBJ(ctx->global_var_obj); prs = find_own_property(&pr, p, prop); if (prs) { - /* XXX: should handle JS_PROP_AUTOINIT properties? */ /* XXX: conformance: do these tests in OP_put_var_ref/OP_get_var_ref ? */ - if (unlikely(JS_IsUninitialized(pr->u.value))) { + if (unlikely(JS_IsUninitialized(*pr->u.var_ref->pvalue))) { JS_ThrowReferenceErrorUninitialized(ctx, prs->atom); return -1; } @@ -10052,66 +10381,6 @@ static int JS_GetGlobalVarRef(JSContext *ctx, JSAtom prop, JSValue *sp) return 0; } -/* use for strict variable access: test if the variable exists */ -static int JS_CheckGlobalVar(JSContext *ctx, JSAtom prop) -{ - JSObject *p; - JSShapeProperty *prs; - int ret; - - /* no exotic behavior is possible in global_var_obj */ - p = JS_VALUE_GET_OBJ(ctx->global_var_obj); - prs = find_own_property1(p, prop); - if (prs) { - ret = TRUE; - } else { - ret = JS_HasProperty(ctx, ctx->global_obj, prop); - if (ret < 0) - return -1; - } - return ret; -} - -/* flag = 0: normal variable write - flag = 1: initialize lexical variable - flag = 2: normal variable write, strict check was done before -*/ -static int JS_SetGlobalVar(JSContext *ctx, JSAtom prop, JSValue val, - int flag) -{ - JSObject *p; - JSShapeProperty *prs; - JSProperty *pr; - int flags; - - /* no exotic behavior is possible in global_var_obj */ - p = JS_VALUE_GET_OBJ(ctx->global_var_obj); - prs = find_own_property(&pr, p, prop); - if (prs) { - /* XXX: should handle JS_PROP_AUTOINIT properties? */ - if (flag != 1) { - if (unlikely(JS_IsUninitialized(pr->u.value))) { - JS_FreeValue(ctx, val); - JS_ThrowReferenceErrorUninitialized(ctx, prs->atom); - return -1; - } - if (unlikely(!(prs->flags & JS_PROP_WRITABLE))) { - JS_FreeValue(ctx, val); - return JS_ThrowTypeErrorReadOnly(ctx, JS_PROP_THROW, prop); - } - } - set_value(ctx, &pr->u.value, val); - return 0; - } - /* XXX: add a fast path where the property exists and the object - is not exotic. Otherwise do as in OP_put_ref_value and remove - JS_PROP_NO_ADD which is no longer necessary */ - flags = JS_PROP_THROW_STRICT; - if (is_strict_mode(ctx)) - flags |= JS_PROP_NO_ADD; - return JS_SetPropertyInternal(ctx, ctx->global_obj, prop, val, ctx->global_obj, flags); -} - /* return -1, FALSE or TRUE */ static int JS_DeleteGlobalVar(JSContext *ctx, JSAtom prop) { @@ -10522,6 +10791,15 @@ static inline js_limb_t js_limb_clz(js_limb_t a) } #endif +/* handle a = 0 too */ +static inline js_limb_t js_limb_safe_clz(js_limb_t a) +{ + if (a == 0) + return JS_LIMB_BITS; + else + return js_limb_clz(a); +} + static js_limb_t mp_add(js_limb_t *res, const js_limb_t *op1, const js_limb_t *op2, js_limb_t n, js_limb_t carry) { @@ -11667,6 +11945,7 @@ static JSBigInt *js_bigint_from_string(JSContext *ctx, const char *str, int radix) { const char *p = str; + size_t n_digits1; int is_neg, n_digits, n_limbs, len, log2_radix, n_bits, i; JSBigInt *r; js_limb_t v, c, h; @@ -11678,10 +11957,16 @@ static JSBigInt *js_bigint_from_string(JSContext *ctx, } while (*p == '0') p++; - n_digits = strlen(p); + n_digits1 = strlen(p); + /* the real check for overflox is done js_bigint_new(). Here + we just avoid integer overflow */ + if (n_digits1 > JS_BIGINT_MAX_SIZE * JS_LIMB_BITS) { + JS_ThrowRangeError(ctx, "BigInt is too large to allocate"); + return NULL; + } + n_digits = n_digits1; log2_radix = 32 - clz32(radix - 1); /* ceil(log2(radix)) */ /* compute the maximum number of limbs */ - /* XXX: overflow */ if (radix == 10) { n_bits = (n_digits * 27 + 7) / 8; /* >= ceil(n_digits * log2(10)) */ } else { @@ -11870,7 +12155,7 @@ static JSValue js_bigint_to_string1(JSContext *ctx, JSValueConst val, int radix) r = tmp; } log2_radix = 31 - clz32(radix); /* floor(log2(radix)) */ - n_bits = r->len * JS_LIMB_BITS - js_limb_clz(r->tab[r->len - 1]); + n_bits = r->len * JS_LIMB_BITS - js_limb_safe_clz(r->tab[r->len - 1]); /* n_digits is exact only if radix is a power of two. Otherwise it is >= the exact number of digits */ n_digits = (n_bits + log2_radix - 1) / log2_radix; @@ -11912,11 +12197,10 @@ static JSValue js_bigint_to_string1(JSContext *ctx, JSValueConst val, int radix) bit_pos = i * log2_radix; pos = bit_pos / JS_LIMB_BITS; shift = bit_pos % JS_LIMB_BITS; - if (likely((shift + log2_radix) <= JS_LIMB_BITS)) { - c = r->tab[pos] >> shift; - } else { - c = (r->tab[pos] >> shift) | - (r->tab[pos + 1] << (JS_LIMB_BITS - shift)); + c = r->tab[pos] >> shift; + if ((shift + log2_radix) > JS_LIMB_BITS && + (pos + 1) < r->len) { + c |= r->tab[pos + 1] << (JS_LIMB_BITS - shift); } c &= (radix - 1); *--q = digits[c]; @@ -12117,7 +12401,7 @@ static JSValue js_atof(JSContext *ctx, const char *str, const char **pp, case ATOD_TYPE_FLOAT64: { double d; - d = js_atod(buf,NULL, radix, is_float ? 0 : JS_ATOD_INT_ONLY, + d = js_atod(buf, NULL, radix, is_float ? 0 : JS_ATOD_INT_ONLY, &atod_mem); /* return int or float64 */ val = JS_NewFloat64(ctx, d); @@ -12129,8 +12413,10 @@ static JSValue js_atof(JSContext *ctx, const char *str, const char **pp, if (has_legacy_octal || is_float) goto fail; r = js_bigint_from_string(ctx, buf, radix); - if (!r) - goto mem_error; + if (!r) { + val = JS_EXCEPTION; + goto done; + } val = JS_CompactBigInt(ctx, r); } break; @@ -12889,93 +13175,48 @@ static JSValue JS_ToStringCheckObject(JSContext *ctx, JSValueConst val) return JS_ToString(ctx, val); } -static JSValue JS_ToQuotedString(JSContext *ctx, JSValueConst val1) -{ - JSValue val; - JSString *p; - int i; - uint32_t c; - StringBuffer b_s, *b = &b_s; - char buf[16]; - - val = JS_ToStringCheckObject(ctx, val1); - if (JS_IsException(val)) - return val; - p = JS_VALUE_GET_STRING(val); - - if (string_buffer_init(ctx, b, p->len + 2)) - goto fail; - - if (string_buffer_putc8(b, '\"')) - goto fail; - for(i = 0; i < p->len; ) { - c = string_getc(p, &i); - switch(c) { - case '\t': - c = 't'; - goto quote; - case '\r': - c = 'r'; - goto quote; - case '\n': - c = 'n'; - goto quote; - case '\b': - c = 'b'; - goto quote; - case '\f': - c = 'f'; - goto quote; - case '\"': - case '\\': - quote: - if (string_buffer_putc8(b, '\\')) - goto fail; - if (string_buffer_putc8(b, c)) - goto fail; - break; - default: - if (c < 32 || is_surrogate(c)) { - snprintf(buf, sizeof(buf), "\\u%04x", c); - if (string_buffer_puts8(b, buf)) - goto fail; - } else { - if (string_buffer_putc(b, c)) - goto fail; - } - break; - } - } - if (string_buffer_putc8(b, '\"')) - goto fail; - JS_FreeValue(ctx, val); - return string_buffer_end(b); - fail: - JS_FreeValue(ctx, val); - string_buffer_free(b); - return JS_EXCEPTION; -} - #define JS_PRINT_MAX_DEPTH 8 typedef struct { JSRuntime *rt; JSContext *ctx; /* may be NULL */ JSPrintValueOptions options; - FILE *fo; + JSPrintValueWrite *write_func; + void *write_opaque; int level; JSObject *print_stack[JS_PRINT_MAX_DEPTH]; /* level values */ } JSPrintValueState; static void js_print_value(JSPrintValueState *s, JSValueConst val); +static void js_putc(JSPrintValueState *s, char c) +{ + s->write_func(s->write_opaque, &c, 1); +} + +static void js_puts(JSPrintValueState *s, const char *str) +{ + s->write_func(s->write_opaque, str, strlen(str)); +} + +static void __attribute__((format(printf, 2, 3))) js_printf(JSPrintValueState *s, const char *fmt, ...) +{ + va_list ap; + char buf[256]; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + s->write_func(s->write_opaque, buf, strlen(buf)); +} + static void js_print_float64(JSPrintValueState *s, double d) { JSDTOATempMem dtoa_mem; char buf[32]; int len; len = js_dtoa(buf, d, 10, 0, JS_DTOA_FORMAT_FREE | JS_DTOA_MINUS_ZERO, &dtoa_mem); - fwrite(buf, 1, len, s->fo); + s->write_func(s->write_opaque, buf, len); } static uint32_t js_string_get_length(JSValueConst val) @@ -12991,24 +13232,80 @@ static uint32_t js_string_get_length(JSValueConst val) } } +/* pretty print the first 'len' characters of 'p' */ +static void js_print_string1(JSPrintValueState *s, JSString *p, int len, int sep) +{ + uint8_t buf[UTF8_CHAR_LEN_MAX]; + int l, i, c, c1; + + for(i = 0; i < len; i++) { + c = string_get(p, i); + switch(c) { + case '\t': + c = 't'; + goto quote; + case '\r': + c = 'r'; + goto quote; + case '\n': + c = 'n'; + goto quote; + case '\b': + c = 'b'; + goto quote; + case '\f': + c = 'f'; + goto quote; + case '\\': + quote: + js_putc(s, '\\'); + js_putc(s, c); + break; + default: + if (c == sep) + goto quote; + if (c >= 32 && c <= 126) { + js_putc(s, c); + } else if (c < 32 || + (c >= 0x7f && c <= 0x9f)) { + escape: + js_printf(s, "\\u%04x", c); + } else { + if (is_hi_surrogate(c)) { + if ((i + 1) >= len) + goto escape; + c1 = string_get(p, i + 1); + if (!is_lo_surrogate(c1)) + goto escape; + i++; + c = from_surrogate(c, c1); + } else if (is_lo_surrogate(c)) { + goto escape; + } + l = unicode_to_utf8(buf, c); + s->write_func(s->write_opaque, (char *)buf, l); + } + break; + } + } +} + static void js_print_string_rec(JSPrintValueState *s, JSValueConst val, - int sep, uint32_t pos) + int sep, uint32_t pos) { if (JS_VALUE_GET_TAG(val) == JS_TAG_STRING) { JSString *p = JS_VALUE_GET_STRING(val); - uint32_t i, len; + uint32_t len; if (pos < s->options.max_string_length) { len = min_uint32(p->len, s->options.max_string_length - pos); - for(i = 0; i < len; i++) { - JS_DumpChar(s->fo, string_get(p, i), sep); - } + js_print_string1(s, p, len, sep); } } else if (JS_VALUE_GET_TAG(val) == JS_TAG_STRING_ROPE) { JSStringRope *r = JS_VALUE_GET_PTR(val); js_print_string_rec(s, r->left, sep, pos); js_print_string_rec(s, r->right, sep, pos + js_string_get_length(r->left)); } else { - fprintf(s->fo, "", (int)JS_VALUE_GET_TAG(val)); + js_printf(s, "", (int)JS_VALUE_GET_TAG(val)); } } @@ -13017,38 +13314,31 @@ static void js_print_string(JSPrintValueState *s, JSValueConst val) int sep; if (s->options.raw_dump && JS_VALUE_GET_TAG(val) == JS_TAG_STRING) { JSString *p = JS_VALUE_GET_STRING(val); - fprintf(s->fo, "%d", p->header.ref_count); + js_printf(s, "%d", p->header.ref_count); sep = (p->header.ref_count == 1) ? '\"' : '\''; } else { sep = '\"'; } - fputc(sep, s->fo); + js_putc(s, sep); js_print_string_rec(s, val, sep, 0); - fputc(sep, s->fo); + js_putc(s, sep); if (js_string_get_length(val) > s->options.max_string_length) { uint32_t n = js_string_get_length(val) - s->options.max_string_length; - fprintf(s->fo, "... %u more character%s", n, n > 1 ? "s" : ""); + js_printf(s, "... %u more character%s", n, n > 1 ? "s" : ""); } } -static void js_print_raw_string2(JSPrintValueState *s, JSValueConst val, BOOL remove_last_lf) +static void js_print_raw_string(JSPrintValueState *s, JSValueConst val) { const char *cstr; size_t len; cstr = JS_ToCStringLen(s->ctx, &len, val); if (cstr) { - if (remove_last_lf && len > 0 && cstr[len - 1] == '\n') - len--; - fwrite(cstr, 1, len, s->fo); + s->write_func(s->write_opaque, cstr, len); JS_FreeCString(s->ctx, cstr); } } -static void js_print_raw_string(JSPrintValueState *s, JSValueConst val) -{ - js_print_raw_string2(s, val, FALSE); -} - static BOOL is_ascii_ident(const JSString *p) { int i, c; @@ -13064,27 +13354,25 @@ static BOOL is_ascii_ident(const JSString *p) return TRUE; } -static void js_print_atom(JSRuntime *rt, FILE *fo, JSAtom atom) +static void js_print_atom(JSPrintValueState *s, JSAtom atom) { int i; if (__JS_AtomIsTaggedInt(atom)) { - fprintf(fo, "%u", __JS_AtomToUInt32(atom)); + js_printf(s, "%u", __JS_AtomToUInt32(atom)); } else if (atom == JS_ATOM_NULL) { - fprintf(fo, ""); + js_puts(s, ""); } else { - assert(atom < rt->atom_size); + assert(atom < s->rt->atom_size); JSString *p; - p = rt->atom_array[atom]; + p = s->rt->atom_array[atom]; if (is_ascii_ident(p)) { for(i = 0; i < p->len; i++) { - fputc(string_get(p, i), fo); + js_putc(s, string_get(p, i)); } } else { - fputc('"', fo); - for(i = 0; i < p->len; i++) { - JS_DumpChar(fo, string_get(p, i), '\"'); - } - fputc('"', fo); + js_putc(s, '"'); + js_print_string1(s, p, p->len, '\"'); + js_putc(s, '"'); } } } @@ -13118,10 +13406,10 @@ static void js_print_comma(JSPrintValueState *s, int *pcomma_state) case 0: break; case 1: - fprintf(s->fo, ", "); + js_printf(s, ", "); break; case 2: - fprintf(s->fo, " { "); + js_printf(s, " { "); break; } *pcomma_state = 1; @@ -13131,7 +13419,110 @@ static void js_print_more_items(JSPrintValueState *s, int *pcomma_state, uint32_t n) { js_print_comma(s, pcomma_state); - fprintf(s->fo, "... %u more item%s", n, n > 1 ? "s" : ""); + js_printf(s, "... %u more item%s", n, n > 1 ? "s" : ""); +} + +/* similar to js_regexp_toString() but without side effect */ +static void js_print_regexp(JSPrintValueState *s, JSObject *p1) +{ + JSRegExp *re = &p1->u.regexp; + JSString *p; + int i, n, c, c2, bra, flags; + static const char regexp_flags[] = { 'g', 'i', 'm', 's', 'u', 'y', 'd', 'v' }; + + if (!re->pattern || !re->bytecode) { + /* the regexp fields are zeroed at init */ + js_puts(s, "[uninitialized_regexp]"); + return; + } + p = re->pattern; + js_putc(s, '/'); + if (p->len == 0) { + js_puts(s, "(?:)"); + } else { + bra = 0; + for (i = 0, n = p->len; i < n;) { + c2 = -1; + switch (c = string_get(p, i++)) { + case '\\': + if (i < n) + c2 = string_get(p, i++); + break; + case ']': + bra = 0; + break; + case '[': + if (!bra) { + if (i < n && string_get(p, i) == ']') + c2 = string_get(p, i++); + bra = 1; + } + break; + case '\n': + c = '\\'; + c2 = 'n'; + break; + case '\r': + c = '\\'; + c2 = 'r'; + break; + case '/': + if (!bra) { + c = '\\'; + c2 = '/'; + } + break; + } + js_putc(s, c); + if (c2 >= 0) + js_putc(s, c2); + } + } + js_putc(s, '/'); + + flags = lre_get_flags(re->bytecode->u.str8); + for(i = 0; i < countof(regexp_flags); i++) { + if ((flags >> i) & 1) { + js_putc(s, regexp_flags[i]); + } + } +} + +/* similar to js_error_toString() but without side effect */ +static void js_print_error(JSPrintValueState *s, JSObject *p) +{ + const char *str; + size_t len; + + str = get_prop_string(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p), JS_ATOM_name); + if (!str) { + js_puts(s, "Error"); + } else { + js_puts(s, str); + JS_FreeCString(s->ctx, str); + } + + str = get_prop_string(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p), JS_ATOM_message); + if (str && str[0] != '\0') { + js_puts(s, ": "); + js_puts(s, str); + } + JS_FreeCString(s->ctx, str); + + /* dump the stack if present */ + str = get_prop_string(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p), JS_ATOM_stack); + if (str) { + js_putc(s, '\n'); + + /* XXX: should remove the last '\n' in stack as + v8. SpiderMonkey does not do it */ + len = strlen(str); + if (len > 0 && str[len - 1] == '\n') + len--; + s->write_func(s->write_opaque, str, len); + + JS_FreeCString(s->ctx, str); + } } static void js_print_object(JSPrintValueState *s, JSObject *p) @@ -13148,7 +13539,7 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) is_array = FALSE; if (p->class_id == JS_CLASS_ARRAY) { is_array = TRUE; - fprintf(s->fo, "[ "); + js_printf(s, "[ "); /* XXX: print array like properties even if not fast array */ if (p->fast_array) { uint32_t len, n, len1; @@ -13164,7 +13555,7 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) if (p->u.array.count < len) { n = len - p->u.array.count; js_print_comma(s, &comma_state); - fprintf(s->fo, "<%u empty item%s>", n, n > 1 ? "s" : ""); + js_printf(s, "<%u empty item%s>", n, n > 1 ? "s" : ""); } } } else if (p->class_id >= JS_CLASS_UINT8C_ARRAY && p->class_id <= JS_CLASS_FLOAT64_ARRAY) { @@ -13172,8 +13563,8 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) uint32_t len1; int64_t v; - js_print_atom(s->rt, s->fo, rt->class_array[p->class_id].class_name); - fprintf(s->fo, "(%u) [ ", p->u.array.count); + js_print_atom(s, rt->class_array[p->class_id].class_name); + js_printf(s, "(%u) [ ", p->u.array.count); is_array = TRUE; len1 = min_uint32(p->u.array.count, s->options.max_item_count); @@ -13203,10 +13594,13 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) case JS_CLASS_BIG_INT64_ARRAY: v = *(int64_t *)ptr; ta_int64: - fprintf(s->fo, "%" PRId64, v); + js_printf(s, "%" PRId64, v); break; case JS_CLASS_BIG_UINT64_ARRAY: - fprintf(s->fo, "%" PRIu64, *(uint64_t *)ptr); + js_printf(s, "%" PRIu64, *(uint64_t *)ptr); + break; + case JS_CLASS_FLOAT16_ARRAY: + js_print_float64(s, fromfp16(*(uint16_t *)ptr)); break; case JS_CLASS_FLOAT32_ARRAY: js_print_float64(s, *(float *)ptr); @@ -13221,19 +13615,19 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) } else if (p->class_id == JS_CLASS_BYTECODE_FUNCTION || (rt->class_array[p->class_id].call != NULL && p->class_id != JS_CLASS_PROXY)) { - fprintf(s->fo, "[Function"); + js_printf(s, "[Function"); /* XXX: allow dump without ctx */ if (!s->options.raw_dump && s->ctx) { const char *func_name_str; - fputc(' ', s->fo); - func_name_str = get_func_name(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p)); + js_putc(s, ' '); + func_name_str = get_prop_string(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p), JS_ATOM_name); if (!func_name_str || func_name_str[0] == '\0') - fputs("(anonymous)", s->fo); + js_puts(s, "(anonymous)"); else - fputs(func_name_str, s->fo); + js_puts(s, func_name_str); JS_FreeCString(s->ctx, func_name_str); } - fprintf(s->fo, "]"); + js_printf(s, "]"); comma_state = 2; } else if (p->class_id == JS_CLASS_MAP || p->class_id == JS_CLASS_SET) { JSMapState *ms = p->u.opaque; @@ -13241,8 +13635,8 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) if (!ms) goto default_obj; - js_print_atom(s->rt, s->fo, rt->class_array[p->class_id].class_name); - fprintf(s->fo, "(%u) { ", ms->record_count); + js_print_atom(s, rt->class_array[p->class_id].class_name); + js_printf(s, "(%u) { ", ms->record_count); i = 0; list_for_each(el, &ms->records) { JSMapRecord *mr = list_entry(el, JSMapRecord, link); @@ -13251,7 +13645,7 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) continue; js_print_value(s, mr->key); if (p->class_id == JS_CLASS_MAP) { - fprintf(s->fo, " => "); + js_printf(s, " => "); js_print_value(s, mr->value); } i++; @@ -13260,43 +13654,27 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) } if (i < ms->record_count) js_print_more_items(s, &comma_state, ms->record_count - i); - } else if (p->class_id == JS_CLASS_REGEXP && s->ctx && !s->options.raw_dump) { - JSValue str = js_regexp_toString(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p), 0, NULL); - if (JS_IsException(str)) - goto default_obj; - js_print_raw_string(s, str); - JS_FreeValueRT(s->rt, str); + } else if (p->class_id == JS_CLASS_REGEXP && s->ctx) { + js_print_regexp(s, p); comma_state = 2; - } else if (p->class_id == JS_CLASS_DATE && s->ctx && !s->options.raw_dump) { + } else if (p->class_id == JS_CLASS_DATE && s->ctx) { + /* get_date_string() has no side effect */ JSValue str = get_date_string(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p), 0, NULL, 0x23); /* toISOString() */ if (JS_IsException(str)) goto default_obj; js_print_raw_string(s, str); JS_FreeValueRT(s->rt, str); comma_state = 2; - } else if (p->class_id == JS_CLASS_ERROR && s->ctx && !s->options.raw_dump) { - JSValue str = js_error_toString(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p), 0, NULL); - if (JS_IsException(str)) - goto default_obj; - js_print_raw_string(s, str); - JS_FreeValueRT(s->rt, str); - /* dump the stack if present */ - str = JS_GetProperty(s->ctx, JS_MKPTR(JS_TAG_OBJECT, p), JS_ATOM_stack); - if (JS_IsString(str)) { - fputc('\n', s->fo); - /* XXX: should remove the last '\n' in stack as - v8. SpiderMonkey does not do it */ - js_print_raw_string2(s, str, TRUE); - } - JS_FreeValueRT(s->rt, str); + } else if (p->class_id == JS_CLASS_ERROR && s->ctx) { + js_print_error(s, p); comma_state = 2; } else { default_obj: if (p->class_id != JS_CLASS_OBJECT) { - js_print_atom(s->rt, s->fo, rt->class_array[p->class_id].class_name); - fprintf(s->fo, " "); + js_print_atom(s, rt->class_array[p->class_id].class_name); + js_printf(s, " "); } - fprintf(s->fo, "{ "); + js_printf(s, "{ "); } sh = p->shape; /* the shape can be NULL while freeing an object */ @@ -13313,39 +13691,39 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) if (j < s->options.max_item_count) { pr = &p->prop[i]; js_print_comma(s, &comma_state); - js_print_atom(s->rt, s->fo, prs->atom); - fprintf(s->fo, ": "); + js_print_atom(s, prs->atom); + js_printf(s, ": "); /* XXX: autoinit property */ if ((prs->flags & JS_PROP_TMASK) == JS_PROP_GETSET) { if (s->options.raw_dump) { - fprintf(s->fo, "[Getter %p Setter %p]", + js_printf(s, "[Getter %p Setter %p]", pr->u.getset.getter, pr->u.getset.setter); } else { if (pr->u.getset.getter && pr->u.getset.setter) { - fprintf(s->fo, "[Getter/Setter]"); + js_printf(s, "[Getter/Setter]"); } else if (pr->u.getset.setter) { - fprintf(s->fo, "[Setter]"); + js_printf(s, "[Setter]"); } else { - fprintf(s->fo, "[Getter]"); + js_printf(s, "[Getter]"); } } } else if ((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF) { if (s->options.raw_dump) { - fprintf(s->fo, "[varref %p]", (void *)pr->u.var_ref); + js_printf(s, "[varref %p]", (void *)pr->u.var_ref); } else { js_print_value(s, *pr->u.var_ref->pvalue); } } else if ((prs->flags & JS_PROP_TMASK) == JS_PROP_AUTOINIT) { if (s->options.raw_dump) { - fprintf(s->fo, "[autoinit %p %d %p]", + js_printf(s, "[autoinit %p %d %p]", (void *)js_autoinit_get_realm(pr), js_autoinit_get_id(pr), (void *)pr->u.init.opaque); } else { /* XXX: could autoinit but need to restart the iteration */ - fprintf(s->fo, "[autoinit]"); + js_printf(s, "[autoinit]"); } } else { js_print_value(s, pr->u.value); @@ -13357,34 +13735,34 @@ static void js_print_object(JSPrintValueState *s, JSObject *p) if (j > s->options.max_item_count) js_print_more_items(s, &comma_state, j - s->options.max_item_count); } - if (s->options.show_closure && js_class_has_bytecode(p->class_id)) { + if (s->options.raw_dump && js_class_has_bytecode(p->class_id)) { JSFunctionBytecode *b = p->u.func.function_bytecode; if (b->closure_var_count) { JSVarRef **var_refs; var_refs = p->u.func.var_refs; js_print_comma(s, &comma_state); - fprintf(s->fo, "[[Closure]]: ["); + js_printf(s, "[[Closure]]: ["); for(i = 0; i < b->closure_var_count; i++) { if (i != 0) - fprintf(s->fo, ", "); + js_printf(s, ", "); js_print_value(s, var_refs[i]->value); } - fprintf(s->fo, " ]"); + js_printf(s, " ]"); } if (p->u.func.home_object) { js_print_comma(s, &comma_state); - fprintf(s->fo, "[[HomeObject]]: "); + js_printf(s, "[[HomeObject]]: "); js_print_value(s, JS_MKPTR(JS_TAG_OBJECT, p->u.func.home_object)); } } if (!is_array) { if (comma_state != 2) { - fprintf(s->fo, " }"); + js_printf(s, " }"); } } else { - fprintf(s->fo, " ]"); + js_printf(s, " ]"); } } @@ -13404,7 +13782,7 @@ static void js_print_value(JSPrintValueState *s, JSValueConst val) switch(tag) { case JS_TAG_INT: - fprintf(s->fo, "%d", JS_VALUE_GET_INT(val)); + js_printf(s, "%d", JS_VALUE_GET_INT(val)); break; case JS_TAG_BOOL: if (JS_VALUE_GET_BOOL(val)) @@ -13424,13 +13802,13 @@ static void js_print_value(JSPrintValueState *s, JSValueConst val) case JS_TAG_UNDEFINED: str = "undefined"; print_str: - fprintf(s->fo, "%s", str); + js_puts(s, str); break; case JS_TAG_FLOAT64: js_print_float64(s, JS_VALUE_GET_FLOAT64(val)); break; case JS_TAG_SHORT_BIG_INT: - fprintf(s->fo, "%" PRId64 "n", (int64_t)JS_VALUE_GET_SHORT_BIG_INT(val)); + js_printf(s, "%" PRId64 "n", (int64_t)JS_VALUE_GET_SHORT_BIG_INT(val)); break; case JS_TAG_BIG_INT: if (!s->options.raw_dump && s->ctx) { @@ -13438,7 +13816,7 @@ static void js_print_value(JSPrintValueState *s, JSValueConst val) if (JS_IsException(str)) goto raw_bigint; js_print_raw_string(s, str); - fputc('n', s->fo); + js_putc(s, 'n'); JS_FreeValueRT(s->rt, str); } else { JSBigInt *p; @@ -13448,27 +13826,27 @@ static void js_print_value(JSPrintValueState *s, JSValueConst val) /* In order to avoid allocations we just dump the limbs */ sgn = js_bigint_sign(p); if (sgn) - fprintf(s->fo, "BigInt.asIntN(%d,", p->len * JS_LIMB_BITS); - fprintf(s->fo, "0x"); + js_printf(s, "BigInt.asIntN(%d,", p->len * JS_LIMB_BITS); + js_printf(s, "0x"); for(i = p->len - 1; i >= 0; i--) { if (i != p->len - 1) - fprintf(s->fo, "_"); + js_putc(s, '_'); #if JS_LIMB_BITS == 32 - fprintf(s->fo, "%08x", p->tab[i]); + js_printf(s, "%08x", p->tab[i]); #else - fprintf(s->fo, "%016" PRIx64, p->tab[i]); + js_printf(s, "%016" PRIx64, p->tab[i]); #endif } - fprintf(s->fo, "n"); + js_putc(s, 'n'); if (sgn) - fprintf(s->fo, ")"); + js_putc(s, ')'); } break; case JS_TAG_STRING: case JS_TAG_STRING_ROPE: if (s->options.raw_dump && tag == JS_TAG_STRING_ROPE) { JSStringRope *r = JS_VALUE_GET_STRING_ROPE(val); - fprintf(s->fo, "[rope len=%d depth=%d]", r->len, r->depth); + js_printf(s, "[rope len=%d depth=%d]", r->len, r->depth); } else { js_print_string(s, val); } @@ -13476,9 +13854,9 @@ static void js_print_value(JSPrintValueState *s, JSValueConst val) case JS_TAG_FUNCTION_BYTECODE: { JSFunctionBytecode *b = JS_VALUE_GET_PTR(val); - fprintf(s->fo, "[bytecode "); - js_print_atom(s->rt, s->fo, b->func_name); - fprintf(s->fo, "]"); + js_puts(s, "[bytecode "); + js_print_atom(s, b->func_name); + js_putc(s, ']'); } break; case JS_TAG_OBJECT: @@ -13487,35 +13865,35 @@ static void js_print_value(JSPrintValueState *s, JSValueConst val) int idx; idx = js_print_stack_index(s, p); if (idx >= 0) { - fprintf(s->fo, "[circular %d]", idx); + js_printf(s, "[circular %d]", idx); } else if (s->level < s->options.max_depth) { s->print_stack[s->level++] = p; js_print_object(s, JS_VALUE_GET_OBJ(val)); s->level--; } else { JSAtom atom = s->rt->class_array[p->class_id].class_name; - fprintf(s->fo, "["); - js_print_atom(s->rt, s->fo, atom); + js_putc(s, '['); + js_print_atom(s, atom); if (s->options.raw_dump) { - fprintf(s->fo, " %p", (void *)p); + js_printf(s, " %p", (void *)p); } - fprintf(s->fo, "]"); + js_putc(s, ']'); } } break; case JS_TAG_SYMBOL: { JSAtomStruct *p = JS_VALUE_GET_PTR(val); - fprintf(s->fo, "Symbol("); - js_print_atom(s->rt, s->fo, js_get_atom_index(s->rt, p)); - fprintf(s->fo, ")"); + js_puts(s, "Symbol("); + js_print_atom(s, js_get_atom_index(s->rt, p)); + js_putc(s, ')'); } break; case JS_TAG_MODULE: - fprintf(s->fo, "[module]"); + js_puts(s, "[module]"); break; default: - fprintf(s->fo, "[unknown tag %d]", tag); + js_printf(s, "[unknown tag %d]", tag); break; } } @@ -13528,8 +13906,9 @@ void JS_PrintValueSetDefaultOptions(JSPrintValueOptions *options) options->max_item_count = 100; } -static void JS_PrintValueInternal(JSRuntime *rt, JSContext *ctx, - FILE *fo, JSValueConst val, const JSPrintValueOptions *options) +static void JS_PrintValueInternal(JSRuntime *rt, JSContext *ctx, + JSPrintValueWrite *write_func, void *write_opaque, + JSValueConst val, const JSPrintValueOptions *options) { JSPrintValueState ss, *s = &ss; if (options) @@ -13546,32 +13925,59 @@ static void JS_PrintValueInternal(JSRuntime *rt, JSContext *ctx, s->options.max_item_count = UINT32_MAX; s->rt = rt; s->ctx = ctx; - s->fo = fo; + s->write_func = write_func; + s->write_opaque = write_opaque; s->level = 0; js_print_value(s, val); } -void JS_PrintValueRT(JSRuntime *rt, FILE *fo, JSValueConst val, const JSPrintValueOptions *options) +void JS_PrintValueRT(JSRuntime *rt, JSPrintValueWrite *write_func, void *write_opaque, + JSValueConst val, const JSPrintValueOptions *options) +{ + JS_PrintValueInternal(rt, NULL, write_func, write_opaque, val, options); +} + +void JS_PrintValue(JSContext *ctx, JSPrintValueWrite *write_func, void *write_opaque, + JSValueConst val, const JSPrintValueOptions *options) +{ + JS_PrintValueInternal(ctx->rt, ctx, write_func, write_opaque, val, options); +} + +static void js_dump_value_write(void *opaque, const char *buf, size_t len) +{ + FILE *fo = opaque; + fwrite(buf, 1, len, fo); +} + +static __maybe_unused void print_atom(JSContext *ctx, JSAtom atom) { - JS_PrintValueInternal(rt, NULL, fo, val, options); + JSPrintValueState ss, *s = &ss; + memset(s, 0, sizeof(*s)); + s->rt = ctx->rt; + s->ctx = ctx; + s->write_func = js_dump_value_write; + s->write_opaque = stdout; + js_print_atom(s, atom); } -void JS_PrintValue(JSContext *ctx, FILE *fo, JSValueConst val, const JSPrintValueOptions *options) +static __maybe_unused void JS_DumpAtom(JSContext *ctx, const char *str, JSAtom atom) { - JS_PrintValueInternal(ctx->rt, ctx, fo, val, options); + printf("%s=", str); + print_atom(ctx, atom); + printf("\n"); } static __maybe_unused void JS_DumpValue(JSContext *ctx, const char *str, JSValueConst val) { printf("%s=", str); - JS_PrintValue(ctx, stdout, val, NULL); + JS_PrintValue(ctx, js_dump_value_write, stdout, val, NULL); printf("\n"); } static __maybe_unused void JS_DumpValueRT(JSRuntime *rt, const char *str, JSValueConst val) { printf("%s=", str); - JS_PrintValueRT(rt, stdout, val, NULL); + JS_PrintValueRT(rt, js_dump_value_write, stdout, val, NULL); printf("\n"); } @@ -13605,7 +14011,7 @@ static __maybe_unused void JS_DumpObject(JSRuntime *rt, JSObject *p) options.max_depth = 1; options.show_hidden = TRUE; options.raw_dump = TRUE; - JS_PrintValueRT(rt, stdout, JS_MKPTR(JS_TAG_OBJECT, p), &options); + JS_PrintValueRT(rt, js_dump_value_write, stdout, JS_MKPTR(JS_TAG_OBJECT, p), &options); printf("\n"); } @@ -13634,6 +14040,9 @@ static __maybe_unused void JS_DumpGCObject(JSRuntime *rt, JSGCObjectHeader *p) case JS_GC_OBJ_TYPE_JS_CONTEXT: printf("[js_context]"); break; + case JS_GC_OBJ_TYPE_MODULE: + printf("[module]"); + break; default: printf("[unknown %d]", p->gc_obj_type); break; @@ -14884,8 +15293,8 @@ static BOOL js_strict_eq2(JSContext *ctx, JSValue op1, JSValue op2, if (!tag_is_string(tag2)) { res = FALSE; } else if (tag1 == JS_TAG_STRING && tag2 == JS_TAG_STRING) { - res = (js_string_compare(ctx, JS_VALUE_GET_STRING(op1), - JS_VALUE_GET_STRING(op2)) == 0); + res = js_string_eq(ctx, JS_VALUE_GET_STRING(op1), + JS_VALUE_GET_STRING(op2)); } else { res = (js_string_rope_compare(ctx, op1, op2, TRUE) == 0); } @@ -15363,26 +15772,6 @@ static JSValue js_build_mapped_arguments(JSContext *ctx, int argc, return JS_EXCEPTION; } -static JSValue js_build_rest(JSContext *ctx, int first, int argc, JSValueConst *argv) -{ - JSValue val; - int i, ret; - - val = JS_NewArray(ctx); - if (JS_IsException(val)) - return val; - for (i = first; i < argc; i++) { - ret = JS_DefinePropertyValueUint32(ctx, val, i - first, - JS_DupValue(ctx, argv[i]), - JS_PROP_C_W_E); - if (ret < 0) { - JS_FreeValue(ctx, val); - return JS_EXCEPTION; - } - } - return val; -} - static JSValue build_for_in_iterator(JSContext *ctx, JSValue obj) { JSObject *p, *p1; @@ -15482,7 +15871,7 @@ static __exception int js_for_in_prepare_prototype_chain_enum(JSContext *ctx, JS_FreeValue(ctx, obj1); goto fail; } - js_free_prop_enum(ctx, tab_atom, tab_atom_count); + JS_FreePropertyEnum(ctx, tab_atom, tab_atom_count); if (tab_atom_count != 0) { JS_FreeValue(ctx, obj1); goto slow_path; @@ -15566,7 +15955,7 @@ static __exception int js_for_in_next(JSContext *ctx, JSValue *sp) JS_GPN_STRING_MASK | JS_GPN_SET_ENUM)) { return -1; } - js_free_prop_enum(ctx, it->tab_atom, it->atom_count); + JS_FreePropertyEnum(ctx, it->tab_atom, it->atom_count); it->tab_atom = tab_atom; it->atom_count = tab_atom_count; it->idx = 0; @@ -15947,6 +16336,7 @@ static __exception int js_append_enumerate(JSContext *ctx, JSValue *sp) int is_array_iterator; JSValue *arrp; uint32_t i, count32, pos; + JSCFunctionType ft; if (JS_VALUE_GET_TAG(sp[-2]) != JS_TAG_INT) { JS_ThrowInternalError(ctx, "invalid index for append"); @@ -15964,8 +16354,8 @@ static __exception int js_append_enumerate(JSContext *ctx, JSValue *sp) iterator = JS_GetProperty(ctx, sp[-1], JS_ATOM_Symbol_iterator); if (JS_IsException(iterator)) return -1; - is_array_iterator = JS_IsCFunction(ctx, iterator, - (JSCFunction *)js_create_array_iterator, + ft.generic_magic = js_create_array_iterator; + is_array_iterator = JS_IsCFunction(ctx, iterator, ft.generic, JS_ITERATOR_KIND_VALUE); JS_FreeValue(ctx, iterator); @@ -15977,8 +16367,10 @@ static __exception int js_append_enumerate(JSContext *ctx, JSValue *sp) JS_FreeValue(ctx, enumobj); return -1; } + + ft.iterator_next = js_array_iterator_next; if (is_array_iterator - && JS_IsCFunction(ctx, method, (JSCFunction *)js_array_iterator_next, 0) + && JS_IsCFunction(ctx, method, ft.generic, 0) && js_get_fast_array(ctx, sp[-1], &arrp, &count32)) { uint32_t len; if (js_get_length32(ctx, &len, sp[-1])) @@ -16089,10 +16481,10 @@ static __exception int JS_CopyDataProperties(JSContext *ctx, if (ret < 0) goto exception; } - js_free_prop_enum(ctx, tab_atom, tab_atom_count); + JS_FreePropertyEnum(ctx, tab_atom, tab_atom_count); return 0; exception: - js_free_prop_enum(ctx, tab_atom, tab_atom_count); + JS_FreePropertyEnum(ctx, tab_atom, tab_atom_count); return -1; } @@ -16102,6 +16494,25 @@ static JSValueConst JS_GetActiveFunction(JSContext *ctx) return ctx->rt->current_stack_frame->cur_func; } +static JSVarRef *js_create_var_ref(JSContext *ctx, BOOL is_lexical) +{ + JSVarRef *var_ref; + var_ref = js_malloc(ctx, sizeof(JSVarRef)); + if (!var_ref) + return NULL; + var_ref->header.ref_count = 1; + if (is_lexical) + var_ref->value = JS_UNINITIALIZED; + else + var_ref->value = JS_UNDEFINED; + var_ref->pvalue = &var_ref->value; + var_ref->is_detached = TRUE; + var_ref->is_lexical = FALSE; + var_ref->is_const = FALSE; + add_gc_object(ctx->rt, &var_ref->header, JS_GC_OBJ_TYPE_VAR_REF); + return var_ref; +} + static JSVarRef *get_var_ref(JSContext *ctx, JSStackFrame *sf, int var_idx, BOOL is_arg) { @@ -16128,6 +16539,8 @@ static JSVarRef *get_var_ref(JSContext *ctx, JSStackFrame *sf, var_ref->header.ref_count = 1; add_gc_object(ctx->rt, &var_ref->header, JS_GC_OBJ_TYPE_VAR_REF); var_ref->is_detached = FALSE; + var_ref->is_lexical = FALSE; + var_ref->is_const = FALSE; list_add_tail(&var_ref->var_ref_link, &sf->var_ref_list); if (sf->js_mode & JS_MODE_ASYNC) { /* The stack frame is detached and may be destroyed at any @@ -16147,10 +16560,217 @@ static JSVarRef *get_var_ref(JSContext *ctx, JSStackFrame *sf, return var_ref; } +static void js_global_object_finalizer(JSRuntime *rt, JSValue obj) +{ + JSObject *p = JS_VALUE_GET_OBJ(obj); + JS_FreeValueRT(rt, p->u.global_object.uninitialized_vars); +} + +static void js_global_object_mark(JSRuntime *rt, JSValueConst val, + JS_MarkFunc *mark_func) +{ + JSObject *p = JS_VALUE_GET_OBJ(val); + JS_MarkValue(rt, p->u.global_object.uninitialized_vars, mark_func); +} + +static JSVarRef *js_global_object_get_uninitialized_var(JSContext *ctx, JSObject *p1, + JSAtom atom) +{ + JSObject *p = JS_VALUE_GET_OBJ(p1->u.global_object.uninitialized_vars); + JSShapeProperty *prs; + JSProperty *pr; + JSVarRef *var_ref; + + prs = find_own_property(&pr, p, atom); + if (prs) { + assert((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF); + var_ref = pr->u.var_ref; + var_ref->header.ref_count++; + return var_ref; + } + + var_ref = js_create_var_ref(ctx, TRUE); + if (!var_ref) + return NULL; + pr = add_property(ctx, p, atom, JS_PROP_C_W_E | JS_PROP_VARREF); + if (unlikely(!pr)) { + free_var_ref(ctx->rt, var_ref); + return NULL; + } + pr->u.var_ref = var_ref; + var_ref->header.ref_count++; + return var_ref; +} + +/* return a new variable reference. Get it from the uninitialized + variables if it is present. Return NULL in case of memory error. */ +static JSVarRef *js_global_object_find_uninitialized_var(JSContext *ctx, JSObject *p, + JSAtom atom, BOOL is_lexical) +{ + JSObject *p1; + JSShapeProperty *prs; + JSProperty *pr; + JSVarRef *var_ref; + + p1 = JS_VALUE_GET_OBJ(p->u.global_object.uninitialized_vars); + prs = find_own_property(&pr, p1, atom); + if (prs) { + assert((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF); + var_ref = pr->u.var_ref; + var_ref->header.ref_count++; + delete_property(ctx, p1, atom); + if (!is_lexical) + var_ref->value = JS_UNDEFINED; + } else { + var_ref = js_create_var_ref(ctx, is_lexical); + if (!var_ref) + return NULL; + } + return var_ref; +} + +static JSVarRef *js_closure_define_global_var(JSContext *ctx, JSClosureVar *cv, + BOOL is_direct_or_indirect_eval) +{ + JSObject *p, *p1; + JSShapeProperty *prs; + int flags; + JSProperty *pr; + JSVarRef *var_ref; + + if (cv->is_lexical) { + p = JS_VALUE_GET_OBJ(ctx->global_var_obj); + flags = JS_PROP_ENUMERABLE | JS_PROP_CONFIGURABLE; + if (!cv->is_const) + flags |= JS_PROP_WRITABLE; + + prs = find_own_property(&pr, p, cv->var_name); + if (prs) { + assert((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF); + var_ref = pr->u.var_ref; + var_ref->header.ref_count++; + return var_ref; + } + + /* if there is a corresponding global variable, reuse its + reference and create a new one for the global variable */ + p1 = JS_VALUE_GET_OBJ(ctx->global_obj); + prs = find_own_property(&pr, p1, cv->var_name); + if (prs && (prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF) { + JSVarRef *var_ref1; + var_ref1 = js_create_var_ref(ctx, FALSE); + if (!var_ref1) + return NULL; + var_ref = pr->u.var_ref; + var_ref1->value = var_ref->value; + var_ref->value = JS_UNINITIALIZED; + pr->u.var_ref = var_ref1; + goto add_var_ref; + } + } else { + p = JS_VALUE_GET_OBJ(ctx->global_obj); + flags = JS_PROP_ENUMERABLE | JS_PROP_WRITABLE; + if (is_direct_or_indirect_eval) + flags |= JS_PROP_CONFIGURABLE; + + retry: + prs = find_own_property(&pr, p, cv->var_name); + if (prs) { + if (unlikely((prs->flags & JS_PROP_TMASK) == JS_PROP_AUTOINIT)) { + if (JS_AutoInitProperty(ctx, p, cv->var_name, pr, prs)) + return NULL; + goto retry; + } else if ((prs->flags & JS_PROP_TMASK) != JS_PROP_VARREF) { + var_ref = js_global_object_get_uninitialized_var(ctx, p, cv->var_name); + if (!var_ref) + return NULL; + } else { + var_ref = pr->u.var_ref; + var_ref->header.ref_count++; + } + if (cv->var_kind == JS_VAR_GLOBAL_FUNCTION_DECL && + (prs->flags & JS_PROP_CONFIGURABLE)) { + /* update the property flags if possible when + declaring a global function */ + if ((prs->flags & JS_PROP_TMASK) == JS_PROP_GETSET) { + free_property(ctx->rt, pr, prs->flags); + prs->flags = flags | JS_PROP_VARREF; + pr->u.var_ref = var_ref; + var_ref->header.ref_count++; + } else { + assert((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF); + prs->flags = (prs->flags & ~JS_PROP_C_W_E) | flags; + } + var_ref->is_const = FALSE; + } + return var_ref; + } + + if (!p->extensible) { + return js_global_object_get_uninitialized_var(ctx, p, cv->var_name); + } + } + + /* if there is a corresponding uninitialized variable, use it */ + p1 = JS_VALUE_GET_OBJ(ctx->global_obj); + var_ref = js_global_object_find_uninitialized_var(ctx, p1, cv->var_name, cv->is_lexical); + if (!var_ref) + return NULL; + add_var_ref: + if (cv->is_lexical) { + var_ref->is_lexical = TRUE; + var_ref->is_const = cv->is_const; + } + + pr = add_property(ctx, p, cv->var_name, flags | JS_PROP_VARREF); + if (unlikely(!pr)) { + free_var_ref(ctx->rt, var_ref); + return NULL; + } + pr->u.var_ref = var_ref; + var_ref->header.ref_count++; + return var_ref; +} + +static JSVarRef *js_closure_global_var(JSContext *ctx, JSClosureVar *cv) +{ + JSObject *p; + JSShapeProperty *prs; + JSProperty *pr; + JSVarRef *var_ref; + + p = JS_VALUE_GET_OBJ(ctx->global_var_obj); + prs = find_own_property(&pr, p, cv->var_name); + if (prs) { + assert((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF); + var_ref = pr->u.var_ref; + var_ref->header.ref_count++; + return var_ref; + } + p = JS_VALUE_GET_OBJ(ctx->global_obj); + redo: + prs = find_own_property(&pr, p, cv->var_name); + if (prs) { + if (unlikely((prs->flags & JS_PROP_TMASK) == JS_PROP_AUTOINIT)) { + /* Instantiate property and retry */ + if (JS_AutoInitProperty(ctx, p, cv->var_name, pr, prs)) + return NULL; + goto redo; + } + if ((prs->flags & JS_PROP_TMASK) == JS_PROP_VARREF) { + var_ref = pr->u.var_ref; + var_ref->header.ref_count++; + return var_ref; + } + } + return js_global_object_get_uninitialized_var(ctx, p, cv->var_name); +} + static JSValue js_closure2(JSContext *ctx, JSValue func_obj, JSFunctionBytecode *b, JSVarRef **cur_var_refs, - JSStackFrame *sf) + JSStackFrame *sf, + BOOL is_eval, JSModuleDef *m) { JSObject *p; JSVarRef **var_refs; @@ -16165,18 +16785,56 @@ static JSValue js_closure2(JSContext *ctx, JSValue func_obj, if (!var_refs) goto fail; p->u.func.var_refs = var_refs; + if (is_eval) { + /* first pass to check the global variable definitions */ + for(i = 0; i < b->closure_var_count; i++) { + JSClosureVar *cv = &b->closure_var[i]; + if (cv->closure_type == JS_CLOSURE_GLOBAL_DECL) { + int flags; + flags = 0; + if (cv->is_lexical) + flags |= DEFINE_GLOBAL_LEX_VAR; + if (cv->var_kind == JS_VAR_GLOBAL_FUNCTION_DECL) + flags |= DEFINE_GLOBAL_FUNC_VAR; + if (JS_CheckDefineGlobalVar(ctx, cv->var_name, flags)) + goto fail; + } + } + } for(i = 0; i < b->closure_var_count; i++) { JSClosureVar *cv = &b->closure_var[i]; JSVarRef *var_ref; - if (cv->is_local) { + switch(cv->closure_type) { + case JS_CLOSURE_MODULE_IMPORT: + /* imported from other modules */ + continue; + case JS_CLOSURE_MODULE_DECL: + var_ref = js_create_var_ref(ctx, cv->is_lexical); + break; + case JS_CLOSURE_GLOBAL_DECL: + var_ref = js_closure_define_global_var(ctx, cv, b->is_direct_or_indirect_eval); + break; + case JS_CLOSURE_GLOBAL: + var_ref = js_closure_global_var(ctx, cv); + break; + case JS_CLOSURE_LOCAL: /* reuse the existing variable reference if it already exists */ - var_ref = get_var_ref(ctx, sf, cv->var_idx, cv->is_arg); - if (!var_ref) - goto fail; - } else { + var_ref = get_var_ref(ctx, sf, cv->var_idx, FALSE); + break; + case JS_CLOSURE_ARG: + /* reuse the existing variable reference if it already exists */ + var_ref = get_var_ref(ctx, sf, cv->var_idx, TRUE); + break; + case JS_CLOSURE_REF: + case JS_CLOSURE_GLOBAL_REF: var_ref = cur_var_refs[cv->var_idx]; var_ref->header.ref_count++; + break; + default: + abort(); } + if (!var_ref) + goto fail; var_refs[i] = var_ref; } } @@ -16217,7 +16875,7 @@ static const uint16_t func_kind_to_class_id[] = { static JSValue js_closure(JSContext *ctx, JSValue bfunc, JSVarRef **cur_var_refs, - JSStackFrame *sf) + JSStackFrame *sf, BOOL is_eval) { JSFunctionBytecode *b; JSValue func_obj; @@ -16229,7 +16887,7 @@ static JSValue js_closure(JSContext *ctx, JSValue bfunc, JS_FreeValue(ctx, bfunc); return JS_EXCEPTION; } - func_obj = js_closure2(ctx, func_obj, b, cur_var_refs, sf); + func_obj = js_closure2(ctx, func_obj, b, cur_var_refs, sf, is_eval, NULL); if (JS_IsException(func_obj)) { /* bfunc has been freed */ goto fail; @@ -16316,7 +16974,7 @@ static int js_op_define_class(JSContext *ctx, JSValue *sp, JS_CLASS_BYTECODE_FUNCTION); if (JS_IsException(ctor)) goto fail; - ctor = js_closure2(ctx, ctor, b, cur_var_refs, sf); + ctor = js_closure2(ctx, ctor, b, cur_var_refs, sf, FALSE, NULL); bfunc = JS_UNDEFINED; if (JS_IsException(ctor)) goto fail; @@ -16749,25 +17407,13 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, *sp++ = JS_DupValue(ctx, b->cpool[*pc++]); BREAK; CASE(OP_fclosure8): - *sp++ = js_closure(ctx, JS_DupValue(ctx, b->cpool[*pc++]), var_refs, sf); + *sp++ = js_closure(ctx, JS_DupValue(ctx, b->cpool[*pc++]), var_refs, sf, FALSE); if (unlikely(JS_IsException(sp[-1]))) goto exception; BREAK; CASE(OP_push_empty_string): *sp++ = JS_AtomToString(ctx, JS_ATOM_empty_string); BREAK; - CASE(OP_get_length): - { - JSValue val; - - sf->cur_pc = pc; - val = JS_GetProperty(ctx, sp[-1], JS_ATOM_length); - if (unlikely(JS_IsException(val))) - goto exception; - JS_FreeValue(ctx, sp[-1]); - sp[-1] = val; - } - BREAK; #endif CASE(OP_push_atom_value): *sp++ = JS_AtomToValue(ctx, get_u32(pc)); @@ -16862,7 +17508,8 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, { int first = get_u16(pc); pc += 2; - *sp++ = js_build_rest(ctx, first, argc, (JSValueConst *)argv); + first = min_int(first, argc); + *sp++ = js_create_array(ctx, argc - first, (JSValueConst *)(argv + first)); if (unlikely(JS_IsException(sp[-1]))) goto exception; } @@ -17014,7 +17661,7 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, { JSValue bfunc = JS_DupValue(ctx, b->cpool[get_u32(pc)]); pc += 4; - *sp++ = js_closure(ctx, bfunc, var_refs, sf); + *sp++ = js_closure(ctx, bfunc, var_refs, sf, FALSE); if (unlikely(JS_IsException(sp[-1]))) goto exception; } @@ -17085,27 +17732,13 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, } BREAK; CASE(OP_array_from): - { - int i, ret; - - call_argc = get_u16(pc); - pc += 2; - ret_val = JS_NewArray(ctx); - if (unlikely(JS_IsException(ret_val))) - goto exception; - call_argv = sp - call_argc; - for(i = 0; i < call_argc; i++) { - ret = JS_DefinePropertyValue(ctx, ret_val, __JS_AtomFromUInt32(i), call_argv[i], - JS_PROP_C_W_E | JS_PROP_THROW); - call_argv[i] = JS_UNDEFINED; - if (ret < 0) { - JS_FreeValue(ctx, ret_val); - goto exception; - } - } - sp -= call_argc; - *sp++ = ret_val; - } + call_argc = get_u16(pc); + pc += 2; + ret_val = js_create_array_free(ctx, call_argc, sp - call_argc); + sp -= call_argc; + if (unlikely(JS_IsException(ret_val))) + goto exception; + *sp++ = ret_val; BREAK; CASE(OP_apply): @@ -17287,8 +17920,13 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, CASE(OP_regexp): { - sp[-2] = js_regexp_constructor_internal(ctx, JS_UNDEFINED, - sp[-2], sp[-1]); + JSValue obj; + obj = JS_NewObjectClass(ctx, JS_CLASS_REGEXP); + if (JS_IsException(obj)) + goto exception; + sp[-2] = js_regexp_set_internal(ctx, obj, sp[-2], sp[-1]); + if (JS_IsException(sp[-2])) + goto exception; sp--; } BREAK; @@ -17309,120 +17947,86 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, { JSValue val; sf->cur_pc = pc; - val = js_dynamic_import(ctx, sp[-1]); + val = js_dynamic_import(ctx, sp[-2], sp[-1]); if (JS_IsException(val)) goto exception; + JS_FreeValue(ctx, sp[-2]); JS_FreeValue(ctx, sp[-1]); + sp--; sp[-1] = val; } BREAK; - CASE(OP_check_var): - { - int ret; - JSAtom atom; - atom = get_u32(pc); - pc += 4; - sf->cur_pc = pc; - - ret = JS_CheckGlobalVar(ctx, atom); - if (ret < 0) - goto exception; - *sp++ = JS_NewBool(ctx, ret); - } - BREAK; - CASE(OP_get_var_undef): CASE(OP_get_var): { + int idx; JSValue val; - JSAtom atom; - atom = get_u32(pc); - pc += 4; - sf->cur_pc = pc; - - val = JS_GetGlobalVar(ctx, atom, opcode - OP_get_var_undef); - if (unlikely(JS_IsException(val))) - goto exception; - *sp++ = val; + idx = get_u16(pc); + pc += 2; + val = *var_refs[idx]->pvalue; + if (unlikely(JS_IsUninitialized(val))) { + JSClosureVar *cv = &b->closure_var[idx]; + if (cv->is_lexical) { + JS_ThrowReferenceErrorUninitialized(ctx, cv->var_name); + goto exception; + } else { + sf->cur_pc = pc; + sp[0] = JS_GetPropertyInternal(ctx, ctx->global_obj, + cv->var_name, + ctx->global_obj, + opcode - OP_get_var_undef); + if (JS_IsException(sp[0])) + goto exception; + } + } else { + sp[0] = JS_DupValue(ctx, val); + } + sp++; } BREAK; CASE(OP_put_var): CASE(OP_put_var_init): { - int ret; - JSAtom atom; - atom = get_u32(pc); - pc += 4; - sf->cur_pc = pc; - - ret = JS_SetGlobalVar(ctx, atom, sp[-1], opcode - OP_put_var); - sp--; - if (unlikely(ret < 0)) - goto exception; - } - BREAK; - - CASE(OP_put_var_strict): - { - int ret; - JSAtom atom; - atom = get_u32(pc); - pc += 4; - sf->cur_pc = pc; - - /* sp[-2] is JS_TRUE or JS_FALSE */ - if (unlikely(!JS_VALUE_GET_INT(sp[-2]))) { - JS_ThrowReferenceErrorNotDefined(ctx, atom); - goto exception; + int idx, ret; + JSVarRef *var_ref; + idx = get_u16(pc); + pc += 2; + var_ref = var_refs[idx]; + if (unlikely(JS_IsUninitialized(*var_ref->pvalue) || + var_ref->is_const)) { + JSClosureVar *cv = &b->closure_var[idx]; + if (var_ref->is_lexical) { + if (opcode == OP_put_var_init) + goto put_var_ok; + if (JS_IsUninitialized(*var_ref->pvalue)) + JS_ThrowReferenceErrorUninitialized(ctx, cv->var_name); + else + JS_ThrowTypeErrorReadOnly(ctx, JS_PROP_THROW, cv->var_name); + goto exception; + } else { + sf->cur_pc = pc; + ret = JS_HasProperty(ctx, ctx->global_obj, cv->var_name); + if (ret < 0) + goto exception; + if (ret == 0 && is_strict_mode(ctx)) { + JS_ThrowReferenceErrorNotDefined(ctx, cv->var_name); + goto exception; + } + ret = JS_SetPropertyInternal(ctx, ctx->global_obj, cv->var_name, sp[-1], + ctx->global_obj, JS_PROP_THROW_STRICT); + sp--; + if (ret < 0) + goto exception; + } + } else { + put_var_ok: + set_value(ctx, var_ref->pvalue, sp[-1]); + sp--; } - ret = JS_SetGlobalVar(ctx, atom, sp[-1], 2); - sp -= 2; - if (unlikely(ret < 0)) - goto exception; - } - BREAK; - - CASE(OP_check_define_var): - { - JSAtom atom; - int flags; - atom = get_u32(pc); - flags = pc[4]; - pc += 5; - sf->cur_pc = pc; - if (JS_CheckDefineGlobalVar(ctx, atom, flags)) - goto exception; } BREAK; - CASE(OP_define_var): - { - JSAtom atom; - int flags; - atom = get_u32(pc); - flags = pc[4]; - pc += 5; - sf->cur_pc = pc; - if (JS_DefineGlobalVar(ctx, atom, flags)) - goto exception; - } - BREAK; - CASE(OP_define_func): - { - JSAtom atom; - int flags; - atom = get_u32(pc); - flags = pc[4]; - pc += 5; - sf->cur_pc = pc; - if (JS_DefineGlobalFunction(ctx, atom, sp[-1], flags)) - goto exception; - JS_FreeValue(ctx, sp[-1]); - sp--; - } - BREAK; - CASE(OP_get_loc): { int idx; @@ -17987,51 +18591,114 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, } BREAK; - CASE(OP_get_field): - { - JSValue val; - JSAtom atom; - atom = get_u32(pc); - pc += 4; - - sf->cur_pc = pc; - val = JS_GetProperty(ctx, sp[-1], atom); - if (unlikely(JS_IsException(val))) - goto exception; - JS_FreeValue(ctx, sp[-1]); - sp[-1] = val; +#define GET_FIELD_INLINE(name, keep, is_length) \ + { \ + JSValue val, obj; \ + JSAtom atom; \ + JSObject *p; \ + JSProperty *pr; \ + JSShapeProperty *prs; \ + \ + if (is_length) { \ + atom = JS_ATOM_length; \ + } else { \ + atom = get_u32(pc); \ + pc += 4; \ + } \ + \ + obj = sp[-1]; \ + if (likely(JS_VALUE_GET_TAG(obj) == JS_TAG_OBJECT)) { \ + p = JS_VALUE_GET_OBJ(obj); \ + for(;;) { \ + prs = find_own_property(&pr, p, atom); \ + if (prs) { \ + /* found */ \ + if (unlikely(prs->flags & JS_PROP_TMASK)) \ + goto name ## _slow_path; \ + val = JS_DupValue(ctx, pr->u.value); \ + break; \ + } \ + if (unlikely(p->is_exotic)) { \ + /* XXX: should avoid the slow path for arrays \ + and typed arrays by ensuring that 'prop' is \ + not numeric */ \ + obj = JS_MKPTR(JS_TAG_OBJECT, p); \ + goto name ## _slow_path; \ + } \ + p = p->shape->proto; \ + if (!p) { \ + val = JS_UNDEFINED; \ + break; \ + } \ + } \ + } else { \ + name ## _slow_path: \ + sf->cur_pc = pc; \ + val = JS_GetPropertyInternal(ctx, obj, atom, sp[-1], 0); \ + if (unlikely(JS_IsException(val))) \ + goto exception; \ + } \ + if (keep) { \ + *sp++ = val; \ + } else { \ + JS_FreeValue(ctx, sp[-1]); \ + sp[-1] = val; \ + } \ } + + + CASE(OP_get_field): + GET_FIELD_INLINE(get_field, 0, 0); BREAK; CASE(OP_get_field2): - { - JSValue val; - JSAtom atom; - atom = get_u32(pc); - pc += 4; - - sf->cur_pc = pc; - val = JS_GetProperty(ctx, sp[-1], atom); - if (unlikely(JS_IsException(val))) - goto exception; - *sp++ = val; - } + GET_FIELD_INLINE(get_field2, 1, 0); BREAK; +#if SHORT_OPCODES + CASE(OP_get_length): + GET_FIELD_INLINE(get_length, 0, 1); + BREAK; +#endif + CASE(OP_put_field): { int ret; + JSValue obj; JSAtom atom; + JSObject *p; + JSProperty *pr; + JSShapeProperty *prs; + atom = get_u32(pc); pc += 4; - sf->cur_pc = pc; - ret = JS_SetPropertyInternal(ctx, sp[-2], atom, sp[-1], sp[-2], - JS_PROP_THROW_STRICT); - JS_FreeValue(ctx, sp[-2]); - sp -= 2; - if (unlikely(ret < 0)) - goto exception; + obj = sp[-2]; + if (likely(JS_VALUE_GET_TAG(obj) == JS_TAG_OBJECT)) { + p = JS_VALUE_GET_OBJ(obj); + prs = find_own_property(&pr, p, atom); + if (!prs) + goto put_field_slow_path; + if (likely((prs->flags & (JS_PROP_TMASK | JS_PROP_WRITABLE | + JS_PROP_LENGTH)) == JS_PROP_WRITABLE)) { + /* fast path */ + set_value(ctx, &pr->u.value, sp[-1]); + } else { + goto put_field_slow_path; + } + JS_FreeValue(ctx, obj); + sp -= 2; + } else { + put_field_slow_path: + sf->cur_pc = pc; + ret = JS_SetPropertyInternal(ctx, obj, atom, sp[-1], obj, + JS_PROP_THROW_STRICT); + JS_FreeValue(ctx, obj); + sp -= 2; + if (unlikely(ret < 0)) + goto exception; + } + } BREAK; @@ -18213,61 +18880,95 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, } BREAK; - CASE(OP_get_array_el): - { - JSValue val; - - sf->cur_pc = pc; - val = JS_GetPropertyValue(ctx, sp[-2], sp[-1]); - JS_FreeValue(ctx, sp[-2]); - sp[-2] = val; - sp--; - if (unlikely(JS_IsException(val))) - goto exception; +#define GET_ARRAY_EL_INLINE(name, keep) \ + { \ + JSValue val, obj, prop; \ + JSObject *p; \ + uint32_t idx; \ + \ + obj = sp[-2]; \ + prop = sp[-1]; \ + if (likely(JS_VALUE_GET_TAG(obj) == JS_TAG_OBJECT && \ + JS_VALUE_GET_TAG(prop) == JS_TAG_INT)) { \ + p = JS_VALUE_GET_OBJ(obj); \ + idx = JS_VALUE_GET_INT(prop); \ + if (unlikely(p->class_id != JS_CLASS_ARRAY)) \ + goto name ## _slow_path; \ + if (unlikely(idx >= p->u.array.count)) \ + goto name ## _slow_path; \ + val = JS_DupValue(ctx, p->u.array.u.values[idx]); \ + } else { \ + name ## _slow_path: \ + sf->cur_pc = pc; \ + val = JS_GetPropertyValue(ctx, obj, prop); \ + if (unlikely(JS_IsException(val))) { \ + if (keep) \ + sp[-1] = JS_UNDEFINED; \ + else \ + sp--; \ + goto exception; \ + } \ + } \ + if (keep) { \ + sp[-1] = val; \ + } else { \ + JS_FreeValue(ctx, obj); \ + sp[-2] = val; \ + sp--; \ + } \ } + + CASE(OP_get_array_el): + GET_ARRAY_EL_INLINE(get_array_el, 0); BREAK; CASE(OP_get_array_el2): - { - JSValue val; - - sf->cur_pc = pc; - val = JS_GetPropertyValue(ctx, sp[-2], sp[-1]); - sp[-1] = val; - if (unlikely(JS_IsException(val))) - goto exception; - } + GET_ARRAY_EL_INLINE(get_array_el2, 1); BREAK; CASE(OP_get_array_el3): { JSValue val; + JSObject *p; + uint32_t idx; - switch (JS_VALUE_GET_TAG(sp[-2])) { - case JS_TAG_INT: - case JS_TAG_STRING: - case JS_TAG_SYMBOL: - /* undefined and null are tested in JS_GetPropertyValue() */ - break; - default: - /* must be tested nefore JS_ToPropertyKey */ - if (unlikely(JS_IsUndefined(sp[-2]) || JS_IsNull(sp[-2]))) { - JS_ThrowTypeError(ctx, "value has no property"); - goto exception; + if (likely(JS_VALUE_GET_TAG(sp[-2]) == JS_TAG_OBJECT && + JS_VALUE_GET_TAG(sp[-1]) == JS_TAG_INT)) { + p = JS_VALUE_GET_OBJ(sp[-2]); + idx = JS_VALUE_GET_INT(sp[-1]); + if (unlikely(p->class_id != JS_CLASS_ARRAY)) + goto get_array_el3_slow_path; + if (unlikely(idx >= p->u.array.count)) + goto get_array_el3_slow_path; + val = JS_DupValue(ctx, p->u.array.u.values[idx]); + } else { + get_array_el3_slow_path: + switch (JS_VALUE_GET_TAG(sp[-1])) { + case JS_TAG_INT: + case JS_TAG_STRING: + case JS_TAG_SYMBOL: + /* undefined and null are tested in JS_GetPropertyValue() */ + break; + default: + /* must be tested before JS_ToPropertyKey */ + if (unlikely(JS_IsUndefined(sp[-2]) || JS_IsNull(sp[-2]))) { + JS_ThrowTypeError(ctx, "value has no property"); + goto exception; + } + sf->cur_pc = pc; + ret_val = JS_ToPropertyKey(ctx, sp[-1]); + if (JS_IsException(ret_val)) + goto exception; + JS_FreeValue(ctx, sp[-1]); + sp[-1] = ret_val; + break; } sf->cur_pc = pc; - ret_val = JS_ToPropertyKey(ctx, sp[-1]); - if (JS_IsException(ret_val)) + val = JS_GetPropertyValue(ctx, sp[-2], JS_DupValue(ctx, sp[-1])); + if (unlikely(JS_IsException(val))) goto exception; - JS_FreeValue(ctx, sp[-1]); - sp[-1] = ret_val; - break; } - sf->cur_pc = pc; - val = JS_GetPropertyValue(ctx, sp[-2], JS_DupValue(ctx, sp[-1])); *sp++ = val; - if (unlikely(JS_IsException(val))) - goto exception; } BREAK; @@ -18332,13 +19033,52 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, CASE(OP_put_array_el): { int ret; + JSObject *p; + uint32_t idx; - sf->cur_pc = pc; - ret = JS_SetPropertyValue(ctx, sp[-3], sp[-2], sp[-1], JS_PROP_THROW_STRICT); - JS_FreeValue(ctx, sp[-3]); - sp -= 3; - if (unlikely(ret < 0)) - goto exception; + if (likely(JS_VALUE_GET_TAG(sp[-3]) == JS_TAG_OBJECT && + JS_VALUE_GET_TAG(sp[-2]) == JS_TAG_INT)) { + p = JS_VALUE_GET_OBJ(sp[-3]); + idx = JS_VALUE_GET_INT(sp[-2]); + if (unlikely(p->class_id != JS_CLASS_ARRAY)) + goto put_array_el_slow_path; + if (unlikely(idx >= (uint32_t)p->u.array.count)) { + uint32_t new_len, array_len; + if (unlikely(idx != (uint32_t)p->u.array.count || + !p->fast_array || + !p->extensible || + p->shape->proto != JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]) || + !ctx->std_array_prototype)) { + goto put_array_el_slow_path; + } + if (likely(JS_VALUE_GET_TAG(p->prop[0].u.value) != JS_TAG_INT)) + goto put_array_el_slow_path; + /* cannot overflow otherwise the length would not be an integer */ + new_len = idx + 1; + if (unlikely(new_len > p->u.array.u1.size)) + goto put_array_el_slow_path; + array_len = JS_VALUE_GET_INT(p->prop[0].u.value); + if (new_len > array_len) { + if (unlikely(!(get_shape_prop(p->shape)->flags & JS_PROP_WRITABLE))) + goto put_array_el_slow_path; + p->prop[0].u.value = JS_NewInt32(ctx, new_len); + } + p->u.array.count = new_len; + p->u.array.u.values[idx] = sp[-1]; + } else { + set_value(ctx, &p->u.array.u.values[idx], sp[-1]); + } + JS_FreeValue(ctx, sp[-3]); + sp -= 3; + } else { + put_array_el_slow_path: + sf->cur_pc = pc; + ret = JS_SetPropertyValue(ctx, sp[-3], sp[-2], sp[-1], JS_PROP_THROW_STRICT); + JS_FreeValue(ctx, sp[-3]); + sp -= 3; + if (unlikely(ret < 0)) + goto exception; + } } BREAK; @@ -18493,12 +19233,10 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, *pv = __JS_NewFloat64(ctx, JS_VALUE_GET_FLOAT64(*pv) + JS_VALUE_GET_FLOAT64(op2)); sp--; - } else if (JS_VALUE_GET_TAG(*pv) == JS_TAG_STRING) { + } else if (JS_VALUE_GET_TAG(*pv) == JS_TAG_STRING && + JS_VALUE_GET_TAG(op2) == JS_TAG_STRING) { sp--; sf->cur_pc = pc; - op2 = JS_ToPrimitiveFree(ctx, op2, HINT_NONE); - if (JS_IsException(op2)) - goto exception; if (JS_ConcatStringInPlace(ctx, JS_VALUE_GET_STRING(*pv), op2)) { JS_FreeValue(ctx, op2); } else { @@ -18703,11 +19441,42 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, } BREAK; CASE(OP_post_inc): + { + JSValue op1; + int val; + op1 = sp[-1]; + if (JS_VALUE_GET_TAG(op1) == JS_TAG_INT) { + val = JS_VALUE_GET_INT(op1); + if (unlikely(val == INT32_MAX)) + goto post_inc_slow; + sp[0] = JS_NewInt32(ctx, val + 1); + } else { + post_inc_slow: + sf->cur_pc = pc; + if (js_post_inc_slow(ctx, sp, opcode)) + goto exception; + } + sp++; + } + BREAK; CASE(OP_post_dec): - sf->cur_pc = pc; - if (js_post_inc_slow(ctx, sp, opcode)) - goto exception; - sp++; + { + JSValue op1; + int val; + op1 = sp[-1]; + if (JS_VALUE_GET_TAG(op1) == JS_TAG_INT) { + val = JS_VALUE_GET_INT(op1); + if (unlikely(val == INT32_MIN)) + goto post_dec_slow; + sp[0] = JS_NewInt32(ctx, val - 1); + } else { + post_dec_slow: + sf->cur_pc = pc; + if (js_post_inc_slow(ctx, sp, opcode)) + goto exception; + } + sp++; + } BREAK; CASE(OP_inc_loc): { @@ -19339,7 +20108,7 @@ static JSValue JS_CallConstructorInternal(JSContext *ctx, goto not_a_function; p = JS_VALUE_GET_OBJ(func_obj); if (unlikely(!p->is_constructor)) - return JS_ThrowTypeError(ctx, "not a constructor"); + return JS_ThrowTypeErrorNotAConstructor(ctx, func_obj); if (unlikely(p->class_id != JS_CLASS_BYTECODE_FUNCTION)) { JSClassCall *call_func; call_func = ctx->rt->class_array[p->class_id].call; @@ -20888,6 +21657,7 @@ static __exception int js_parse_template_part(JSParseState *s, const uint8_t *p) { uint32_t c; StringBuffer b_s, *b = &b_s; + JSValue str; /* p points to the first byte of the template part */ if (string_buffer_init(s->ctx, b, 32)) @@ -20930,9 +21700,12 @@ static __exception int js_parse_template_part(JSParseState *s, const uint8_t *p) if (string_buffer_putc(b, c)) goto fail; } + str = string_buffer_end(b); + if (JS_IsException(str)) + return -1; s->token.val = TOK_TEMPLATE; s->token.u.str.sep = c; - s->token.u.str.str = string_buffer_end(b); + s->token.u.str.str = str; s->buf_ptr = p; return 0; @@ -20951,7 +21724,8 @@ static __exception int js_parse_string(JSParseState *s, int sep, uint32_t c; StringBuffer b_s, *b = &b_s; const uint8_t *p_escape; - + JSValue str; + /* string */ if (string_buffer_init(s->ctx, b, 32)) goto fail; @@ -20960,11 +21734,6 @@ static __exception int js_parse_string(JSParseState *s, int sep, goto invalid_char; c = *p; if (c < 0x20) { - if (!s->cur_func) { - if (do_throw) - js_parse_error_pos(s, p, "invalid character in a JSON string"); - goto fail; - } if (sep == '`') { if (c == '\r') { if (p[1] == '\n') @@ -21010,8 +21779,6 @@ static __exception int js_parse_string(JSParseState *s, int sep, continue; default: if (c >= '0' && c <= '9') { - if (!s->cur_func) - goto invalid_escape; /* JSON case */ if (!(s->cur_func->js_mode & JS_MODE_STRICT) && sep != '`') goto parse_escape; if (c == '0' && !(p[1] >= '0' && p[1] <= '9')) { @@ -21065,9 +21832,12 @@ static __exception int js_parse_string(JSParseState *s, int sep, if (string_buffer_putc(b, c)) goto fail; } + str = string_buffer_end(b); + if (JS_IsException(str)) + return -1; token->val = TOK_STRING; token->u.str.sep = c; - token->u.str.str = string_buffer_end(b); + token->u.str.str = str; *pp = p; return 0; @@ -21095,6 +21865,7 @@ static __exception int js_parse_regexp(JSParseState *s) StringBuffer b_s, *b = &b_s; StringBuffer b2_s, *b2 = &b2_s; uint32_t c; + JSValue body_str, flags_str; p = s->buf_ptr; p++; @@ -21176,9 +21947,17 @@ static __exception int js_parse_regexp(JSParseState *s) p = p_next; } + body_str = string_buffer_end(b); + flags_str = string_buffer_end(b2); + if (JS_IsException(body_str) || + JS_IsException(flags_str)) { + JS_FreeValue(s->ctx, body_str); + JS_FreeValue(s->ctx, flags_str); + return -1; + } s->token.val = TOK_REGEXP; - s->token.u.regexp.body = string_buffer_end(b); - s->token.u.regexp.flags = string_buffer_end(b2); + s->token.u.regexp.body = body_str; + s->token.u.regexp.flags = flags_str; s->buf_ptr = p; return 0; fail: @@ -21748,6 +22527,7 @@ static __exception int next_token(JSParseState *s) } /* 'c' is the first character. Return JS_ATOM_NULL in case of error */ +/* XXX: accept unicode identifiers as JSON5 ? */ static JSAtom json_parse_ident(JSParseState *s, const uint8_t **pp, int c) { const uint8_t *p; @@ -21780,6 +22560,178 @@ static JSAtom json_parse_ident(JSParseState *s, const uint8_t **pp, int c) return atom; } +static int json_parse_string(JSParseState *s, const uint8_t **pp, int sep) +{ + const uint8_t *p, *p_next; + int i; + uint32_t c; + StringBuffer b_s, *b = &b_s; + + if (string_buffer_init(s->ctx, b, 32)) + goto fail; + + p = *pp; + for(;;) { + if (p >= s->buf_end) { + goto end_of_input; + } + c = *p++; + if (c == sep) + break; + if (c < 0x20) { + js_parse_error_pos(s, p - 1, "Bad control character in string literal"); + goto fail; + } + if (c == '\\') { + c = *p++; + switch(c) { + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case '\\': break; + case '/': break; + case 'u': + c = 0; + for(i = 0; i < 4; i++) { + int h = from_hex(*p++); + if (h < 0) { + js_parse_error_pos(s, p - 1, "Bad Unicode escape"); + goto fail; + } + c = (c << 4) | h; + } + break; + case '\n': + if (s->ext_json) + continue; + goto bad_escape; + case 'v': + if (s->ext_json) { + c = '\v'; + break; + } + goto bad_escape; + default: + if (c == sep) + break; + if (p > s->buf_end) + goto end_of_input; + bad_escape: + js_parse_error_pos(s, p - 1, "Bad escaped character"); + goto fail; + } + } else + if (c >= 0x80) { + c = unicode_from_utf8(p - 1, UTF8_CHAR_LEN_MAX, &p_next); + if (c > 0x10FFFF) { + js_parse_error_pos(s, p - 1, "Bad UTF-8 sequence"); + goto fail; + } + p = p_next; + } + if (string_buffer_putc(b, c)) + goto fail; + } + s->token.val = TOK_STRING; + s->token.u.str.sep = sep; + s->token.u.str.str = string_buffer_end(b); + *pp = p; + return 0; + + end_of_input: + js_parse_error(s, "Unexpected end of JSON input"); + fail: + string_buffer_free(b); + return -1; +} + +static int json_parse_number(JSParseState *s, const uint8_t **pp) +{ + const uint8_t *p = *pp; + const uint8_t *p_start = p; + int radix; + double d; + JSATODTempMem atod_mem; + + if (*p == '+' || *p == '-') + p++; + + if (!is_digit(*p)) { + if (s->ext_json) { + if (strstart((const char *)p, "Infinity", (const char **)&p)) { + d = 1.0 / 0.0; + if (*p_start == '-') + d = -d; + goto done; + } else if (strstart((const char *)p, "NaN", (const char **)&p)) { + d = NAN; + goto done; + } else if (*p != '.') { + goto unexpected_token; + } + } else { + goto unexpected_token; + } + } + + if (p[0] == '0') { + if (s->ext_json) { + /* also accepts base 16, 8 and 2 prefix for integers */ + radix = 10; + if (p[1] == 'x' || p[1] == 'X') { + p += 2; + radix = 16; + } else if ((p[1] == 'o' || p[1] == 'O')) { + p += 2; + radix = 8; + } else if ((p[1] == 'b' || p[1] == 'B')) { + p += 2; + radix = 2; + } + if (radix != 10) { + /* prefix is present */ + if (to_digit(*p) >= radix) { + unexpected_token: + return js_parse_error_pos(s, p, "Unexpected token '%c'", *p); + } + d = js_atod((const char *)p_start, (const char **)&p, 0, + JS_ATOD_INT_ONLY | JS_ATOD_ACCEPT_BIN_OCT, &atod_mem); + goto done; + } + } + if (is_digit(p[1])) + return js_parse_error_pos(s, p, "Unexpected number"); + } + + while (is_digit(*p)) + p++; + + if (*p == '.') { + p++; + if (!is_digit(*p)) + return js_parse_error_pos(s, p, "Unterminated fractional number"); + while (is_digit(*p)) + p++; + } + if (*p == 'e' || *p == 'E') { + p++; + if (*p == '+' || *p == '-') + p++; + if (!is_digit(*p)) + return js_parse_error_pos(s, p, "Exponent part is missing a number"); + while (is_digit(*p)) + p++; + } + d = js_atod((const char *)p_start, NULL, 10, 0, &atod_mem); + done: + s->token.val = TOK_NUMBER; + s->token.u.num.val = JS_NewFloat64(s->ctx, d); + *pp = p; + return 0; +} + static __exception int json_next_token(JSParseState *s) { const uint8_t *p; @@ -21811,7 +22763,8 @@ static __exception int json_next_token(JSParseState *s) } /* fall through */ case '\"': - if (js_parse_string(s, c, TRUE, p + 1, &s->token, &p)) + p++; + if (json_parse_string(s, &p, c)) goto fail; break; case '\r': /* accept DOS and MAC newline sequences */ @@ -21901,7 +22854,6 @@ static __exception int json_next_token(JSParseState *s) case 'Y': case 'Z': case '_': case '$': - /* identifier : only pure ascii characters are accepted */ p++; atom = json_parse_ident(s, &p, c); if (atom == JS_ATOM_NULL) @@ -21912,39 +22864,23 @@ static __exception int json_next_token(JSParseState *s) s->token.val = TOK_IDENT; break; case '+': - if (!s->ext_json || !is_digit(p[1])) + if (!s->ext_json) goto def_token; goto parse_number; - case '0': - if (is_digit(p[1])) + case '.': + if (s->ext_json && is_digit(p[1])) + goto parse_number; + else goto def_token; - goto parse_number; case '-': - if (!is_digit(p[1])) - goto def_token; - goto parse_number; + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': /* number */ parse_number: - { - JSValue ret; - int flags, radix; - if (!s->ext_json) { - flags = 0; - radix = 10; - } else { - flags = ATOD_ACCEPT_BIN_OCT; - radix = 0; - } - ret = js_atof(s->ctx, (const char *)p, (const char **)&p, radix, - flags); - if (JS_IsException(ret)) - goto fail; - s->token.val = TOK_NUMBER; - s->token.u.num.val = ret; - } + if (json_parse_number(s, &p)) + goto fail; break; default: if (c >= 128) { @@ -22129,7 +23065,7 @@ BOOL JS_DetectModule(const char *input, size_t input_len) } static inline int get_prev_opcode(JSFunctionDef *fd) { - if (fd->last_opcode_pos < 0) + if (fd->last_opcode_pos < 0 || dbuf_error(&fd->byte_code)) return OP_invalid; else return fd->byte_code.buf[fd->last_opcode_pos]; @@ -22194,7 +23130,11 @@ static void emit_op(JSParseState *s, uint8_t val) static void emit_atom(JSParseState *s, JSAtom name) { - emit_u32(s, JS_DupAtom(s->ctx, name)); + DynBuf *bc = &s->cur_func->byte_code; + if (dbuf_claim(bc, 4)) + return; /* not enough memory : don't duplicate the atom */ + put_u32(bc->buf + bc->size, JS_DupAtom(s->ctx, name)); + bc->size += 4; } static int update_label(JSFunctionDef *s, int label, int delta) @@ -22208,29 +23148,33 @@ static int update_label(JSFunctionDef *s, int label, int delta) return ls->ref_count; } -static int new_label_fd(JSFunctionDef *fd, int label) +static int new_label_fd(JSFunctionDef *fd) { + int label; LabelSlot *ls; - if (label < 0) { - if (js_resize_array(fd->ctx, (void *)&fd->label_slots, - sizeof(fd->label_slots[0]), - &fd->label_size, fd->label_count + 1)) - return -1; - label = fd->label_count++; - ls = &fd->label_slots[label]; - ls->ref_count = 0; - ls->pos = -1; - ls->pos2 = -1; - ls->addr = -1; - ls->first_reloc = NULL; - } + if (js_resize_array(fd->ctx, (void *)&fd->label_slots, + sizeof(fd->label_slots[0]), + &fd->label_size, fd->label_count + 1)) + return -1; + label = fd->label_count++; + ls = &fd->label_slots[label]; + ls->ref_count = 0; + ls->pos = -1; + ls->pos2 = -1; + ls->addr = -1; + ls->first_reloc = NULL; return label; } static int new_label(JSParseState *s) { - return new_label_fd(s->cur_func, -1); + int label; + label = new_label_fd(s->cur_func); + if (unlikely(label < 0)) { + dbuf_set_error(&s->cur_func->byte_code); + } + return label; } /* don't update the last opcode and don't emit line number info */ @@ -22258,8 +23202,11 @@ static int emit_label(JSParseState *s, int label) static int emit_goto(JSParseState *s, int opcode, int label) { if (js_is_live_code(s)) { - if (label < 0) + if (label < 0) { label = new_label(s); + if (label < 0) + return -1; + } emit_op(s, opcode); emit_u32(s, label); s->cur_func->label_slots[label].ref_count++; @@ -23017,7 +23964,7 @@ static int __exception js_parse_property_name(JSParseState *s, } else if (s->token.val == '[') { if (next_token(s)) goto fail; - if (js_parse_expr(s)) + if (js_parse_assign_expr(s)) goto fail; if (js_parse_expect(s, ']')) goto fail; @@ -24218,18 +25165,19 @@ static __exception int js_parse_array_literal(JSParseState *s) return js_parse_expect(s, ']'); } -/* XXX: remove */ +/* check if scope chain contains a with statement */ static BOOL has_with_scope(JSFunctionDef *s, int scope_level) { - /* check if scope chain contains a with statement */ while (s) { - int scope_idx = s->scopes[scope_level].first; - while (scope_idx >= 0) { - JSVarDef *vd = &s->vars[scope_idx]; - - if (vd->var_name == JS_ATOM__with_) - return TRUE; - scope_idx = vd->scope_next; + /* no with in strict mode */ + if (!(s->js_mode & JS_MODE_STRICT)) { + int scope_idx = s->scopes[scope_level].first; + while (scope_idx >= 0) { + JSVarDef *vd = &s->vars[scope_idx]; + if (vd->var_name == JS_ATOM__with_) + return TRUE; + scope_idx = vd->scope_next; + } } /* check parent scopes */ scope_level = s->parent_scope_level; @@ -24262,7 +25210,11 @@ static __exception int get_lvalue(JSParseState *s, int *popcode, int *pscope, } if (name == JS_ATOM_this || name == JS_ATOM_new_target) goto invalid_lvalue; - depth = 2; /* will generate OP_get_ref_value */ + if (has_with_scope(fd, scope)) { + depth = 2; /* will generate OP_get_ref_value */ + } else { + depth = 0; + } break; case OP_get_field: name = get_u32(fd->byte_code.buf + fd->last_opcode_pos + 1); @@ -24299,14 +25251,22 @@ static __exception int get_lvalue(JSParseState *s, int *popcode, int *pscope, /* get the value but keep the object/fields on the stack */ switch(opcode) { case OP_scope_get_var: - label = new_label(s); - emit_op(s, OP_scope_make_ref); - emit_atom(s, name); - emit_u32(s, label); - emit_u16(s, scope); - update_label(fd, label, 1); - emit_op(s, OP_get_ref_value); - opcode = OP_get_ref_value; + if (depth != 0) { + label = new_label(s); + if (label < 0) + return -1; + emit_op(s, OP_scope_make_ref); + emit_atom(s, name); + emit_u32(s, label); + emit_u16(s, scope); + update_label(fd, label, 1); + emit_op(s, OP_get_ref_value); + opcode = OP_get_ref_value; + } else { + emit_op(s, OP_scope_get_var); + emit_atom(s, name); + emit_u16(s, scope); + } break; case OP_get_field: emit_op(s, OP_get_field2); @@ -24331,13 +25291,17 @@ static __exception int get_lvalue(JSParseState *s, int *popcode, int *pscope, } else { switch(opcode) { case OP_scope_get_var: - label = new_label(s); - emit_op(s, OP_scope_make_ref); - emit_atom(s, name); - emit_u32(s, label); - emit_u16(s, scope); - update_label(fd, label, 1); - opcode = OP_get_ref_value; + if (depth != 0) { + label = new_label(s); + if (label < 0) + return -1; + emit_op(s, OP_scope_make_ref); + emit_atom(s, name); + emit_u32(s, label); + emit_u16(s, scope); + update_label(fd, label, 1); + opcode = OP_get_ref_value; + } break; default: break; @@ -24371,6 +25335,21 @@ static void put_lvalue(JSParseState *s, int opcode, int scope, BOOL is_let) { switch(opcode) { + case OP_scope_get_var: + /* depth = 0 */ + switch(special) { + case PUT_LVALUE_NOKEEP: + case PUT_LVALUE_NOKEEP_DEPTH: + case PUT_LVALUE_KEEP_SECOND: + case PUT_LVALUE_NOKEEP_BOTTOM: + break; + case PUT_LVALUE_KEEP_TOP: + emit_op(s, OP_dup); + break; + default: + abort(); + } + break; case OP_get_field: case OP_scope_get_private_field: /* depth = 1 */ @@ -24442,8 +25421,6 @@ static void put_lvalue(JSParseState *s, int opcode, int scope, switch(opcode) { case OP_scope_get_var: /* val -- */ - assert(special == PUT_LVALUE_NOKEEP || - special == PUT_LVALUE_NOKEEP_DEPTH); emit_op(s, is_let ? OP_scope_put_var_init : OP_scope_put_var); emit_u32(s, name); /* has refcount */ emit_u16(s, scope); @@ -24797,6 +25774,8 @@ static int js_parse_destructuring_element(JSParseState *s, int tok, int is_arg, /* swap ref and lvalue object if any */ if (prop_name == JS_ATOM_NULL) { switch(depth_lvalue) { + case 0: + break; case 1: /* source prop x -> x source prop */ emit_op(s, OP_rot3r); @@ -24810,9 +25789,13 @@ static int js_parse_destructuring_element(JSParseState *s, int tok, int is_arg, emit_op(s, OP_rot5l); emit_op(s, OP_rot5l); break; + default: + abort(); } } else { switch(depth_lvalue) { + case 0: + break; case 1: /* source x -> x source */ emit_op(s, OP_swap); @@ -24825,6 +25808,8 @@ static int js_parse_destructuring_element(JSParseState *s, int tok, int is_arg, /* source x y z -> x y z source */ emit_op(s, OP_rot4l); break; + default: + abort(); } } } @@ -25341,6 +26326,23 @@ static __exception int js_parse_postfix_expr(JSParseState *s, int parse_flags) return js_parse_error(s, "invalid use of 'import()'"); if (js_parse_assign_expr(s)) return -1; + if (s->token.val == ',') { + if (next_token(s)) + return -1; + if (s->token.val != ')') { + if (js_parse_assign_expr(s)) + return -1; + /* accept a trailing comma */ + if (s->token.val == ',') { + if (next_token(s)) + return -1; + } + } else { + emit_op(s, OP_undefined); + } + } else { + emit_op(s, OP_undefined); + } if (js_parse_expect(s, ')')) return -1; emit_op(s, OP_import); @@ -25357,6 +26359,8 @@ static __exception int js_parse_postfix_expr(JSParseState *s, int parse_flags) BOOL has_optional_chain = FALSE; if (s->token.val == TOK_QUESTION_MARK_DOT) { + if ((parse_flags & PF_POSTFIX_CALL) == 0) + return js_parse_error(s, "new keyword cannot be used with an optional chain"); op_token_ptr = s->token.ptr; /* optional chaining */ if (next_token(s)) @@ -26457,7 +27461,7 @@ static __exception int js_parse_assign_expr2(JSParseState *s, int parse_flags) } if (op == '=') { - if (opcode == OP_get_ref_value && name == name0) { + if ((opcode == OP_get_ref_value || opcode == OP_scope_get_var) && name == name0) { set_object_name(s, name); } } else { @@ -26492,11 +27496,14 @@ static __exception int js_parse_assign_expr2(JSParseState *s, int parse_flags) return -1; } - if (opcode == OP_get_ref_value && name == name0) { + if ((opcode == OP_get_ref_value || opcode == OP_scope_get_var) && name == name0) { set_object_name(s, name); } switch(depth_lvalue) { + case 0: + emit_op(s, OP_dup); + break; case 1: emit_op(s, OP_insert2); break; @@ -27077,7 +28084,7 @@ static __exception int js_parse_for_in_of(JSParseState *s, int label_name, int chunk_size = pos_expr - pos_next; int offset = bc->size - pos_next; int i; - dbuf_realloc(bc, bc->size + chunk_size); + dbuf_claim(bc, chunk_size); dbuf_put(bc, bc->buf + pos_next, chunk_size); memset(bc->buf + pos_next, OP_nop, chunk_size); /* `next` part ends with a goto */ @@ -27483,7 +28490,7 @@ static __exception int js_parse_statement_or_decl(JSParseState *s, int chunk_size = pos_body - pos_cont; int offset = bc->size - pos_cont; int i; - dbuf_realloc(bc, bc->size + chunk_size); + dbuf_claim(bc, chunk_size); dbuf_put(bc, bc->buf + pos_cont, chunk_size); memset(bc->buf + pos_cont, OP_nop, chunk_size); /* increment part ends with a goto */ @@ -27900,7 +28907,7 @@ static __exception int js_parse_statement_or_decl(JSParseState *s, return -1; } -/* 'name' is freed */ +/* 'name' is freed. The module is referenced by 'ctx->loaded_modules' */ static JSModuleDef *js_new_module_def(JSContext *ctx, JSAtom name) { JSModuleDef *m; @@ -27910,6 +28917,7 @@ static JSModuleDef *js_new_module_def(JSContext *ctx, JSAtom name) return NULL; } m->header.ref_count = 1; + add_gc_object(ctx->rt, &m->header, JS_GC_OBJ_TYPE_MODULE); m->module_name = name; m->module_ns = JS_UNDEFINED; m->func_obj = JS_UNDEFINED; @@ -27918,6 +28926,7 @@ static JSModuleDef *js_new_module_def(JSContext *ctx, JSAtom name) m->promise = JS_UNDEFINED; m->resolving_funcs[0] = JS_UNDEFINED; m->resolving_funcs[1] = JS_UNDEFINED; + m->private_value = JS_UNDEFINED; list_add_tail(&m->link, &ctx->loaded_modules); return m; } @@ -27927,6 +28936,11 @@ static void js_mark_module_def(JSRuntime *rt, JSModuleDef *m, { int i; + for(i = 0; i < m->req_module_entries_count; i++) { + JSReqModuleEntry *rme = &m->req_module_entries[i]; + JS_MarkValue(rt, rme->attributes, mark_func); + } + for(i = 0; i < m->export_entries_count; i++) { JSExportEntry *me = &m->export_entries[i]; if (me->export_type == JS_EXPORT_TYPE_LOCAL && @@ -27942,61 +28956,65 @@ static void js_mark_module_def(JSRuntime *rt, JSModuleDef *m, JS_MarkValue(rt, m->promise, mark_func); JS_MarkValue(rt, m->resolving_funcs[0], mark_func); JS_MarkValue(rt, m->resolving_funcs[1], mark_func); + JS_MarkValue(rt, m->private_value, mark_func); } -static void js_free_module_def(JSContext *ctx, JSModuleDef *m) +static void js_free_module_def(JSRuntime *rt, JSModuleDef *m) { int i; - JS_FreeAtom(ctx, m->module_name); + JS_FreeAtomRT(rt, m->module_name); for(i = 0; i < m->req_module_entries_count; i++) { JSReqModuleEntry *rme = &m->req_module_entries[i]; - JS_FreeAtom(ctx, rme->module_name); + JS_FreeAtomRT(rt, rme->module_name); + JS_FreeValueRT(rt, rme->attributes); } - js_free(ctx, m->req_module_entries); + js_free_rt(rt, m->req_module_entries); for(i = 0; i < m->export_entries_count; i++) { JSExportEntry *me = &m->export_entries[i]; if (me->export_type == JS_EXPORT_TYPE_LOCAL) - free_var_ref(ctx->rt, me->u.local.var_ref); - JS_FreeAtom(ctx, me->export_name); - JS_FreeAtom(ctx, me->local_name); + free_var_ref(rt, me->u.local.var_ref); + JS_FreeAtomRT(rt, me->export_name); + JS_FreeAtomRT(rt, me->local_name); } - js_free(ctx, m->export_entries); + js_free_rt(rt, m->export_entries); - js_free(ctx, m->star_export_entries); + js_free_rt(rt, m->star_export_entries); for(i = 0; i < m->import_entries_count; i++) { JSImportEntry *mi = &m->import_entries[i]; - JS_FreeAtom(ctx, mi->import_name); + JS_FreeAtomRT(rt, mi->import_name); } - js_free(ctx, m->import_entries); - js_free(ctx, m->async_parent_modules); + js_free_rt(rt, m->import_entries); + js_free_rt(rt, m->async_parent_modules); - JS_FreeValue(ctx, m->module_ns); - JS_FreeValue(ctx, m->func_obj); - JS_FreeValue(ctx, m->eval_exception); - JS_FreeValue(ctx, m->meta_obj); - JS_FreeValue(ctx, m->promise); - JS_FreeValue(ctx, m->resolving_funcs[0]); - JS_FreeValue(ctx, m->resolving_funcs[1]); - list_del(&m->link); - js_free(ctx, m); + JS_FreeValueRT(rt, m->module_ns); + JS_FreeValueRT(rt, m->func_obj); + JS_FreeValueRT(rt, m->eval_exception); + JS_FreeValueRT(rt, m->meta_obj); + JS_FreeValueRT(rt, m->promise); + JS_FreeValueRT(rt, m->resolving_funcs[0]); + JS_FreeValueRT(rt, m->resolving_funcs[1]); + JS_FreeValueRT(rt, m->private_value); + /* during the GC the finalizers are called in an arbitrary + order so the module may no longer be referenced by the JSContext list */ + if (m->link.next) { + list_del(&m->link); + } + remove_gc_object(&m->header); + if (rt->gc_phase == JS_GC_PHASE_REMOVE_CYCLES && m->header.ref_count != 0) { + list_add_tail(&m->header.link, &rt->gc_zero_ref_count_list); + } else { + js_free_rt(rt, m); + } } static int add_req_module_entry(JSContext *ctx, JSModuleDef *m, JSAtom module_name) { JSReqModuleEntry *rme; - int i; - - /* no need to add the module request if it is already present */ - for(i = 0; i < m->req_module_entries_count; i++) { - rme = &m->req_module_entries[i]; - if (rme->module_name == module_name) - return i; - } if (js_resize_array(ctx, (void **)&m->req_module_entries, sizeof(JSReqModuleEntry), @@ -28006,7 +29024,8 @@ static int add_req_module_entry(JSContext *ctx, JSModuleDef *m, rme = &m->req_module_entries[m->req_module_entries_count++]; rme->module_name = JS_DupAtom(ctx, module_name); rme->module = NULL; - return i; + rme->attributes = JS_UNDEFINED; + return m->req_module_entries_count - 1; } static JSExportEntry *find_export_entry(JSContext *ctx, JSModuleDef *m, @@ -28086,6 +29105,8 @@ JSModuleDef *JS_NewCModule(JSContext *ctx, const char *name_str, if (name == JS_ATOM_NULL) return NULL; m = js_new_module_def(ctx, name); + if (!m) + return NULL; m->init_func = func; return m; } @@ -28125,12 +29146,38 @@ int JS_SetModuleExport(JSContext *ctx, JSModuleDef *m, const char *export_name, return -1; } +int JS_SetModulePrivateValue(JSContext *ctx, JSModuleDef *m, JSValue val) +{ + set_value(ctx, &m->private_value, val); + return 0; +} + +JSValue JS_GetModulePrivateValue(JSContext *ctx, JSModuleDef *m) +{ + return JS_DupValue(ctx, m->private_value); +} + void JS_SetModuleLoaderFunc(JSRuntime *rt, JSModuleNormalizeFunc *module_normalize, JSModuleLoaderFunc *module_loader, void *opaque) { rt->module_normalize_func = module_normalize; - rt->module_loader_func = module_loader; + rt->module_loader_has_attr = FALSE; + rt->u.module_loader_func = module_loader; + rt->module_check_attrs = NULL; + rt->module_loader_opaque = opaque; +} + +void JS_SetModuleLoaderFunc2(JSRuntime *rt, + JSModuleNormalizeFunc *module_normalize, + JSModuleLoaderFunc2 *module_loader, + JSModuleCheckSupportedImportAttributes *module_check_attrs, + void *opaque) +{ + rt->module_normalize_func = module_normalize; + rt->module_loader_has_attr = TRUE; + rt->u.module_loader_func2 = module_loader; + rt->module_check_attrs = module_check_attrs; rt->module_loader_opaque = opaque; } @@ -28211,7 +29258,8 @@ static JSModuleDef *js_find_loaded_module(JSContext *ctx, JSAtom name) /* return NULL in case of exception (e.g. module could not be loaded) */ static JSModuleDef *js_host_resolve_imported_module(JSContext *ctx, const char *base_cname, - const char *cname1) + const char *cname1, + JSValueConst attributes) { JSRuntime *rt = ctx->rt; JSModuleDef *m; @@ -28244,22 +29292,26 @@ static JSModuleDef *js_host_resolve_imported_module(JSContext *ctx, JS_FreeAtom(ctx, module_name); /* load the module */ - if (!rt->module_loader_func) { + if (!rt->u.module_loader_func) { /* XXX: use a syntax error ? */ JS_ThrowReferenceError(ctx, "could not load module '%s'", cname); js_free(ctx, cname); return NULL; } - - m = rt->module_loader_func(ctx, cname, rt->module_loader_opaque); + if (rt->module_loader_has_attr) { + m = rt->u.module_loader_func2(ctx, cname, rt->module_loader_opaque, attributes); + } else { + m = rt->u.module_loader_func(ctx, cname, rt->module_loader_opaque); + } js_free(ctx, cname); return m; } static JSModuleDef *js_host_resolve_imported_module_atom(JSContext *ctx, - JSAtom base_module_name, - JSAtom module_name1) + JSAtom base_module_name, + JSAtom module_name1, + JSValueConst attributes) { const char *base_cname, *cname; JSModuleDef *m; @@ -28272,7 +29324,7 @@ static JSModuleDef *js_host_resolve_imported_module_atom(JSContext *ctx, JS_FreeCString(ctx, base_cname); return NULL; } - m = js_host_resolve_imported_module(ctx, base_cname, cname); + m = js_host_resolve_imported_module(ctx, base_cname, cname, attributes); JS_FreeCString(ctx, base_cname); JS_FreeCString(ctx, cname); return m; @@ -28683,6 +29735,7 @@ static JSValue js_build_module_ns(JSContext *ctx, JSModuleDef *m) en->export_name, JS_AUTOINIT_ID_MODULE_NS, m, JS_PROP_ENUMERABLE | JS_PROP_WRITABLE) < 0) + goto fail; break; default: break; @@ -28734,7 +29787,8 @@ static int js_resolve_module(JSContext *ctx, JSModuleDef *m) for(i = 0; i < m->req_module_entries_count; i++) { JSReqModuleEntry *rme = &m->req_module_entries[i]; m1 = js_host_resolve_imported_module_atom(ctx, m->module_name, - rme->module_name); + rme->module_name, + rme->attributes); if (!m1) return -1; rme->module = m1; @@ -28746,31 +29800,11 @@ static int js_resolve_module(JSContext *ctx, JSModuleDef *m) return 0; } -static JSVarRef *js_create_module_var(JSContext *ctx, BOOL is_lexical) -{ - JSVarRef *var_ref; - var_ref = js_malloc(ctx, sizeof(JSVarRef)); - if (!var_ref) - return NULL; - var_ref->header.ref_count = 1; - if (is_lexical) - var_ref->value = JS_UNINITIALIZED; - else - var_ref->value = JS_UNDEFINED; - var_ref->pvalue = &var_ref->value; - var_ref->is_detached = TRUE; - add_gc_object(ctx->rt, &var_ref->header, JS_GC_OBJ_TYPE_VAR_REF); - return var_ref; -} - /* Create the function associated with the module */ static int js_create_module_bytecode_function(JSContext *ctx, JSModuleDef *m) { JSFunctionBytecode *b; - int i; - JSVarRef **var_refs; JSValue func_obj, bfunc; - JSObject *p; bfunc = m->func_obj; func_obj = JS_NewObjectProtoClass(ctx, ctx->function_proto, @@ -28779,40 +29813,14 @@ static int js_create_module_bytecode_function(JSContext *ctx, JSModuleDef *m) if (JS_IsException(func_obj)) return -1; b = JS_VALUE_GET_PTR(bfunc); - - p = JS_VALUE_GET_OBJ(func_obj); - p->u.func.function_bytecode = b; - b->header.ref_count++; - p->u.func.home_object = NULL; - p->u.func.var_refs = NULL; - if (b->closure_var_count) { - var_refs = js_mallocz(ctx, sizeof(var_refs[0]) * b->closure_var_count); - if (!var_refs) - goto fail; - p->u.func.var_refs = var_refs; - - /* create the global variables. The other variables are - imported from other modules */ - for(i = 0; i < b->closure_var_count; i++) { - JSClosureVar *cv = &b->closure_var[i]; - JSVarRef *var_ref; - if (cv->is_local) { - var_ref = js_create_module_var(ctx, cv->is_lexical); - if (!var_ref) - goto fail; -#ifdef DUMP_MODULE_RESOLVE - printf("local %d: %p\n", i, var_ref); -#endif - var_refs[i] = var_ref; - } - } + func_obj = js_closure2(ctx, func_obj, b, NULL, NULL, TRUE, m); + if (JS_IsException(func_obj)) { + m->func_obj = JS_UNDEFINED; /* XXX: keep it ? */ + JS_FreeValue(ctx, func_obj); + return -1; } m->func_obj = func_obj; - JS_FreeValue(ctx, bfunc); return 0; - fail: - JS_FreeValue(ctx, func_obj); - return -1; } /* must be done before js_link_module() because of cyclic references */ @@ -28832,7 +29840,7 @@ static int js_create_module_function(JSContext *ctx, JSModuleDef *m) for(i = 0; i < m->export_entries_count; i++) { JSExportEntry *me = &m->export_entries[i]; if (me->export_type == JS_EXPORT_TYPE_LOCAL) { - var_ref = js_create_module_var(ctx, FALSE); + var_ref = js_create_var_ref(ctx, FALSE); if (!var_ref) return -1; me->u.local.var_ref = var_ref; @@ -28991,7 +29999,7 @@ static int js_inner_module_linking(JSContext *ctx, JSModuleDef *m, val = JS_GetModuleNamespace(ctx, m2); if (JS_IsException(val)) goto fail; - var_ref = js_create_module_var(ctx, TRUE); + var_ref = js_create_var_ref(ctx, TRUE); if (!var_ref) { JS_FreeValue(ctx, val); goto fail; @@ -29211,14 +30219,15 @@ static JSValue js_load_module_fulfilled(JSContext *ctx, JSValueConst this_val, static void JS_LoadModuleInternal(JSContext *ctx, const char *basename, const char *filename, - JSValueConst *resolving_funcs) + JSValueConst *resolving_funcs, + JSValueConst attributes) { JSValue evaluate_promise; JSModuleDef *m; JSValue ret, err, func_obj, evaluate_resolving_funcs[2]; JSValueConst func_data[3]; - m = js_host_resolve_imported_module(ctx, basename, filename); + m = js_host_resolve_imported_module(ctx, basename, filename, attributes); if (!m) goto fail; @@ -29265,7 +30274,7 @@ JSValue JS_LoadModule(JSContext *ctx, const char *basename, if (JS_IsException(promise)) return JS_EXCEPTION; JS_LoadModuleInternal(ctx, basename, filename, - (JSValueConst *)resolving_funcs); + (JSValueConst *)resolving_funcs, JS_UNDEFINED); JS_FreeValue(ctx, resolving_funcs[0]); JS_FreeValue(ctx, resolving_funcs[1]); return promise; @@ -29277,6 +30286,7 @@ static JSValue js_dynamic_import_job(JSContext *ctx, JSValueConst *resolving_funcs = argv; JSValueConst basename_val = argv[2]; JSValueConst specifier = argv[3]; + JSValueConst attributes = argv[4]; const char *basename = NULL, *filename; JSValue ret, err; @@ -29293,7 +30303,7 @@ static JSValue js_dynamic_import_job(JSContext *ctx, goto exception; JS_LoadModuleInternal(ctx, basename, filename, - resolving_funcs); + resolving_funcs, attributes); JS_FreeCString(ctx, filename); JS_FreeCString(ctx, basename); return JS_UNDEFINED; @@ -29307,11 +30317,12 @@ static JSValue js_dynamic_import_job(JSContext *ctx, return JS_UNDEFINED; } -static JSValue js_dynamic_import(JSContext *ctx, JSValueConst specifier) +static JSValue js_dynamic_import(JSContext *ctx, JSValueConst specifier, JSValueConst options) { JSAtom basename; - JSValue promise, resolving_funcs[2], basename_val; - JSValueConst args[4]; + JSValue promise, resolving_funcs[2], basename_val, err, ret; + JSValue specifier_str = JS_UNDEFINED, attributes = JS_UNDEFINED, attributes_obj = JS_UNDEFINED; + JSValueConst args[5]; basename = JS_GetScriptOrModuleName(ctx, 0); if (basename == JS_ATOM_NULL) @@ -29328,19 +30339,82 @@ static JSValue js_dynamic_import(JSContext *ctx, JSValueConst specifier) return promise; } + /* the string conversion must occur here */ + specifier_str = JS_ToString(ctx, specifier); + if (JS_IsException(specifier_str)) + goto exception; + + if (!JS_IsUndefined(options)) { + if (!JS_IsObject(options)) { + JS_ThrowTypeError(ctx, "options must be an object"); + goto exception; + } + attributes_obj = JS_GetProperty(ctx, options, JS_ATOM_with); + if (JS_IsException(attributes_obj)) + goto exception; + if (!JS_IsUndefined(attributes_obj)) { + JSPropertyEnum *atoms; + uint32_t atoms_len, i; + JSValue val; + + if (!JS_IsObject(attributes_obj)) { + JS_ThrowTypeError(ctx, "options.with must be an object"); + goto exception; + } + attributes = JS_NewObjectProto(ctx, JS_NULL); + if (JS_GetOwnPropertyNamesInternal(ctx, &atoms, &atoms_len, JS_VALUE_GET_OBJ(attributes_obj), + JS_GPN_STRING_MASK | JS_GPN_ENUM_ONLY)) { + goto exception; + } + for(i = 0; i < atoms_len; i++) { + val = JS_GetProperty(ctx, attributes_obj, atoms[i].atom); + if (JS_IsException(val)) + goto exception1; + if (!JS_IsString(val)) { + JS_FreeValue(ctx, val); + JS_ThrowTypeError(ctx, "module attribute values must be strings"); + goto exception1; + } + if (JS_DefinePropertyValue(ctx, attributes, atoms[i].atom, val, + JS_PROP_C_W_E) < 0) { + exception1: + JS_FreePropertyEnum(ctx, atoms, atoms_len); + goto exception; + } + } + JS_FreePropertyEnum(ctx, atoms, atoms_len); + if (ctx->rt->module_check_attrs && + ctx->rt->module_check_attrs(ctx, ctx->rt->module_loader_opaque, attributes) < 0) { + goto exception; + } + JS_FreeValue(ctx, attributes_obj); + } + } + args[0] = resolving_funcs[0]; args[1] = resolving_funcs[1]; args[2] = basename_val; - args[3] = specifier; - + args[3] = specifier_str; + args[4] = attributes; + /* cannot run JS_LoadModuleInternal synchronously because it would cause an unexpected recursion in js_evaluate_module() */ - JS_EnqueueJob(ctx, js_dynamic_import_job, 4, args); - + JS_EnqueueJob(ctx, js_dynamic_import_job, 5, args); + done: JS_FreeValue(ctx, basename_val); JS_FreeValue(ctx, resolving_funcs[0]); JS_FreeValue(ctx, resolving_funcs[1]); + JS_FreeValue(ctx, specifier_str); + JS_FreeValue(ctx, attributes); return promise; + exception: + JS_FreeValue(ctx, attributes_obj); + err = JS_GetException(ctx); + ret = JS_Call(ctx, resolving_funcs[1], JS_UNDEFINED, + 1, (JSValueConst *)&err); + JS_FreeValue(ctx, ret); + JS_FreeValue(ctx, err); + goto done; } static void js_set_module_evaluated(JSContext *ctx, JSModuleDef *m) @@ -29417,6 +30491,14 @@ static int exec_module_list_cmp(const void *p1, const void *p2, void *opaque) static int js_execute_async_module(JSContext *ctx, JSModuleDef *m); static int js_execute_sync_module(JSContext *ctx, JSModuleDef *m, JSValue *pvalue); +#ifdef DUMP_MODULE_EXEC +static void js_dump_module(JSContext *ctx, const char *str, JSModuleDef *m) +{ + char buf1[ATOM_GET_STR_BUF_SIZE]; + static const char *module_status_str[] = { "unlinked", "linking", "linked", "evaluating", "evaluating_async", "evaluated" }; + printf("%s: %s status=%s\n", str, JS_AtomGetStr(ctx, buf1, sizeof(buf1), m->module_name), module_status_str[m->status]); +} +#endif static JSValue js_async_module_execution_rejected(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, int magic, JSValue *func_data) @@ -29425,6 +30507,9 @@ static JSValue js_async_module_execution_rejected(JSContext *ctx, JSValueConst t JSValueConst error = argv[0]; int i; +#ifdef DUMP_MODULE_EXEC + js_dump_module(ctx, __func__, module); +#endif if (js_check_stack_overflow(ctx->rt, 0)) return JS_ThrowStackOverflow(ctx); @@ -29440,14 +30525,7 @@ static JSValue js_async_module_execution_rejected(JSContext *ctx, JSValueConst t module->eval_has_exception = TRUE; module->eval_exception = JS_DupValue(ctx, error); module->status = JS_MODULE_STATUS_EVALUATED; - - for(i = 0; i < module->async_parent_modules_count; i++) { - JSModuleDef *m = module->async_parent_modules[i]; - JSValue m_obj = JS_NewModuleValue(ctx, m); - js_async_module_execution_rejected(ctx, JS_UNDEFINED, 1, &error, 0, - &m_obj); - JS_FreeValue(ctx, m_obj); - } + module->async_evaluation = FALSE; if (!JS_IsUndefined(module->promise)) { JSValue ret_val; @@ -29456,6 +30534,14 @@ static JSValue js_async_module_execution_rejected(JSContext *ctx, JSValueConst t 1, &error); JS_FreeValue(ctx, ret_val); } + + for(i = 0; i < module->async_parent_modules_count; i++) { + JSModuleDef *m = module->async_parent_modules[i]; + JSValue m_obj = JS_NewModuleValue(ctx, m); + js_async_module_execution_rejected(ctx, JS_UNDEFINED, 1, &error, 0, + &m_obj); + JS_FreeValue(ctx, m_obj); + } return JS_UNDEFINED; } @@ -29466,6 +30552,9 @@ static JSValue js_async_module_execution_fulfilled(JSContext *ctx, JSValueConst ExecModuleList exec_list_s, *exec_list = &exec_list_s; int i; +#ifdef DUMP_MODULE_EXEC + js_dump_module(ctx, __func__, module); +#endif if (module->status == JS_MODULE_STATUS_EVALUATED) { assert(module->eval_has_exception); return JS_UNDEFINED; @@ -29491,6 +30580,9 @@ static JSValue js_async_module_execution_fulfilled(JSContext *ctx, JSValueConst for(i = 0; i < exec_list->count; i++) { JSModuleDef *m = exec_list->tab[i]; +#ifdef DUMP_MODULE_EXEC + printf(" %d/%d", i, exec_list->count); js_dump_module(ctx, "", m); +#endif if (m->status == JS_MODULE_STATUS_EVALUATED) { assert(m->eval_has_exception); } else if (m->has_tla) { @@ -29505,6 +30597,7 @@ static JSValue js_async_module_execution_fulfilled(JSContext *ctx, JSValueConst JS_FreeValue(ctx, m_obj); JS_FreeValue(ctx, error); } else { + m->async_evaluation = FALSE; js_set_module_evaluated(ctx, m); } } @@ -29517,6 +30610,9 @@ static int js_execute_async_module(JSContext *ctx, JSModuleDef *m) { JSValue promise, m_obj; JSValue resolve_funcs[2], ret_val; +#ifdef DUMP_MODULE_EXEC + js_dump_module(ctx, __func__, m); +#endif promise = js_async_function_call(ctx, m->func_obj, JS_UNDEFINED, 0, NULL, 0); if (JS_IsException(promise)) return -1; @@ -29536,6 +30632,9 @@ static int js_execute_async_module(JSContext *ctx, JSModuleDef *m) static int js_execute_sync_module(JSContext *ctx, JSModuleDef *m, JSValue *pvalue) { +#ifdef DUMP_MODULE_EXEC + js_dump_module(ctx, __func__, m); +#endif if (m->init_func) { /* C module init : no asynchronous execution */ if (m->init_func(ctx, m) < 0) @@ -29575,19 +30674,16 @@ static int js_inner_module_evaluation(JSContext *ctx, JSModuleDef *m, JSModuleDef *m1; int i; +#ifdef DUMP_MODULE_EXEC + js_dump_module(ctx, __func__, m); +#endif + if (js_check_stack_overflow(ctx->rt, 0)) { JS_ThrowStackOverflow(ctx); *pvalue = JS_GetException(ctx); return -1; } -#ifdef DUMP_MODULE_RESOLVE - { - char buf1[ATOM_GET_STR_BUF_SIZE]; - printf("js_inner_module_evaluation '%s':\n", JS_AtomGetStr(ctx, buf1, sizeof(buf1), m->module_name)); - } -#endif - if (m->status == JS_MODULE_STATUS_EVALUATING_ASYNC || m->status == JS_MODULE_STATUS_EVALUATED) { if (m->eval_has_exception) { @@ -29688,6 +30784,9 @@ static JSValue js_evaluate_module(JSContext *ctx, JSModuleDef *m) JSModuleDef *m1, *stack_top; JSValue ret_val, result; +#ifdef DUMP_MODULE_EXEC + js_dump_module(ctx, __func__, m); +#endif assert(m->status == JS_MODULE_STATUS_LINKED || m->status == JS_MODULE_STATUS_EVALUATING_ASYNC || m->status == JS_MODULE_STATUS_EVALUATED); @@ -29720,6 +30819,9 @@ static JSValue js_evaluate_module(JSContext *ctx, JSModuleDef *m) 1, (JSValueConst *)&m->eval_exception); JS_FreeValue(ctx, ret_val); } else { +#ifdef DUMP_MODULE_EXEC + js_dump_module(ctx, " done", m); +#endif assert(m->status == JS_MODULE_STATUS_EVALUATING_ASYNC || m->status == JS_MODULE_STATUS_EVALUATED); assert(!m->eval_has_exception); @@ -29736,27 +30838,109 @@ static JSValue js_evaluate_module(JSContext *ctx, JSModuleDef *m) return JS_DupValue(ctx, m->promise); } -static __exception JSAtom js_parse_from_clause(JSParseState *s) +static __exception int js_parse_with_clause(JSParseState *s, JSReqModuleEntry *rme) +{ + JSContext *ctx = s->ctx; + JSAtom key; + int ret; + const uint8_t *key_token_ptr; + + if (next_token(s)) + return -1; + if (js_parse_expect(s, '{')) + return -1; + while (s->token.val != '}') { + key_token_ptr = s->token.ptr; + if (s->token.val == TOK_STRING) { + key = JS_ValueToAtom(ctx, s->token.u.str.str); + if (key == JS_ATOM_NULL) + return -1; + } else { + if (!token_is_ident(s->token.val)) { + js_parse_error(s, "identifier expected"); + return -1; + } + key = JS_DupAtom(ctx, s->token.u.ident.atom); + } + if (next_token(s)) + return -1; + if (js_parse_expect(s, ':')) { + JS_FreeAtom(ctx, key); + return -1; + } + if (s->token.val != TOK_STRING) { + js_parse_error_pos(s, key_token_ptr, "string expected"); + return -1; + } + if (JS_IsUndefined(rme->attributes)) { + JSValue attributes = JS_NewObjectProto(ctx, JS_NULL); + if (JS_IsException(attributes)) { + JS_FreeAtom(ctx, key); + return -1; + } + rme->attributes = attributes; + } + ret = JS_HasProperty(ctx, rme->attributes, key); + if (ret != 0) { + JS_FreeAtom(ctx, key); + if (ret < 0) + return -1; + else + return js_parse_error(s, "duplicate with key"); + } + ret = JS_DefinePropertyValue(ctx, rme->attributes, key, + JS_DupValue(ctx, s->token.u.str.str), JS_PROP_C_W_E); + JS_FreeAtom(ctx, key); + if (ret < 0) + return -1; + if (next_token(s)) + return -1; + if (s->token.val != ',') + break; + if (next_token(s)) + return -1; + } + if (!JS_IsUndefined(rme->attributes) && + ctx->rt->module_check_attrs && + ctx->rt->module_check_attrs(ctx, ctx->rt->module_loader_opaque, rme->attributes) < 0) { + return -1; + } + return js_parse_expect(s, '}'); +} + +/* return the module index in m->req_module_entries[] or < 0 if error */ +static __exception int js_parse_from_clause(JSParseState *s, JSModuleDef *m) { JSAtom module_name; + int idx; + if (!token_is_pseudo_keyword(s, JS_ATOM_from)) { js_parse_error(s, "from clause expected"); - return JS_ATOM_NULL; + return -1; } if (next_token(s)) - return JS_ATOM_NULL; + return -1; if (s->token.val != TOK_STRING) { js_parse_error(s, "string expected"); - return JS_ATOM_NULL; + return -1; } module_name = JS_ValueToAtom(s->ctx, s->token.u.str.str); if (module_name == JS_ATOM_NULL) - return JS_ATOM_NULL; + return -1; if (next_token(s)) { JS_FreeAtom(s->ctx, module_name); - return JS_ATOM_NULL; + return -1; + } + + idx = add_req_module_entry(s->ctx, m, module_name); + JS_FreeAtom(s->ctx, module_name); + if (idx < 0) + return -1; + if (s->token.val == TOK_WITH) { + if (js_parse_with_clause(s, &m->req_module_entries[idx])) + return -1; } - return module_name; + return idx; } static __exception int js_parse_export(JSParseState *s) @@ -29765,7 +30949,6 @@ static __exception int js_parse_export(JSParseState *s) JSModuleDef *m = s->cur_func->module; JSAtom local_name, export_name; int first_export, idx, i, tok; - JSAtom module_name; JSExportEntry *me; if (next_token(s)) @@ -29840,11 +31023,7 @@ static __exception int js_parse_export(JSParseState *s) if (js_parse_expect(s, '}')) return -1; if (token_is_pseudo_keyword(s, JS_ATOM_from)) { - module_name = js_parse_from_clause(s); - if (module_name == JS_ATOM_NULL) - return -1; - idx = add_req_module_entry(ctx, m, module_name); - JS_FreeAtom(ctx, module_name); + idx = js_parse_from_clause(s, m); if (idx < 0) return -1; for(i = first_export; i < m->export_entries_count; i++) { @@ -29866,11 +31045,7 @@ static __exception int js_parse_export(JSParseState *s) export_name = JS_DupAtom(ctx, s->token.u.ident.atom); if (next_token(s)) goto fail1; - module_name = js_parse_from_clause(s); - if (module_name == JS_ATOM_NULL) - goto fail1; - idx = add_req_module_entry(ctx, m, module_name); - JS_FreeAtom(ctx, module_name); + idx = js_parse_from_clause(s, m); if (idx < 0) goto fail1; me = add_export_entry(s, m, JS_ATOM__star_, export_name, @@ -29880,11 +31055,7 @@ static __exception int js_parse_export(JSParseState *s) return -1; me->u.req_module_idx = idx; } else { - module_name = js_parse_from_clause(s); - if (module_name == JS_ATOM_NULL) - return -1; - idx = add_req_module_entry(ctx, m, module_name); - JS_FreeAtom(ctx, module_name); + idx = js_parse_from_clause(s, m); if (idx < 0) return -1; if (add_star_export_entry(ctx, m, idx) < 0) @@ -29932,7 +31103,7 @@ static __exception int js_parse_export(JSParseState *s) } static int add_closure_var(JSContext *ctx, JSFunctionDef *s, - BOOL is_local, BOOL is_arg, + JSClosureTypeEnum closure_type, int var_idx, JSAtom var_name, BOOL is_const, BOOL is_lexical, JSVarKindEnum var_kind); @@ -29954,9 +31125,10 @@ static int add_import(JSParseState *s, JSModuleDef *m, } } - var_idx = add_closure_var(ctx, s->cur_func, is_star, FALSE, + var_idx = add_closure_var(ctx, s->cur_func, + is_star ? JS_CLOSURE_MODULE_DECL : JS_CLOSURE_MODULE_IMPORT, m->import_entries_count, - local_name, TRUE, TRUE, FALSE); + local_name, TRUE, TRUE, JS_VAR_NORMAL); if (var_idx < 0) return -1; if (js_resize_array(ctx, (void **)&m->import_entries, @@ -29990,6 +31162,14 @@ static __exception int js_parse_import(JSParseState *s) JS_FreeAtom(ctx, module_name); return -1; } + idx = add_req_module_entry(ctx, m, module_name); + JS_FreeAtom(ctx, module_name); + if (idx < 0) + return -1; + if (s->token.val == TOK_WITH) { + if (js_parse_with_clause(s, &m->req_module_entries[idx])) + return -1; + } } else { if (s->token.val == TOK_IDENT) { if (s->token.u.ident.is_reserved) { @@ -30088,14 +31268,10 @@ static __exception int js_parse_import(JSParseState *s) return -1; } end_import_clause: - module_name = js_parse_from_clause(s); - if (module_name == JS_ATOM_NULL) + idx = js_parse_from_clause(s, m); + if (idx < 0) return -1; } - idx = add_req_module_entry(ctx, m, module_name); - JS_FreeAtom(ctx, module_name); - if (idx < 0) - return -1; for(i = first_import; i < m->import_entries_count; i++) m->import_entries[i].req_module_idx = idx; @@ -30174,7 +31350,7 @@ static JSFunctionDef *js_new_function_def(JSContext *ctx, fd->is_eval = is_eval; fd->is_func_expr = is_func_expr; - js_dbuf_init(ctx, &fd->byte_code); + js_dbuf_bytecode_init(ctx, &fd->byte_code); fd->last_opcode_pos = -1; fd->func_name = JS_ATOM_NULL; fd->var_object_idx = -1; @@ -30232,6 +31408,8 @@ static void free_bytecode_atoms(JSRuntime *rt, case OP_FMT_atom_u16: case OP_FMT_atom_label_u8: case OP_FMT_atom_label_u16: + if ((pos + 1 + 4) > bc_len) + break; /* may happen if there is not enough memory when emiting bytecode */ atom = get_u32(bc_buf + pos + 1); JS_FreeAtomRT(rt, atom); break; @@ -30522,7 +31700,7 @@ static void dump_byte_code(JSContext *ctx, int pass, has_pool_idx: printf(" %u: ", idx); if (idx < cpool_count) { - JS_PrintValue(ctx, stdout, cpool[idx], NULL); + JS_PrintValue(ctx, js_dump_value_write, stdout, cpool[idx], NULL); } break; case OP_FMT_atom: @@ -30714,12 +31892,39 @@ static __maybe_unused void js_dump_function_bytecode(JSContext *ctx, JSFunctionB printf(" closure vars:\n"); for(i = 0; i < b->closure_var_count; i++) { JSClosureVar *cv = &b->closure_var[i]; - printf("%5d: %s %s:%s%d %s\n", i, - JS_AtomGetStr(ctx, atom_buf, sizeof(atom_buf), cv->var_name), - cv->is_local ? "local" : "parent", - cv->is_arg ? "arg" : "loc", cv->var_idx, + printf("%5d: %s %s", i, cv->is_const ? "const" : - cv->is_lexical ? "let" : "var"); + cv->is_lexical ? "let" : "var", + JS_AtomGetStr(ctx, atom_buf, sizeof(atom_buf), cv->var_name)); + switch(cv->closure_type) { + case JS_CLOSURE_LOCAL: + printf(" [loc%d]\n", cv->var_idx); + break; + case JS_CLOSURE_ARG: + printf(" [arg%d]\n", cv->var_idx); + break; + case JS_CLOSURE_REF: + printf(" [ref%d]\n", cv->var_idx); + break; + case JS_CLOSURE_GLOBAL_REF: + printf(" [global_ref%d]\n", cv->var_idx); + break; + case JS_CLOSURE_GLOBAL_DECL: + printf(" [global_decl]\n"); + break; + case JS_CLOSURE_GLOBAL: + printf(" [global]\n"); + break; + case JS_CLOSURE_MODULE_DECL: + printf(" [module_decl]\n"); + break; + case JS_CLOSURE_MODULE_IMPORT: + printf(" [module_import]\n"); + break; + default: + printf(" [?]\n"); + break; + } } } printf(" stack_size: %d\n", b->stack_size); @@ -30740,7 +31945,7 @@ static __maybe_unused void js_dump_function_bytecode(JSContext *ctx, JSFunctionB #endif static int add_closure_var(JSContext *ctx, JSFunctionDef *s, - BOOL is_local, BOOL is_arg, + JSClosureTypeEnum closure_type, int var_idx, JSAtom var_name, BOOL is_const, BOOL is_lexical, JSVarKindEnum var_kind) @@ -30758,8 +31963,7 @@ static int add_closure_var(JSContext *ctx, JSFunctionDef *s, &s->closure_var_size, s->closure_var_count + 1)) return -1; cv = &s->closure_var[s->closure_var_count++]; - cv->is_local = is_local; - cv->is_arg = is_arg; + cv->closure_type = closure_type; cv->is_const = is_const; cv->is_lexical = is_lexical; cv->var_kind = var_kind; @@ -30780,46 +31984,34 @@ static int find_closure_var(JSContext *ctx, JSFunctionDef *s, return -1; } -/* 'fd' must be a parent of 's'. Create in 's' a closure referencing a - local variable (is_local = TRUE) or a closure (is_local = FALSE) in - 'fd' */ -static int get_closure_var2(JSContext *ctx, JSFunctionDef *s, - JSFunctionDef *fd, BOOL is_local, - BOOL is_arg, int var_idx, JSAtom var_name, - BOOL is_const, BOOL is_lexical, - JSVarKindEnum var_kind) +/* 'fd' must be a parent of 's'. Create in 's' a closure referencing + another one in 'fd' */ +static int get_closure_var(JSContext *ctx, JSFunctionDef *s, + JSFunctionDef *fd, JSClosureTypeEnum closure_type, + int var_idx, JSAtom var_name, + BOOL is_const, BOOL is_lexical, + JSVarKindEnum var_kind) { int i; if (fd != s->parent) { - var_idx = get_closure_var2(ctx, s->parent, fd, is_local, - is_arg, var_idx, var_name, - is_const, is_lexical, var_kind); + var_idx = get_closure_var(ctx, s->parent, fd, closure_type, + var_idx, var_name, + is_const, is_lexical, var_kind); if (var_idx < 0) return -1; - is_local = FALSE; + if (closure_type != JS_CLOSURE_GLOBAL_REF) + closure_type = JS_CLOSURE_REF; } for(i = 0; i < s->closure_var_count; i++) { JSClosureVar *cv = &s->closure_var[i]; - if (cv->var_idx == var_idx && cv->is_arg == is_arg && - cv->is_local == is_local) + if (cv->var_idx == var_idx && cv->closure_type == closure_type) return i; } - return add_closure_var(ctx, s, is_local, is_arg, var_idx, var_name, + return add_closure_var(ctx, s, closure_type, var_idx, var_name, is_const, is_lexical, var_kind); } -static int get_closure_var(JSContext *ctx, JSFunctionDef *s, - JSFunctionDef *fd, BOOL is_arg, - int var_idx, JSAtom var_name, - BOOL is_const, BOOL is_lexical, - JSVarKindEnum var_kind) -{ - return get_closure_var2(ctx, s, fd, TRUE, is_arg, - var_idx, var_name, is_const, is_lexical, - var_kind); -} - static int get_with_scope_opcode(int op) { if (op == OP_scope_get_var_undef) @@ -30892,75 +32084,6 @@ static int optimize_scope_make_ref(JSContext *ctx, JSFunctionDef *s, return pos_next; } -static int optimize_scope_make_global_ref(JSContext *ctx, JSFunctionDef *s, - DynBuf *bc, uint8_t *bc_buf, - LabelSlot *ls, int pos_next, - JSAtom var_name) -{ - int label_pos, end_pos, pos, op; - BOOL is_strict; - is_strict = ((s->js_mode & JS_MODE_STRICT) != 0); - - /* replace the reference get/put with normal variable - accesses */ - if (is_strict) { - /* need to check if the variable exists before evaluating the right - expression */ - /* XXX: need an extra OP_true if destructuring an array */ - dbuf_putc(bc, OP_check_var); - dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); - } else { - /* XXX: need 2 extra OP_true if destructuring an array */ - } - if (bc_buf[pos_next] == OP_get_ref_value) { - dbuf_putc(bc, OP_get_var); - dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); - pos_next++; - } - /* remove the OP_label to make room for replacement */ - /* label should have a refcount of 0 anyway */ - /* XXX: should have emitted several OP_nop to avoid this kludge */ - label_pos = ls->pos; - pos = label_pos - 5; - assert(bc_buf[pos] == OP_label); - end_pos = label_pos + 2; - op = bc_buf[label_pos]; - if (is_strict) { - if (op != OP_nop) { - switch(op) { - case OP_insert3: - op = OP_insert2; - break; - case OP_perm4: - op = OP_perm3; - break; - case OP_rot3l: - op = OP_swap; - break; - default: - abort(); - } - bc_buf[pos++] = op; - } - } else { - if (op == OP_insert3) - bc_buf[pos++] = OP_dup; - } - if (is_strict) { - bc_buf[pos] = OP_put_var_strict; - /* XXX: need 1 extra OP_drop if destructuring an array */ - } else { - bc_buf[pos] = OP_put_var; - /* XXX: need 2 extra OP_drop if destructuring an array */ - } - put_u32(bc_buf + pos + 1, JS_DupAtom(ctx, var_name)); - pos += 5; - /* pad with OP_nop */ - while (pos < end_pos) - bc_buf[pos++] = OP_nop; - return pos_next; -} - static int add_var_this(JSContext *ctx, JSFunctionDef *fd) { int idx; @@ -31021,14 +32144,20 @@ static void var_object_test(JSContext *ctx, JSFunctionDef *s, { dbuf_putc(bc, get_with_scope_opcode(op)); dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); - *plabel_done = new_label_fd(s, *plabel_done); + if (*plabel_done < 0) { + *plabel_done = new_label_fd(s); + if (*plabel_done < 0) { + dbuf_set_error(bc); + return; + } + } dbuf_put_u32(bc, *plabel_done); dbuf_putc(bc, is_with); update_label(s, *plabel_done, 1); s->jump_size++; } -/* return the position of the next opcode */ +/* return the position of the next opcode or -1 if error */ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, JSAtom var_name, int scope_level, int op, DynBuf *bc, uint8_t *bc_buf, @@ -31147,14 +32276,22 @@ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, } } break; + case OP_scope_put_var: + if (!(var_idx & ARGUMENT_VAR_OFFSET) && + s->vars[var_idx].var_kind == JS_VAR_FUNCTION_NAME) { + /* in non strict mode, modifying the function name is ignored */ + dbuf_putc(bc, OP_drop); + goto done; + } + goto local_scope_var; case OP_scope_get_ref: dbuf_putc(bc, OP_undefined); - /* fall thru */ + goto local_scope_var; case OP_scope_get_var_checkthis: case OP_scope_get_var_undef: case OP_scope_get_var: - case OP_scope_put_var: case OP_scope_put_var_init: + local_scope_var: is_put = (op == OP_scope_put_var || op == OP_scope_put_var_init); if (var_idx & ARGUMENT_VAR_OFFSET) { dbuf_putc(bc, OP_get_arg + is_put); @@ -31227,7 +32364,7 @@ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, break; } else if (vd->var_name == JS_ATOM__with_ && !is_pseudo_var) { vd->is_captured = 1; - idx = get_closure_var(ctx, s, fd, FALSE, idx, vd->var_name, FALSE, FALSE, JS_VAR_NORMAL); + idx = get_closure_var(ctx, s, fd, JS_CLOSURE_LOCAL, idx, vd->var_name, FALSE, FALSE, JS_VAR_NORMAL); if (idx >= 0) { dbuf_putc(bc, OP_get_var_ref); dbuf_put_u16(bc, idx); @@ -31264,7 +32401,7 @@ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, if (!is_arg_scope && fd->var_object_idx >= 0 && !is_pseudo_var) { vd = &fd->vars[fd->var_object_idx]; vd->is_captured = 1; - idx = get_closure_var(ctx, s, fd, FALSE, + idx = get_closure_var(ctx, s, fd, JS_CLOSURE_LOCAL, fd->var_object_idx, vd->var_name, FALSE, FALSE, JS_VAR_NORMAL); dbuf_putc(bc, OP_get_var_ref); @@ -31276,7 +32413,7 @@ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, if (fd->arg_var_object_idx >= 0 && !is_pseudo_var) { vd = &fd->vars[fd->arg_var_object_idx]; vd->is_captured = 1; - idx = get_closure_var(ctx, s, fd, FALSE, + idx = get_closure_var(ctx, s, fd, JS_CLOSURE_LOCAL, fd->arg_var_object_idx, vd->var_name, FALSE, FALSE, JS_VAR_NORMAL); dbuf_putc(bc, OP_get_var_ref); @@ -31298,25 +32435,37 @@ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, JSClosureVar *cv = &fd->closure_var[idx1]; if (var_name == cv->var_name) { if (fd != s) { - idx = get_closure_var2(ctx, s, fd, - FALSE, - cv->is_arg, idx1, - cv->var_name, cv->is_const, - cv->is_lexical, cv->var_kind); + JSClosureTypeEnum closure_type; + if (cv->closure_type == JS_CLOSURE_GLOBAL || + cv->closure_type == JS_CLOSURE_GLOBAL_DECL || + cv->closure_type == JS_CLOSURE_GLOBAL_REF) + closure_type = JS_CLOSURE_GLOBAL_REF; + else + closure_type = JS_CLOSURE_REF; + idx = get_closure_var(ctx, s, fd, + closure_type, + idx1, + cv->var_name, cv->is_const, + cv->is_lexical, cv->var_kind); } else { idx = idx1; } - goto has_idx; + if (cv->closure_type == JS_CLOSURE_GLOBAL || + cv->closure_type == JS_CLOSURE_GLOBAL_DECL || + cv->closure_type == JS_CLOSURE_GLOBAL_REF) + goto has_global_idx; + else + goto has_idx; } else if ((cv->var_name == JS_ATOM__var_ || cv->var_name == JS_ATOM__arg_var_ || cv->var_name == JS_ATOM__with_) && !is_pseudo_var) { int is_with = (cv->var_name == JS_ATOM__with_); if (fd != s) { - idx = get_closure_var2(ctx, s, fd, - FALSE, - cv->is_arg, idx1, - cv->var_name, FALSE, FALSE, - JS_VAR_NORMAL); + idx = get_closure_var(ctx, s, fd, + JS_CLOSURE_REF, + idx1, + cv->var_name, FALSE, FALSE, + JS_VAR_NORMAL); } else { idx = idx1; } @@ -31325,19 +32474,67 @@ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, var_object_test(ctx, s, var_name, op, bc, &label_done, is_with); } } - } - if (var_idx >= 0) { + /* not found: add a closure for a global variable access */ + idx1 = add_closure_var(ctx, fd, JS_CLOSURE_GLOBAL, 0, var_name, + FALSE, FALSE, JS_VAR_NORMAL); + if (idx1 < 0) + return -1; + if (fd != s) { + idx = get_closure_var(ctx, s, fd, + JS_CLOSURE_GLOBAL_REF, + idx1, + var_name, FALSE, FALSE, + JS_VAR_NORMAL); + } else { + idx = idx1; + } + has_global_idx: + /* global variable access */ + switch (op) { + case OP_scope_make_ref: + if (label_done == -1 && can_opt_put_global_ref_value(bc_buf, ls->pos)) { + pos_next = optimize_scope_make_ref(ctx, s, bc, bc_buf, ls, + pos_next, + OP_get_var, idx); + } else { + dbuf_putc(bc, OP_make_var_ref); + dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); + } + break; + case OP_scope_get_ref: + /* XXX: should create a dummy object with a named slot that is + a reference to the global variable */ + dbuf_putc(bc, OP_undefined); + dbuf_putc(bc, OP_get_var); + dbuf_put_u16(bc, idx); + break; + case OP_scope_get_var_undef: + case OP_scope_get_var: + case OP_scope_put_var: + dbuf_putc(bc, OP_get_var_undef + (op - OP_scope_get_var_undef)); + dbuf_put_u16(bc, idx); + break; + case OP_scope_put_var_init: + dbuf_putc(bc, OP_put_var_init); + dbuf_put_u16(bc, idx); + break; + case OP_scope_delete_var: + dbuf_putc(bc, OP_delete_var); + dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); + break; + } + } else { /* find the corresponding closure variable */ if (var_idx & ARGUMENT_VAR_OFFSET) { fd->args[var_idx - ARGUMENT_VAR_OFFSET].is_captured = 1; idx = get_closure_var(ctx, s, fd, - TRUE, var_idx - ARGUMENT_VAR_OFFSET, + JS_CLOSURE_ARG, var_idx - ARGUMENT_VAR_OFFSET, var_name, FALSE, FALSE, JS_VAR_NORMAL); } else { fd->vars[var_idx].is_captured = 1; idx = get_closure_var(ctx, s, fd, - FALSE, var_idx, + JS_CLOSURE_LOCAL, var_idx, var_name, fd->vars[var_idx].is_const, fd->vars[var_idx].is_lexical, @@ -31382,15 +32579,22 @@ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, dbuf_put_u16(bc, idx); } break; + case OP_scope_put_var: + if (s->closure_var[idx].var_kind == JS_VAR_FUNCTION_NAME) { + /* in non strict mode, modifying the function name is ignored */ + dbuf_putc(bc, OP_drop); + goto done; + } + goto closure_scope_var; case OP_scope_get_ref: /* XXX: should create a dummy object with a named slot that is a reference to the closure variable */ dbuf_putc(bc, OP_undefined); - /* fall thru */ + goto closure_scope_var; case OP_scope_get_var_undef: case OP_scope_get_var: - case OP_scope_put_var: case OP_scope_put_var_init: + closure_scope_var: is_put = (op == OP_scope_put_var || op == OP_scope_put_var_init); if (is_put) { @@ -31424,40 +32628,6 @@ static int resolve_scope_var(JSContext *ctx, JSFunctionDef *s, } } - /* global variable access */ - - switch (op) { - case OP_scope_make_ref: - if (label_done == -1 && can_opt_put_global_ref_value(bc_buf, ls->pos)) { - pos_next = optimize_scope_make_global_ref(ctx, s, bc, bc_buf, ls, - pos_next, var_name); - } else { - dbuf_putc(bc, OP_make_var_ref); - dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); - } - break; - case OP_scope_get_ref: - /* XXX: should create a dummy object with a named slot that is - a reference to the global variable */ - dbuf_putc(bc, OP_undefined); - dbuf_putc(bc, OP_get_var); - dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); - break; - case OP_scope_get_var_undef: - case OP_scope_get_var: - case OP_scope_put_var: - dbuf_putc(bc, OP_get_var_undef + (op - OP_scope_get_var_undef)); - dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); - break; - case OP_scope_put_var_init: - dbuf_putc(bc, OP_put_var_init); - dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); - break; - case OP_scope_delete_var: - dbuf_putc(bc, OP_delete_var); - dbuf_put_u32(bc, JS_DupAtom(ctx, var_name)); - break; - } done: if (label_done >= 0) { dbuf_putc(bc, OP_label); @@ -31509,7 +32679,7 @@ static int resolve_scope_private_field1(JSContext *ctx, if (idx >= 0) { var_kind = fd->vars[idx].var_kind; if (is_ref) { - idx = get_closure_var(ctx, s, fd, FALSE, idx, var_name, + idx = get_closure_var(ctx, s, fd, JS_CLOSURE_LOCAL, idx, var_name, TRUE, TRUE, JS_VAR_NORMAL); if (idx < 0) return -1; @@ -31526,12 +32696,12 @@ static int resolve_scope_private_field1(JSContext *ctx, var_kind = cv->var_kind; is_ref = TRUE; if (fd != s) { - idx = get_closure_var2(ctx, s, fd, - FALSE, - cv->is_arg, idx, - cv->var_name, cv->is_const, - cv->is_lexical, - cv->var_kind); + idx = get_closure_var(ctx, s, fd, + JS_CLOSURE_REF, + idx, + cv->var_name, cv->is_const, + cv->is_lexical, + cv->var_kind); if (idx < 0) return -1; } @@ -31760,7 +32930,7 @@ static void add_eval_variables(JSContext *ctx, JSFunctionDef *s) while (scope_idx >= 0) { vd = &fd->vars[scope_idx]; vd->is_captured = 1; - get_closure_var(ctx, s, fd, FALSE, scope_idx, + get_closure_var(ctx, s, fd, JS_CLOSURE_LOCAL, scope_idx, vd->var_name, vd->is_const, vd->is_lexical, vd->var_kind); scope_idx = vd->scope_next; } @@ -31772,7 +32942,7 @@ static void add_eval_variables(JSContext *ctx, JSFunctionDef *s) vd = &fd->args[i]; if (vd->var_name != JS_ATOM_NULL) { get_closure_var(ctx, s, fd, - TRUE, i, vd->var_name, FALSE, + JS_CLOSURE_ARG, i, vd->var_name, FALSE, vd->is_lexical, JS_VAR_NORMAL); } } @@ -31783,7 +32953,7 @@ static void add_eval_variables(JSContext *ctx, JSFunctionDef *s) vd->var_name != JS_ATOM__ret_ && vd->var_name != JS_ATOM_NULL) { get_closure_var(ctx, s, fd, - FALSE, i, vd->var_name, FALSE, + JS_CLOSURE_LOCAL, i, vd->var_name, FALSE, vd->is_lexical, JS_VAR_NORMAL); } } @@ -31793,7 +32963,7 @@ static void add_eval_variables(JSContext *ctx, JSFunctionDef *s) /* do not close top level last result */ if (vd->scope_level == 0 && is_var_in_arg_scope(vd)) { get_closure_var(ctx, s, fd, - FALSE, i, vd->var_name, FALSE, + JS_CLOSURE_LOCAL, i, vd->var_name, FALSE, vd->is_lexical, JS_VAR_NORMAL); } } @@ -31801,13 +32971,19 @@ static void add_eval_variables(JSContext *ctx, JSFunctionDef *s) if (fd->is_eval) { int idx; /* add direct eval variables (we are necessarily at the - top level) */ + top level). */ for (idx = 0; idx < fd->closure_var_count; idx++) { JSClosureVar *cv = &fd->closure_var[idx]; - get_closure_var2(ctx, s, fd, - FALSE, cv->is_arg, - idx, cv->var_name, cv->is_const, - cv->is_lexical, cv->var_kind); + /* Global variables are removed but module + definitions are kept. */ + if (cv->closure_type != JS_CLOSURE_GLOBAL_REF && + cv->closure_type != JS_CLOSURE_GLOBAL_DECL && + cv->closure_type != JS_CLOSURE_GLOBAL) { + get_closure_var(ctx, s, fd, + JS_CLOSURE_REF, + idx, cv->var_name, cv->is_const, + cv->is_lexical, cv->var_kind); + } } } } @@ -31816,8 +32992,7 @@ static void add_eval_variables(JSContext *ctx, JSFunctionDef *s) static void set_closure_from_var(JSContext *ctx, JSClosureVar *cv, JSVarDef *vd, int var_idx) { - cv->is_local = TRUE; - cv->is_arg = FALSE; + cv->closure_type = JS_CLOSURE_LOCAL; cv->is_const = vd->is_const; cv->is_lexical = vd->is_lexical; cv->var_kind = vd->var_kind; @@ -31858,8 +33033,7 @@ static __exception int add_closure_variables(JSContext *ctx, JSFunctionDef *s, for(i = 0; i < b->arg_count; i++) { JSClosureVar *cv = &s->closure_var[s->closure_var_count++]; vd = &b->vardefs[i]; - cv->is_local = TRUE; - cv->is_arg = TRUE; + cv->closure_type = JS_CLOSURE_ARG; cv->is_const = FALSE; cv->is_lexical = FALSE; cv->var_kind = JS_VAR_NORMAL; @@ -31886,9 +33060,24 @@ static __exception int add_closure_variables(JSContext *ctx, JSFunctionDef *s, } for(i = 0; i < b->closure_var_count; i++) { JSClosureVar *cv0 = &b->closure_var[i]; - JSClosureVar *cv = &s->closure_var[s->closure_var_count++]; - cv->is_local = FALSE; - cv->is_arg = cv0->is_arg; + JSClosureVar *cv; + + switch(cv0->closure_type) { + case JS_CLOSURE_LOCAL: + case JS_CLOSURE_ARG: + case JS_CLOSURE_REF: + case JS_CLOSURE_MODULE_DECL: + case JS_CLOSURE_MODULE_IMPORT: + break; + case JS_CLOSURE_GLOBAL_REF: + case JS_CLOSURE_GLOBAL_DECL: + case JS_CLOSURE_GLOBAL: + continue; /* not necessary to add global variables */ + default: + abort(); + } + cv = &s->closure_var[s->closure_var_count++]; + cv->closure_type = JS_CLOSURE_REF; cv->is_const = cv0->is_const; cv->is_lexical = cv0->is_lexical; cv->var_kind = cv0->var_kind; @@ -32066,8 +33255,11 @@ static void instantiate_hoisted_definitions(JSContext *ctx, JSFunctionDef *s, Dy evaluating the module so that the exported functions are visible if there are cyclic module references */ if (s->module) { - label_next = new_label_fd(s, -1); - + label_next = new_label_fd(s); + if (label_next < 0) { + dbuf_set_error(bc); + return; + } /* if 'this' is true, initialize the global variables and return */ dbuf_putc(bc, OP_push_this); dbuf_putc(bc, OP_if_false); @@ -32078,9 +33270,10 @@ static void instantiate_hoisted_definitions(JSContext *ctx, JSFunctionDef *s, Dy /* add the global variables (only happens if s->is_global_var is true) */ + /* XXX: inefficient, add a closure index in JSGlobalVar */ for(i = 0; i < s->global_var_count; i++) { JSGlobalVar *hf = &s->global_vars[i]; - int has_closure = 0; + BOOL has_var_obj = FALSE; BOOL force_init = hf->force_init; /* we are in an eval, so the closure contains all the enclosing variables */ @@ -32089,46 +33282,20 @@ static void instantiate_hoisted_definitions(JSContext *ctx, JSFunctionDef *s, Dy for(idx = 0; idx < s->closure_var_count; idx++) { JSClosureVar *cv = &s->closure_var[idx]; if (cv->var_name == hf->var_name) { - has_closure = 2; force_init = FALSE; - break; + goto closure_found; } if (cv->var_name == JS_ATOM__var_ || cv->var_name == JS_ATOM__arg_var_) { dbuf_putc(bc, OP_get_var_ref); dbuf_put_u16(bc, idx); - has_closure = 1; + has_var_obj = TRUE; force_init = TRUE; - break; - } - } - if (!has_closure) { - int flags; - - flags = 0; - if (s->eval_type != JS_EVAL_TYPE_GLOBAL) - flags |= JS_PROP_CONFIGURABLE; - if (hf->cpool_idx >= 0 && !hf->is_lexical) { - /* global function definitions need a specific handling */ - dbuf_putc(bc, OP_fclosure); - dbuf_put_u32(bc, hf->cpool_idx); - - dbuf_putc(bc, OP_define_func); - dbuf_put_u32(bc, JS_DupAtom(ctx, hf->var_name)); - dbuf_putc(bc, flags); - - goto done_global_var; - } else { - if (hf->is_lexical) { - flags |= DEFINE_GLOBAL_LEX_VAR; - if (!hf->is_const) - flags |= JS_PROP_WRITABLE; - } - dbuf_putc(bc, OP_define_var); - dbuf_put_u32(bc, JS_DupAtom(ctx, hf->var_name)); - dbuf_putc(bc, flags); + goto closure_found; } } + abort(); + closure_found: if (hf->cpool_idx >= 0 || force_init) { if (hf->cpool_idx >= 0) { dbuf_putc(bc, OP_fclosure); @@ -32141,20 +33308,15 @@ static void instantiate_hoisted_definitions(JSContext *ctx, JSFunctionDef *s, Dy } else { dbuf_putc(bc, OP_undefined); } - if (has_closure == 2) { + if (!has_var_obj) { dbuf_putc(bc, OP_put_var_ref); dbuf_put_u16(bc, idx); - } else if (has_closure == 1) { + } else { dbuf_putc(bc, OP_define_field); dbuf_put_u32(bc, JS_DupAtom(ctx, hf->var_name)); dbuf_putc(bc, OP_drop); - } else { - /* XXX: Check if variable is writable and enumerable */ - dbuf_putc(bc, OP_put_var); - dbuf_put_u32(bc, JS_DupAtom(ctx, hf->var_name)); } } - done_global_var: JS_FreeAtom(ctx, hf->var_name); } @@ -32249,7 +33411,7 @@ static int get_label_pos(JSFunctionDef *s, int label) variables when necessary */ static __exception int resolve_variables(JSContext *ctx, JSFunctionDef *s) { - int pos, pos_next, bc_len, op, len, i, idx, line_num; + int pos, pos_next, bc_len, op, len, line_num, i, idx; uint8_t *bc_buf; JSAtom var_name; DynBuf bc_out; @@ -32258,17 +33420,23 @@ static __exception int resolve_variables(JSContext *ctx, JSFunctionDef *s) cc.bc_buf = bc_buf = s->byte_code.buf; cc.bc_len = bc_len = s->byte_code.size; - js_dbuf_init(ctx, &bc_out); + js_dbuf_bytecode_init(ctx, &bc_out); /* first pass for runtime checks (must be done before the variables are created) */ + /* XXX: inefficient */ for(i = 0; i < s->global_var_count; i++) { JSGlobalVar *hf = &s->global_vars[i]; - int flags; /* check if global variable (XXX: simplify) */ for(idx = 0; idx < s->closure_var_count; idx++) { JSClosureVar *cv = &s->closure_var[idx]; + if (cv->closure_type == JS_CLOSURE_GLOBAL_REF || + cv->closure_type == JS_CLOSURE_GLOBAL_DECL || + cv->closure_type == JS_CLOSURE_GLOBAL || + cv->closure_type == JS_CLOSURE_MODULE_DECL || + cv->closure_type == JS_CLOSURE_MODULE_IMPORT) + goto next; /* don't look at global variables (they are at the end) */ if (cv->var_name == hf->var_name) { if (s->eval_type == JS_EVAL_TYPE_DIRECT && cv->is_lexical) { @@ -32287,15 +33455,6 @@ static __exception int resolve_variables(JSContext *ctx, JSFunctionDef *s) cv->var_name == JS_ATOM__arg_var_) goto next; } - - dbuf_putc(&bc_out, OP_check_define_var); - dbuf_put_u32(&bc_out, JS_DupAtom(ctx, hf->var_name)); - flags = 0; - if (hf->is_lexical) - flags |= DEFINE_GLOBAL_LEX_VAR; - if (hf->cpool_idx >= 0) - flags |= DEFINE_GLOBAL_FUNC_VAR; - dbuf_putc(&bc_out, flags); next: ; } @@ -32878,7 +34037,7 @@ static __exception int resolve_labels(JSContext *ctx, JSFunctionDef *s) cc.bc_buf = bc_buf = s->byte_code.buf; cc.bc_len = bc_len = s->byte_code.size; - js_dbuf_init(ctx, &bc_out); + js_dbuf_bytecode_init(ctx, &bc_out); #if SHORT_OPCODES if (s->jump_size) { @@ -33414,12 +34573,11 @@ static __exception int resolve_labels(JSContext *ctx, JSFunctionDef *s) if (OPTIMIZE) { /* Transformation: insert2 put_field(a) drop -> put_field(a) - insert2 put_var_strict(a) drop -> put_var_strict(a) */ - if (code_match(&cc, pos_next, M2(OP_put_field, OP_put_var_strict), OP_drop, -1)) { + if (code_match(&cc, pos_next, OP_put_field, OP_drop, -1)) { if (cc.line_num >= 0) line_num = cc.line_num; add_pc2line_info(s, bc_out.size, line_num); - dbuf_putc(&bc_out, cc.op); + dbuf_putc(&bc_out, OP_put_field); dbuf_put_u32(&bc_out, cc.atom); pos_next = cc.pos; break; @@ -33565,7 +34723,6 @@ static __exception int resolve_labels(JSContext *ctx, JSFunctionDef *s) /* transformation: post_inc put_x drop -> inc put_x post_inc perm3 put_field drop -> inc put_field - post_inc perm3 put_var_strict drop -> inc put_var_strict post_inc perm4 put_array_el drop -> inc put_array_el */ int op1, idx; @@ -33584,11 +34741,11 @@ static __exception int resolve_labels(JSContext *ctx, JSFunctionDef *s) put_short_code(&bc_out, op1, idx); break; } - if (code_match(&cc, pos_next, OP_perm3, M2(OP_put_field, OP_put_var_strict), OP_drop, -1)) { + if (code_match(&cc, pos_next, OP_perm3, OP_put_field, OP_drop, -1)) { if (cc.line_num >= 0) line_num = cc.line_num; add_pc2line_info(s, bc_out.size, line_num); dbuf_putc(&bc_out, OP_dec + (op - OP_post_dec)); - dbuf_putc(&bc_out, cc.op); + dbuf_putc(&bc_out, OP_put_field); dbuf_put_u32(&bc_out, cc.atom); pos_next = cc.pos; break; @@ -34013,35 +35170,68 @@ static __exception int compute_stack_size(JSContext *ctx, return -1; } -static int add_module_variables(JSContext *ctx, JSFunctionDef *fd) +static int add_global_variables(JSContext *ctx, JSFunctionDef *fd) { int i, idx; JSModuleDef *m = fd->module; JSExportEntry *me; JSGlobalVar *hf; + BOOL need_global_closures; + + /* Script: add the defined global variables. In the non strict + direct eval not in global scope, the global variables are + created in the enclosing scope so they are not created as + variable references. - /* The imported global variables were added as closure variables - in js_parse_import(). We add here the module global - variables. */ - - for(i = 0; i < fd->global_var_count; i++) { - hf = &fd->global_vars[i]; - if (add_closure_var(ctx, fd, TRUE, FALSE, i, hf->var_name, hf->is_const, - hf->is_lexical, FALSE) < 0) - return -1; + In modules, the imported global variables were added as closure + global variables in js_parse_import(). + */ + need_global_closures = TRUE; + if (fd->eval_type == JS_EVAL_TYPE_DIRECT && !(fd->js_mode & JS_MODE_STRICT)) { + /* XXX: add a flag ? */ + for(idx = 0; idx < fd->closure_var_count; idx++) { + JSClosureVar *cv = &fd->closure_var[idx]; + if (cv->var_name == JS_ATOM__var_ || + cv->var_name == JS_ATOM__arg_var_) { + need_global_closures = FALSE; + break; + } + } } - /* resolve the variable names of the local exports */ - for(i = 0; i < m->export_entries_count; i++) { - me = &m->export_entries[i]; - if (me->export_type == JS_EXPORT_TYPE_LOCAL) { - idx = find_closure_var(ctx, fd, me->local_name); - if (idx < 0) { - JS_ThrowSyntaxErrorAtom(ctx, "exported variable '%s' does not exist", - me->local_name); + if (need_global_closures) { + JSClosureTypeEnum closure_type; + if (fd->module) + closure_type = JS_CLOSURE_MODULE_DECL; + else + closure_type = JS_CLOSURE_GLOBAL_DECL; + for(i = 0; i < fd->global_var_count; i++) { + JSVarKindEnum var_kind; + hf = &fd->global_vars[i]; + if (hf->cpool_idx >= 0 && !hf->is_lexical) { + var_kind = JS_VAR_GLOBAL_FUNCTION_DECL; + } else { + var_kind = JS_VAR_NORMAL; + } + if (add_closure_var(ctx, fd, closure_type, i, hf->var_name, hf->is_const, + hf->is_lexical, var_kind) < 0) return -1; + } + } + + if (fd->module) { + /* resolve the variable names of the local exports */ + for(i = 0; i < m->export_entries_count; i++) { + me = &m->export_entries[i]; + if (me->export_type == JS_EXPORT_TYPE_LOCAL) { + idx = find_closure_var(ctx, fd, me->local_name); + if (idx < 0) { + JS_ThrowSyntaxErrorAtom(ctx, "exported variable '%s' does not exist", + me->local_name); + return -1; + } + me->u.local.var_idx = idx; } - me->u.local.var_idx = idx; } } return 0; @@ -34093,10 +35283,10 @@ static JSValue js_create_function(JSContext *ctx, JSFunctionDef *fd) add_eval_variables(ctx, fd); /* add the module global variables in the closure */ - if (fd->module) { - if (add_module_variables(ctx, fd)) + if (fd->is_eval) { + if (add_global_variables(ctx, fd)) goto fail; - } + } /* first create all the child functions */ list_for_each_safe(el, el1, &fd->child_list) { @@ -34286,7 +35476,8 @@ static void free_function_bytecode(JSRuntime *rt, JSFunctionBytecode *b) JS_AtomGetStrRT(rt, buf, sizeof(buf), b->func_name)); } #endif - free_bytecode_atoms(rt, b->byte_code_buf, b->byte_code_len, TRUE); + if (b->byte_code_buf) + free_bytecode_atoms(rt, b->byte_code_buf, b->byte_code_len, TRUE); if (b->vardefs) { for(i = 0; i < b->arg_count + b->var_count; i++) { @@ -34878,7 +36069,7 @@ static __exception int js_parse_function_decl2(JSParseState *s, push_scope(s); /* enter body scope */ fd->body_scope = fd->scope_level; - if (s->token.val == TOK_ARROW) { + if (s->token.val == TOK_ARROW && func_type == JS_PARSE_FUNC_ARROW) { if (next_token(s)) goto fail; @@ -35148,7 +36339,9 @@ static JSValue JS_EvalFunctionInternal(JSContext *ctx, JSValue fun_obj, tag = JS_VALUE_GET_TAG(fun_obj); if (tag == JS_TAG_FUNCTION_BYTECODE) { - fun_obj = js_closure(ctx, fun_obj, var_refs, sf); + fun_obj = js_closure(ctx, fun_obj, var_refs, sf, TRUE); + if (JS_IsException(fun_obj)) + return JS_EXCEPTION; ret_val = JS_CallFree(ctx, fun_obj, this_obj, 0, NULL); } else if (tag == JS_TAG_MODULE) { JSModuleDef *m; @@ -35288,7 +36481,7 @@ static JSValue __JS_EvalInternal(JSContext *ctx, JSValueConst this_obj, fail1: /* XXX: should free all the unresolved dependencies */ if (m) - js_free_module_def(ctx, m); + JS_FreeValue(ctx, JS_MKPTR(JS_TAG_MODULE, m)); return JS_EXCEPTION; } @@ -35872,13 +37065,12 @@ static int JS_WriteFunctionTag(BCWriterState *s, JSValueConst obj) bc_put_atom(s, cv->var_name); bc_put_leb128(s, cv->var_idx); flags = idx = 0; - bc_set_flags(&flags, &idx, cv->is_local, 1); - bc_set_flags(&flags, &idx, cv->is_arg, 1); + bc_set_flags(&flags, &idx, cv->closure_type, 3); bc_set_flags(&flags, &idx, cv->is_const, 1); bc_set_flags(&flags, &idx, cv->is_lexical, 1); bc_set_flags(&flags, &idx, cv->var_kind, 4); - assert(idx <= 8); - bc_put_u8(s, flags); + assert(idx <= 16); + bc_put_u16(s, flags); } if (JS_WriteFunctionBytecode(s, b->byte_code_buf, b->byte_code_len)) @@ -35917,6 +37109,8 @@ static int JS_WriteModule(BCWriterState *s, JSValueConst obj) for(i = 0; i < m->req_module_entries_count; i++) { JSReqModuleEntry *rme = &m->req_module_entries[i]; bc_put_atom(s, rme->module_name); + if (JS_WriteObjectRec(s, rme->attributes)) + goto fail; } bc_put_leb128(s, m->export_entries_count); @@ -36062,6 +37256,7 @@ static int JS_WriteArrayBuffer(BCWriterState *s, JSValueConst obj) } bc_put_u8(s, BC_TAG_ARRAY_BUFFER); bc_put_leb128(s, abuf->byte_length); + bc_put_leb128(s, abuf->max_byte_length); dbuf_put(&s->dbuf, abuf->data, abuf->byte_length); return 0; } @@ -36073,6 +37268,7 @@ static int JS_WriteSharedArrayBuffer(BCWriterState *s, JSValueConst obj) assert(!abuf->detached); /* SharedArrayBuffer are never detached */ bc_put_u8(s, BC_TAG_SHARED_ARRAY_BUFFER); bc_put_leb128(s, abuf->byte_length); + bc_put_leb128(s, abuf->max_byte_length); bc_put_u64(s, (uintptr_t)abuf->data); if (js_resize_array(s->ctx, (void **)&s->sab_tab, sizeof(s->sab_tab[0]), &s->sab_tab_size, s->sab_tab_len + 1)) @@ -36244,7 +37440,7 @@ static int JS_WriteObjectAtoms(BCWriterState *s) /* XXX: could just append dbuf1 data, but it uses more memory if dbuf1 is larger than dbuf */ atoms_size = s->dbuf.size; - if (dbuf_realloc(&dbuf1, dbuf1.size + atoms_size)) + if (dbuf_claim(&dbuf1, atoms_size)) goto fail; memmove(dbuf1.buf + atoms_size, dbuf1.buf, dbuf1.size); memcpy(dbuf1.buf, s->dbuf.buf, atoms_size); @@ -36822,14 +38018,13 @@ static JSValue JS_ReadFunctionTag(BCReaderState *s) if (bc_get_leb128_int(s, &var_idx)) goto fail; cv->var_idx = var_idx; - if (bc_get_u8(s, &v8)) + if (bc_get_u16(s, &v16)) goto fail; idx = 0; - cv->is_local = bc_get_flags(v8, &idx, 1); - cv->is_arg = bc_get_flags(v8, &idx, 1); - cv->is_const = bc_get_flags(v8, &idx, 1); - cv->is_lexical = bc_get_flags(v8, &idx, 1); - cv->var_kind = bc_get_flags(v8, &idx, 4); + cv->closure_type = bc_get_flags(v16, &idx, 3); + cv->is_const = bc_get_flags(v16, &idx, 1); + cv->is_lexical = bc_get_flags(v16, &idx, 1); + cv->var_kind = bc_get_flags(v16, &idx, 4); #ifdef DUMP_READ_OBJECT bc_read_trace(s, "name: "); print_atom(s->ctx, cv->var_name); printf("\n"); #endif @@ -36916,8 +38111,13 @@ static JSValue JS_ReadModule(BCReaderState *s) goto fail; for(i = 0; i < m->req_module_entries_count; i++) { JSReqModuleEntry *rme = &m->req_module_entries[i]; + JSValue val; if (bc_get_atom(s, &rme->module_name)) goto fail; + val = JS_ReadObjectRec(s); + if (JS_IsException(val)) + goto fail; + rme->attributes = val; } } @@ -36993,7 +38193,7 @@ static JSValue JS_ReadModule(BCReaderState *s) return obj; fail: if (m) { - js_free_module_def(ctx, m); + JS_FreeValue(ctx, JS_MKPTR(JS_TAG_MODULE, m)); } return JS_EXCEPTION; } @@ -37129,16 +38329,31 @@ static JSValue JS_ReadTypedArray(BCReaderState *s) static JSValue JS_ReadArrayBuffer(BCReaderState *s) { JSContext *ctx = s->ctx; - uint32_t byte_length; + uint32_t byte_length, max_byte_length; + uint64_t max_byte_length_u64, *pmax_byte_length = NULL; JSValue obj; if (bc_get_leb128(s, &byte_length)) return JS_EXCEPTION; + if (bc_get_leb128(s, &max_byte_length)) + return JS_EXCEPTION; + if (max_byte_length < byte_length) + return JS_ThrowTypeError(ctx, "invalid array buffer"); + if (max_byte_length != UINT32_MAX) { + max_byte_length_u64 = max_byte_length; + pmax_byte_length = &max_byte_length_u64; + } if (unlikely(s->buf_end - s->ptr < byte_length)) { bc_read_error_end(s); return JS_EXCEPTION; } - obj = JS_NewArrayBufferCopy(ctx, s->ptr, byte_length); + // makes a copy of the input + obj = js_array_buffer_constructor3(ctx, JS_UNDEFINED, + byte_length, pmax_byte_length, + JS_CLASS_ARRAY_BUFFER, + (uint8_t*)s->ptr, + js_array_buffer_free, NULL, + /*alloc_flag*/TRUE); if (JS_IsException(obj)) goto fail; if (BC_add_object_ref(s, obj)) @@ -37153,18 +38368,28 @@ static JSValue JS_ReadArrayBuffer(BCReaderState *s) static JSValue JS_ReadSharedArrayBuffer(BCReaderState *s) { JSContext *ctx = s->ctx; - uint32_t byte_length; + uint32_t byte_length, max_byte_length; + uint64_t max_byte_length_u64, *pmax_byte_length = NULL; uint8_t *data_ptr; JSValue obj; uint64_t u64; if (bc_get_leb128(s, &byte_length)) return JS_EXCEPTION; + if (bc_get_leb128(s, &max_byte_length)) + return JS_EXCEPTION; + if (max_byte_length < byte_length) + return JS_ThrowTypeError(ctx, "invalid array buffer"); + if (max_byte_length != UINT32_MAX) { + max_byte_length_u64 = max_byte_length; + pmax_byte_length = &max_byte_length_u64; + } if (bc_get_u64(s, &u64)) return JS_EXCEPTION; data_ptr = (uint8_t *)(uintptr_t)u64; /* the SharedArrayBuffer is cloned */ - obj = js_array_buffer_constructor3(ctx, JS_UNDEFINED, byte_length, + obj = js_array_buffer_constructor3(ctx, JS_UNDEFINED, + byte_length, pmax_byte_length, JS_CLASS_SHARED_ARRAY_BUFFER, data_ptr, NULL, NULL, FALSE); @@ -37468,11 +38693,25 @@ static JSAtom find_atom(JSContext *ctx, const char *name) return atom; } +static JSValue JS_NewObjectProtoList(JSContext *ctx, JSValueConst proto, + const JSCFunctionListEntry *fields, int n_fields) +{ + JSValue obj; + obj = JS_NewObjectProtoClassAlloc(ctx, proto, JS_CLASS_OBJECT, n_fields); + if (JS_IsException(obj)) + return obj; + if (JS_SetPropertyFunctionList(ctx, obj, fields, n_fields)) { + JS_FreeValue(ctx, obj); + return JS_EXCEPTION; + } + return obj; +} + static JSValue JS_InstantiateFunctionListItem2(JSContext *ctx, JSObject *p, JSAtom atom, void *opaque) { const JSCFunctionListEntry *e = opaque; - JSValue val; + JSValue val, proto; switch(e->def_type) { case JS_DEF_CFUNC: @@ -37483,8 +38722,13 @@ static JSValue JS_InstantiateFunctionListItem2(JSContext *ctx, JSObject *p, val = JS_NewAtomString(ctx, e->u.str); break; case JS_DEF_OBJECT: - val = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, val, e->u.prop_list.tab, e->u.prop_list.len); + /* XXX: could add a flag */ + if (atom == JS_ATOM_Symbol_unscopables) + proto = JS_NULL; + else + proto = ctx->class_proto[JS_CLASS_OBJECT]; + val = JS_NewObjectProtoList(ctx, proto, + e->u.prop_list.tab, e->u.prop_list.len); break; default: abort(); @@ -37573,6 +38817,12 @@ static int JS_InstantiateFunctionListItem(JSContext *ctx, JSValueConst obj, case JS_DEF_PROP_UNDEFINED: val = JS_UNDEFINED; break; + case JS_DEF_PROP_ATOM: + val = JS_AtomToValue(ctx, e->u.i32); + break; + case JS_DEF_PROP_BOOL: + val = JS_NewBool(ctx, e->u.i32); + break; case JS_DEF_PROP_STRING: case JS_DEF_OBJECT: JS_DefineAutoInitProperty(ctx, obj, atom, JS_AUTOINIT_ID_PROP, @@ -37585,17 +38835,22 @@ static int JS_InstantiateFunctionListItem(JSContext *ctx, JSValueConst obj, return 0; } -void JS_SetPropertyFunctionList(JSContext *ctx, JSValueConst obj, - const JSCFunctionListEntry *tab, int len) +int JS_SetPropertyFunctionList(JSContext *ctx, JSValueConst obj, + const JSCFunctionListEntry *tab, int len) { - int i; + int i, ret; for (i = 0; i < len; i++) { const JSCFunctionListEntry *e = &tab[i]; JSAtom atom = find_atom(ctx, e->name); - JS_InstantiateFunctionListItem(ctx, obj, atom, e); + if (atom == JS_ATOM_NULL) + return -1; + ret = JS_InstantiateFunctionListItem(ctx, obj, atom, e); JS_FreeAtom(ctx, atom); + if (ret) + return -1; } + return 0; } int JS_AddModuleExportList(JSContext *ctx, JSModuleDef *m, @@ -37635,8 +38890,8 @@ int JS_SetModuleExportList(JSContext *ctx, JSModuleDef *m, val = __JS_NewFloat64(ctx, e->u.f64); break; case JS_DEF_OBJECT: - val = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, val, e->u.prop_list.tab, e->u.prop_list.len); + val = JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_OBJECT], + e->u.prop_list.tab, e->u.prop_list.len); break; default: abort(); @@ -37648,57 +38903,107 @@ int JS_SetModuleExportList(JSContext *ctx, JSModuleDef *m, } /* Note: 'func_obj' is not necessarily a constructor */ -static void JS_SetConstructor2(JSContext *ctx, - JSValueConst func_obj, - JSValueConst proto, - int proto_flags, int ctor_flags) -{ - JS_DefinePropertyValue(ctx, func_obj, JS_ATOM_prototype, - JS_DupValue(ctx, proto), proto_flags); - JS_DefinePropertyValue(ctx, proto, JS_ATOM_constructor, - JS_DupValue(ctx, func_obj), - ctor_flags); +static int JS_SetConstructor2(JSContext *ctx, + JSValueConst func_obj, + JSValueConst proto, + int proto_flags, int ctor_flags) +{ + if (JS_DefinePropertyValue(ctx, func_obj, JS_ATOM_prototype, + JS_DupValue(ctx, proto), proto_flags) < 0) + return -1; + if (JS_DefinePropertyValue(ctx, proto, JS_ATOM_constructor, + JS_DupValue(ctx, func_obj), + ctor_flags) < 0) + return -1; set_cycle_flag(ctx, func_obj); set_cycle_flag(ctx, proto); + return 0; } -void JS_SetConstructor(JSContext *ctx, JSValueConst func_obj, - JSValueConst proto) +/* return 0 if OK, -1 if exception */ +int JS_SetConstructor(JSContext *ctx, JSValueConst func_obj, + JSValueConst proto) { - JS_SetConstructor2(ctx, func_obj, proto, - 0, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); + return JS_SetConstructor2(ctx, func_obj, proto, + 0, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); } -static void JS_NewGlobalCConstructor2(JSContext *ctx, - JSValue func_obj, - const char *name, - JSValueConst proto) -{ - JS_DefinePropertyValueStr(ctx, ctx->global_obj, name, - JS_DupValue(ctx, func_obj), - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - JS_SetConstructor(ctx, func_obj, proto); - JS_FreeValue(ctx, func_obj); -} +#define JS_NEW_CTOR_NO_GLOBAL (1 << 0) /* don't create a global binding */ +#define JS_NEW_CTOR_PROTO_CLASS (1 << 1) /* the prototype class is 'class_id' instead of JS_CLASS_OBJECT */ +#define JS_NEW_CTOR_PROTO_EXIST (1 << 2) /* the prototype is already defined */ +#define JS_NEW_CTOR_READONLY (1 << 3) /* read-only constructor field */ -static JSValueConst JS_NewGlobalCConstructor(JSContext *ctx, const char *name, - JSCFunction *func, int length, - JSValueConst proto) +/* Return the constructor and. Define it as a global variable unless + JS_NEW_CTOR_NO_GLOBAL is set. The new class inherit from + parent_ctor if it is not JS_UNDEFINED. if class_id is != -1, + class_proto[class_id] is set. */ +static JSValue JS_NewCConstructor(JSContext *ctx, int class_id, const char *name, + JSCFunction *func, int length, JSCFunctionEnum cproto, int magic, + JSValueConst parent_ctor, + const JSCFunctionListEntry *ctor_fields, int n_ctor_fields, + const JSCFunctionListEntry *proto_fields, int n_proto_fields, + int flags) { - JSValue func_obj; - func_obj = JS_NewCFunction2(ctx, func, name, length, JS_CFUNC_constructor_or_func, 0); - JS_NewGlobalCConstructor2(ctx, func_obj, name, proto); - return func_obj; -} + JSValue ctor = JS_UNDEFINED, proto, parent_proto; + int proto_class_id, proto_flags, ctor_flags; -static JSValueConst JS_NewGlobalCConstructorOnly(JSContext *ctx, const char *name, - JSCFunction *func, int length, - JSValueConst proto) -{ - JSValue func_obj; - func_obj = JS_NewCFunction2(ctx, func, name, length, JS_CFUNC_constructor, 0); - JS_NewGlobalCConstructor2(ctx, func_obj, name, proto); - return func_obj; + proto_flags = 0; + if (flags & JS_NEW_CTOR_READONLY) { + ctor_flags = JS_PROP_CONFIGURABLE; + } else { + ctor_flags = JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE; + } + + if (JS_IsUndefined(parent_ctor)) { + parent_proto = JS_DupValue(ctx, ctx->class_proto[JS_CLASS_OBJECT]); + parent_ctor = ctx->function_proto; + } else { + parent_proto = JS_GetProperty(ctx, parent_ctor, JS_ATOM_prototype); + if (JS_IsException(parent_proto)) + return JS_EXCEPTION; + } + + if (flags & JS_NEW_CTOR_PROTO_EXIST) { + proto = JS_DupValue(ctx, ctx->class_proto[class_id]); + } else { + if (flags & JS_NEW_CTOR_PROTO_CLASS) + proto_class_id = class_id; + else + proto_class_id = JS_CLASS_OBJECT; + /* one additional field: constructor */ + proto = JS_NewObjectProtoClassAlloc(ctx, parent_proto, proto_class_id, + n_proto_fields + 1); + if (JS_IsException(proto)) + goto fail; + if (class_id >= 0) + ctx->class_proto[class_id] = JS_DupValue(ctx, proto); + } + if (JS_SetPropertyFunctionList(ctx, proto, proto_fields, n_proto_fields)) + goto fail; + + /* additional fields: name, length, prototype */ + ctor = JS_NewCFunction3(ctx, func, name, length, cproto, magic, parent_ctor, + n_ctor_fields + 3); + if (JS_IsException(ctor)) + goto fail; + if (JS_SetPropertyFunctionList(ctx, ctor, ctor_fields, n_ctor_fields)) + goto fail; + if (!(flags & JS_NEW_CTOR_NO_GLOBAL)) { + if (JS_DefinePropertyValueStr(ctx, ctx->global_obj, name, + JS_DupValue(ctx, ctor), + JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE) < 0) + goto fail; + } + JS_SetConstructor2(ctx, ctor, proto, proto_flags, ctor_flags); + + JS_FreeValue(ctx, proto); + JS_FreeValue(ctx, parent_proto); + return ctor; + fail: + JS_FreeValue(ctx, proto); + JS_FreeValue(ctx, parent_proto); + JS_FreeValue(ctx, ctor); + return JS_EXCEPTION; } static JSValue js_global_eval(JSContext *ctx, JSValueConst this_val, @@ -37913,7 +39218,7 @@ static __exception int JS_ObjectDefineProperties(JSContext *ctx, ret = 0; exception: - js_free_prop_enum(ctx, atoms, len); + JS_FreePropertyEnum(ctx, atoms, len); JS_FreeValue(ctx, props); JS_FreeValue(ctx, desc); return ret; @@ -38184,12 +39489,12 @@ static JSValue js_object_getOwnPropertyDescriptors(JSContext *ctx, JSValueConst goto exception; } } - js_free_prop_enum(ctx, props, len); + JS_FreePropertyEnum(ctx, props, len); JS_FreeValue(ctx, obj); return r; exception: - js_free_prop_enum(ctx, props, len); + JS_FreePropertyEnum(ctx, props, len); JS_FreeValue(ctx, obj); JS_FreeValue(ctx, r); return JS_EXCEPTION; @@ -38269,7 +39574,7 @@ static JSValue JS_GetOwnPropertyNames2(JSContext *ctx, JSValueConst obj1, JS_FreeValue(ctx, r); r = JS_EXCEPTION; done: - js_free_prop_enum(ctx, atoms, len); + JS_FreePropertyEnum(ctx, atoms, len); JS_FreeValue(ctx, obj); return r; } @@ -38530,11 +39835,11 @@ static JSValue js_object_seal(JSContext *ctx, JSValueConst this_val, JS_UNDEFINED, JS_UNDEFINED, desc_flags) < 0) goto exception; } - js_free_prop_enum(ctx, props, len); + JS_FreePropertyEnum(ctx, props, len); return JS_DupValue(ctx, obj); exception: - js_free_prop_enum(ctx, props, len); + JS_FreePropertyEnum(ctx, props, len); return JS_EXCEPTION; } @@ -38576,11 +39881,11 @@ static JSValue js_object_isSealed(JSContext *ctx, JSValueConst this_val, return JS_EXCEPTION; res ^= 1; done: - js_free_prop_enum(ctx, props, len); + JS_FreePropertyEnum(ctx, props, len); return JS_NewBool(ctx, res); exception: - js_free_prop_enum(ctx, props, len); + JS_FreePropertyEnum(ctx, props, len); return JS_EXCEPTION; } @@ -38650,107 +39955,12 @@ static JSValue js_object_fromEntries(JSContext *ctx, JSValueConst this_val, return JS_EXCEPTION; } -#if 0 -/* Note: corresponds to ECMA spec: CreateDataPropertyOrThrow() */ -static JSValue js_object___setOwnProperty(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - int ret; - ret = JS_DefinePropertyValueValue(ctx, argv[0], JS_DupValue(ctx, argv[1]), - JS_DupValue(ctx, argv[2]), - JS_PROP_C_W_E | JS_PROP_THROW); - if (ret < 0) - return JS_EXCEPTION; - else - return JS_NewBool(ctx, ret); -} - -static JSValue js_object___toObject(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - return JS_ToObject(ctx, argv[0]); -} - -static JSValue js_object___toPrimitive(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - int hint = HINT_NONE; - - if (JS_VALUE_GET_TAG(argv[1]) == JS_TAG_INT) - hint = JS_VALUE_GET_INT(argv[1]); - - return JS_ToPrimitive(ctx, argv[0], hint); -} -#endif - -/* return an empty string if not an object */ -static JSValue js_object___getClass(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - JSAtom atom; - JSObject *p; - uint32_t tag; - int class_id; - - tag = JS_VALUE_GET_NORM_TAG(argv[0]); - if (tag == JS_TAG_OBJECT) { - p = JS_VALUE_GET_OBJ(argv[0]); - class_id = p->class_id; - if (class_id == JS_CLASS_PROXY && JS_IsFunction(ctx, argv[0])) - class_id = JS_CLASS_BYTECODE_FUNCTION; - atom = ctx->rt->class_array[class_id].class_name; - } else { - atom = JS_ATOM_empty_string; - } - return JS_AtomToString(ctx, atom); -} - static JSValue js_object_is(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { return JS_NewBool(ctx, js_same_value(ctx, argv[0], argv[1])); } -#if 0 -static JSValue js_object___getObjectData(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - return JS_GetObjectData(ctx, argv[0]); -} - -static JSValue js_object___setObjectData(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - if (JS_SetObjectData(ctx, argv[0], JS_DupValue(ctx, argv[1]))) - return JS_EXCEPTION; - return JS_DupValue(ctx, argv[1]); -} - -static JSValue js_object___toPropertyKey(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - return JS_ToPropertyKey(ctx, argv[0]); -} - -static JSValue js_object___isObject(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - return JS_NewBool(ctx, JS_IsObject(argv[0])); -} - -static JSValue js_object___isSameValueZero(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - return JS_NewBool(ctx, js_same_value_zero(ctx, argv[0], argv[1])); -} - -static JSValue js_object___isConstructor(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - return JS_NewBool(ctx, JS_IsConstructor(ctx, argv[0])); -} -#endif - static JSValue JS_SpeciesConstructor(JSContext *ctx, JSValueConst obj, JSValueConst defaultConstructor) { @@ -38774,20 +39984,13 @@ static JSValue JS_SpeciesConstructor(JSContext *ctx, JSValueConst obj, if (JS_IsUndefined(species) || JS_IsNull(species)) return JS_DupValue(ctx, defaultConstructor); if (!JS_IsConstructor(ctx, species)) { + JS_ThrowTypeErrorNotAConstructor(ctx, species); JS_FreeValue(ctx, species); - return JS_ThrowTypeError(ctx, "not a constructor"); + return JS_EXCEPTION; } return species; } -#if 0 -static JSValue js_object___speciesConstructor(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - return JS_SpeciesConstructor(ctx, argv[0], argv[1]); -} -#endif - static JSValue js_object_get___proto__(JSContext *ctx, JSValueConst this_val) { JSValue val, ret; @@ -38951,17 +40154,6 @@ static const JSCFunctionListEntry js_object_funcs[] = { JS_CFUNC_MAGIC_DEF("freeze", 1, js_object_seal, 1 ), JS_CFUNC_MAGIC_DEF("isSealed", 1, js_object_isSealed, 0 ), JS_CFUNC_MAGIC_DEF("isFrozen", 1, js_object_isSealed, 1 ), - JS_CFUNC_DEF("__getClass", 1, js_object___getClass ), - //JS_CFUNC_DEF("__isObject", 1, js_object___isObject ), - //JS_CFUNC_DEF("__isConstructor", 1, js_object___isConstructor ), - //JS_CFUNC_DEF("__toObject", 1, js_object___toObject ), - //JS_CFUNC_DEF("__setOwnProperty", 3, js_object___setOwnProperty ), - //JS_CFUNC_DEF("__toPrimitive", 2, js_object___toPrimitive ), - //JS_CFUNC_DEF("__toPropertyKey", 1, js_object___toPropertyKey ), - //JS_CFUNC_DEF("__speciesConstructor", 2, js_object___speciesConstructor ), - //JS_CFUNC_DEF("__isSameValueZero", 2, js_object___isSameValueZero ), - //JS_CFUNC_DEF("__getObjectData", 1, js_object___getObjectData ), - //JS_CFUNC_DEF("__setObjectData", 2, js_object___setObjectData ), JS_CFUNC_DEF("fromEntries", 1, js_object_fromEntries ), JS_CFUNC_DEF("hasOwn", 2, js_object_hasOwn ), }; @@ -39478,6 +40670,35 @@ static const JSCFunctionListEntry js_error_proto_funcs[] = { JS_PROP_STRING_DEF("message", "", JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE ), }; +/* 2 entries for each native error class */ +/* Note: we use an atom to avoid the autoinit definition which does + not work in get_prop_string() */ +static const JSCFunctionListEntry js_native_error_proto_funcs[] = { +#define DEF(name) \ + JS_PROP_ATOM_DEF("name", name, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE ),\ + JS_PROP_STRING_DEF("message", "", JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE ), + + DEF(JS_ATOM_EvalError) + DEF(JS_ATOM_RangeError) + DEF(JS_ATOM_ReferenceError) + DEF(JS_ATOM_SyntaxError) + DEF(JS_ATOM_TypeError) + DEF(JS_ATOM_URIError) + DEF(JS_ATOM_InternalError) + DEF(JS_ATOM_AggregateError) +#undef DEF +}; + +static JSValue js_error_isError(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + return JS_NewBool(ctx, JS_IsError(ctx, argv[0])); +} + +static const JSCFunctionListEntry js_error_funcs[] = { + JS_CFUNC_DEF("isError", 1, js_error_isError), +}; + /* AggregateError */ /* used by C code. */ @@ -39999,8 +41220,6 @@ static JSValue js_array_concat(JSContext *ctx, JSValueConst this_val, #define special_filter 4 #define special_TA 8 -static int js_typed_array_get_length_internal(JSContext *ctx, JSValueConst obj); - static JSValue js_typed_array___speciesCreate(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv); @@ -40018,7 +41237,7 @@ static JSValue js_array_every(JSContext *ctx, JSValueConst this_val, val = JS_UNDEFINED; if (special & special_TA) { obj = JS_DupValue(ctx, this_val); - len = js_typed_array_get_length_internal(ctx, obj); + len = js_typed_array_get_length_unsafe(ctx, obj); if (len < 0) goto exception; } else { @@ -40141,8 +41360,10 @@ static JSValue js_array_every(JSContext *ctx, JSValueConst this_val, goto exception; args[0] = ret; res = JS_Invoke(ctx, arr, JS_ATOM_set, 1, args); - if (check_exception_free(ctx, res)) + if (check_exception_free(ctx, res)) { + JS_FreeValue(ctx, arr); goto exception; + } JS_FreeValue(ctx, ret); ret = arr; } @@ -40173,7 +41394,7 @@ static JSValue js_array_reduce(JSContext *ctx, JSValueConst this_val, val = JS_UNDEFINED; if (special & special_TA) { obj = JS_DupValue(ctx, this_val); - len = js_typed_array_get_length_internal(ctx, obj); + len = js_typed_array_get_length_unsafe(ctx, obj); if (len < 0) goto exception; } else { @@ -40645,6 +41866,31 @@ static JSValue js_array_push(JSContext *ctx, JSValueConst this_val, int i; int64_t len, from, newLen; + if (likely(JS_VALUE_GET_TAG(this_val) == JS_TAG_OBJECT && !unshift)) { + JSObject *p = JS_VALUE_GET_OBJ(this_val); + if (likely(p->class_id == JS_CLASS_ARRAY && p->fast_array && + p->extensible && + p->shape->proto == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]) && + ctx->std_array_prototype && + JS_VALUE_GET_TAG(p->prop[0].u.value) == JS_TAG_INT && + JS_VALUE_GET_INT(p->prop[0].u.value) == p->u.array.count && + (get_shape_prop(p->shape)->flags & JS_PROP_WRITABLE) != 0)) { + /* fast case */ + uint32_t new_len; + new_len = p->u.array.count + argc; + if (likely(new_len <= INT32_MAX)) { + if (unlikely(new_len > p->u.array.u1.size)) { + if (expand_fast_array(ctx, p, new_len)) + return JS_EXCEPTION; + } + for(i = 0; i < argc; i++) + p->u.array.u.values[p->u.array.count + i] = JS_DupValue(ctx, argv[i]); + p->prop[0].u.value = JS_NewInt32(ctx, new_len); + p->u.array.count = new_len; + return JS_NewInt32(ctx, new_len); + } + } + } obj = JS_ToObject(ctx, this_val); if (js_get_length64(ctx, &len, obj)) goto exception; @@ -41383,23 +42629,6 @@ static void js_array_iterator_mark(JSRuntime *rt, JSValueConst val, } } -static JSValue js_create_array(JSContext *ctx, int len, JSValueConst *tab) -{ - JSValue obj; - int i; - - obj = JS_NewArray(ctx); - if (JS_IsException(obj)) - return JS_EXCEPTION; - for(i = 0; i < len; i++) { - if (JS_CreateDataPropertyUint32(ctx, obj, i, JS_DupValue(ctx, tab[i]), 0) < 0) { - JS_FreeValue(ctx, obj); - return JS_EXCEPTION; - } - } - return obj; -} - static JSValue js_create_array_iterator(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, int magic) { @@ -41454,8 +42683,8 @@ static JSValue js_array_iterator_next(JSContext *ctx, JSValueConst this_val, p = JS_VALUE_GET_OBJ(it->obj); if (p->class_id >= JS_CLASS_UINT8C_ARRAY && p->class_id <= JS_CLASS_FLOAT64_ARRAY) { - if (typed_array_is_detached(ctx, p)) { - JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + if (typed_array_is_oob(p)) { + JS_ThrowTypeErrorArrayBufferOOB(ctx); goto fail1; } len = p->u.array.count; @@ -41498,14 +42727,889 @@ static JSValue js_array_iterator_next(JSContext *ctx, JSValueConst this_val, } } +/* Iterator Wrap */ + +typedef struct JSIteratorWrapData { + JSValue wrapped_iter; + JSValue wrapped_next; +} JSIteratorWrapData; + +static void js_iterator_wrap_finalizer(JSRuntime *rt, JSValue val) +{ + JSObject *p = JS_VALUE_GET_OBJ(val); + JSIteratorWrapData *it = p->u.iterator_wrap_data; + if (it) { + JS_FreeValueRT(rt, it->wrapped_iter); + JS_FreeValueRT(rt, it->wrapped_next); + js_free_rt(rt, it); + } +} + +static void js_iterator_wrap_mark(JSRuntime *rt, JSValueConst val, + JS_MarkFunc *mark_func) +{ + JSObject *p = JS_VALUE_GET_OBJ(val); + JSIteratorWrapData *it = p->u.iterator_wrap_data; + if (it) { + JS_MarkValue(rt, it->wrapped_iter, mark_func); + JS_MarkValue(rt, it->wrapped_next, mark_func); + } +} + +static JSValue js_iterator_wrap_next(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv, + int *pdone, int magic) +{ + JSIteratorWrapData *it; + JSValue method, ret; + it = JS_GetOpaque2(ctx, this_val, JS_CLASS_ITERATOR_WRAP); + if (!it) + return JS_EXCEPTION; + if (magic == GEN_MAGIC_NEXT) { + return JS_IteratorNext(ctx, it->wrapped_iter, it->wrapped_next, 0, NULL, pdone); + } else { + method = JS_GetProperty(ctx, it->wrapped_iter, JS_ATOM_return); + if (JS_IsException(method)) + return JS_EXCEPTION; + if (JS_IsNull(method) || JS_IsUndefined(method)) { + *pdone = TRUE; + return JS_UNDEFINED; + } + ret = JS_IteratorNext2(ctx, it->wrapped_iter, method, 0, NULL, pdone); + JS_FreeValue(ctx, method); + return ret; + } +} + +static const JSCFunctionListEntry js_iterator_wrap_proto_funcs[] = { + JS_ITERATOR_NEXT_DEF("next", 0, js_iterator_wrap_next, GEN_MAGIC_NEXT ), + JS_ITERATOR_NEXT_DEF("return", 0, js_iterator_wrap_next, GEN_MAGIC_RETURN ), +}; + +/* Iterator */ + +static JSValue js_iterator_constructor_getset(JSContext *ctx, + JSValueConst this_val, + int argc, JSValueConst *argv, + int magic, + JSValue *func_data) +{ + int ret; + + if (argc > 0) { // if setter + if (!JS_IsObject(argv[0])) + return JS_ThrowTypeErrorNotAnObject(ctx); + ret = JS_DefinePropertyValue(ctx, this_val, JS_ATOM_constructor, + JS_DupValue(ctx, argv[0]), + JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); + if (ret < 0) + return JS_EXCEPTION; + return JS_UNDEFINED; + } else { + return JS_DupValue(ctx, func_data[0]); + } +} + +static JSValue js_iterator_constructor(JSContext *ctx, JSValueConst new_target, + int argc, JSValueConst *argv) +{ + JSObject *p; + + if (JS_TAG_OBJECT != JS_VALUE_GET_TAG(new_target)) + return JS_ThrowTypeError(ctx, "constructor requires 'new'"); + p = JS_VALUE_GET_OBJ(new_target); + if (p->class_id == JS_CLASS_C_FUNCTION && + p->u.cfunc.c_function.generic == js_iterator_constructor) { + return JS_ThrowTypeError(ctx, "abstract class not constructable"); + } + return js_create_from_ctor(ctx, new_target, JS_CLASS_ITERATOR); +} + +static JSValue js_iterator_from(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValueConst obj = argv[0]; + JSValue method, iter, wrapper; + JSIteratorWrapData *it; + int ret; + + if (!JS_IsObject(obj)) { + if (!JS_IsString(obj)) + return JS_ThrowTypeError(ctx, "Iterator.from called on non-object"); + } + method = JS_GetProperty(ctx, obj, JS_ATOM_Symbol_iterator); + if (JS_IsException(method)) + return JS_EXCEPTION; + if (JS_IsNull(method) || JS_IsUndefined(method)) { + iter = JS_DupValue(ctx, obj); + } else { + iter = JS_GetIterator2(ctx, obj, method); + JS_FreeValue(ctx, method); + if (JS_IsException(iter)) + return JS_EXCEPTION; + } + + wrapper = JS_UNDEFINED; + method = JS_GetProperty(ctx, iter, JS_ATOM_next); + if (JS_IsException(method)) + goto fail; + + ret = JS_OrdinaryIsInstanceOf(ctx, iter, ctx->iterator_ctor); + if (ret < 0) + goto fail; + if (ret) { + JS_FreeValue(ctx, method); + return iter; + } + + wrapper = JS_NewObjectClass(ctx, JS_CLASS_ITERATOR_WRAP); + if (JS_IsException(wrapper)) + goto fail; + it = js_malloc(ctx, sizeof(*it)); + if (!it) + goto fail; + it->wrapped_iter = iter; + it->wrapped_next = method; + JS_SetOpaque(wrapper, it); + return wrapper; + + fail: + JS_FreeValue(ctx, method); + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, wrapper); + return JS_EXCEPTION; +} + +typedef enum JSIteratorHelperKindEnum { + JS_ITERATOR_HELPER_KIND_DROP, + JS_ITERATOR_HELPER_KIND_EVERY, + JS_ITERATOR_HELPER_KIND_FILTER, + JS_ITERATOR_HELPER_KIND_FIND, + JS_ITERATOR_HELPER_KIND_FLAT_MAP, + JS_ITERATOR_HELPER_KIND_FOR_EACH, + JS_ITERATOR_HELPER_KIND_MAP, + JS_ITERATOR_HELPER_KIND_SOME, + JS_ITERATOR_HELPER_KIND_TAKE, +} JSIteratorHelperKindEnum; + +typedef struct JSIteratorHelperData { + JSValue obj; + JSValue next; + JSValue func; // predicate (filter) or mapper (flatMap, map) + JSValue inner; // innerValue (flatMap) + int64_t count; // limit (drop, take) or counter (filter, map, flatMap) + JSIteratorHelperKindEnum kind : 8; + uint8_t executing : 1; + uint8_t done : 1; +} JSIteratorHelperData; + +static JSValue js_create_iterator_helper(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv, int magic) +{ + JSValueConst func; + JSValue obj, method; + int64_t count; + JSIteratorHelperData *it; + + if (!JS_IsObject(this_val)) + return JS_ThrowTypeErrorNotAnObject(ctx); + func = JS_UNDEFINED; + count = 0; + + switch(magic) { + case JS_ITERATOR_HELPER_KIND_DROP: + case JS_ITERATOR_HELPER_KIND_TAKE: + { + JSValue v; + double dlimit; + v = JS_ToNumber(ctx, argv[0]); + if (JS_IsException(v)) + goto fail; + // Check for Infinity. + if (JS_ToFloat64(ctx, &dlimit, v)) { + JS_FreeValue(ctx, v); + goto fail; + } + if (isnan(dlimit)) { + JS_FreeValue(ctx, v); + goto range_error; + } + if (!isfinite(dlimit)) { + JS_FreeValue(ctx, v); + if (dlimit < 0) + goto range_error; + else + count = MAX_SAFE_INTEGER; + } else { + v = JS_ToIntegerFree(ctx, v); + if (JS_IsException(v)) + goto fail; + if (JS_ToInt64Free(ctx, &count, v)) + goto fail; + } + if (count < 0) + goto range_error; + } + break; + case JS_ITERATOR_HELPER_KIND_FILTER: + case JS_ITERATOR_HELPER_KIND_FLAT_MAP: + case JS_ITERATOR_HELPER_KIND_MAP: + { + func = argv[0]; + if (check_function(ctx, func)) + goto fail; + } + break; + default: + abort(); + break; + } + + method = JS_GetProperty(ctx, this_val, JS_ATOM_next); + if (JS_IsException(method)) + goto fail; + obj = JS_NewObjectClass(ctx, JS_CLASS_ITERATOR_HELPER); + if (JS_IsException(obj)) { + JS_FreeValue(ctx, method); + goto fail; + } + it = js_malloc(ctx, sizeof(*it)); + if (!it) { + JS_FreeValue(ctx, obj); + JS_FreeValue(ctx, method); + goto fail; + } + it->kind = magic; + it->obj = JS_DupValue(ctx, this_val); + it->func = JS_DupValue(ctx, func); + it->next = method; + it->inner = JS_UNDEFINED; + it->count = count; + it->executing = 0; + it->done = 0; + JS_SetOpaque(obj, it); + return obj; +range_error: + JS_ThrowRangeError(ctx, "must be positive"); +fail: + JS_IteratorClose(ctx, this_val, TRUE); + return JS_EXCEPTION; +} + +static JSValue js_iterator_proto_func(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv, int magic) +{ + JSValue item, method, ret, func, index_val, r; + JSValueConst args[2]; + int64_t idx; + int done; + + if (!JS_IsObject(this_val)) + return JS_ThrowTypeErrorNotAnObject(ctx); + func = JS_UNDEFINED; + method = JS_UNDEFINED; + + if (check_function(ctx, argv[0])) + goto fail; + func = JS_DupValue(ctx, argv[0]); + method = JS_GetProperty(ctx, this_val, JS_ATOM_next); + if (JS_IsException(method)) + goto fail_no_close; + + r = JS_UNDEFINED; + + switch(magic) { + case JS_ITERATOR_HELPER_KIND_EVERY: + { + r = JS_TRUE; + for (idx = 0; /*empty*/; idx++) { + item = JS_IteratorNext(ctx, this_val, method, 0, NULL, &done); + if (JS_IsException(item)) + goto fail_no_close; + if (done) + break; + index_val = JS_NewInt64(ctx, idx); + args[0] = item; + args[1] = index_val; + ret = JS_Call(ctx, func, JS_UNDEFINED, countof(args), args); + JS_FreeValue(ctx, item); + JS_FreeValue(ctx, index_val); + if (JS_IsException(ret)) + goto fail; + if (!JS_ToBoolFree(ctx, ret)) { + if (JS_IteratorClose(ctx, this_val, FALSE) < 0) + r = JS_EXCEPTION; + else + r = JS_FALSE; + break; + } + index_val = JS_UNDEFINED; + ret = JS_UNDEFINED; + item = JS_UNDEFINED; + } + } + break; + case JS_ITERATOR_HELPER_KIND_FIND: + { + for (idx = 0; /*empty*/; idx++) { + item = JS_IteratorNext(ctx, this_val, method, 0, NULL, &done); + if (JS_IsException(item)) + goto fail_no_close; + if (done) + break; + index_val = JS_NewInt64(ctx, idx); + args[0] = item; + args[1] = index_val; + ret = JS_Call(ctx, func, JS_UNDEFINED, countof(args), args); + JS_FreeValue(ctx, index_val); + if (JS_IsException(ret)) { + JS_FreeValue(ctx, item); + goto fail; + } + if (JS_ToBoolFree(ctx, ret)) { + if (JS_IteratorClose(ctx, this_val, FALSE) < 0) { + JS_FreeValue(ctx, item); + r = JS_EXCEPTION; + } else { + r = item; + } + break; + } + JS_FreeValue(ctx, item); + index_val = JS_UNDEFINED; + ret = JS_UNDEFINED; + item = JS_UNDEFINED; + } + } + break; + case JS_ITERATOR_HELPER_KIND_FOR_EACH: + { + for (idx = 0; /*empty*/; idx++) { + item = JS_IteratorNext(ctx, this_val, method, 0, NULL, &done); + if (JS_IsException(item)) + goto fail_no_close; + if (done) + break; + index_val = JS_NewInt64(ctx, idx); + args[0] = item; + args[1] = index_val; + ret = JS_Call(ctx, func, JS_UNDEFINED, countof(args), args); + JS_FreeValue(ctx, item); + JS_FreeValue(ctx, index_val); + if (JS_IsException(ret)) + goto fail; + JS_FreeValue(ctx, ret); + index_val = JS_UNDEFINED; + ret = JS_UNDEFINED; + item = JS_UNDEFINED; + } + } + break; + case JS_ITERATOR_HELPER_KIND_SOME: + { + r = JS_FALSE; + for (idx = 0; /*empty*/; idx++) { + item = JS_IteratorNext(ctx, this_val, method, 0, NULL, &done); + if (JS_IsException(item)) + goto fail_no_close; + if (done) + break; + index_val = JS_NewInt64(ctx, idx); + args[0] = item; + args[1] = index_val; + ret = JS_Call(ctx, func, JS_UNDEFINED, countof(args), args); + JS_FreeValue(ctx, item); + JS_FreeValue(ctx, index_val); + if (JS_IsException(ret)) + goto fail; + if (JS_ToBoolFree(ctx, ret)) { + if (JS_IteratorClose(ctx, this_val, FALSE) < 0) + r = JS_EXCEPTION; + else + r = JS_TRUE; + break; + } + index_val = JS_UNDEFINED; + ret = JS_UNDEFINED; + item = JS_UNDEFINED; + } + } + break; + default: + abort(); + break; + } + + JS_FreeValue(ctx, func); + JS_FreeValue(ctx, method); + return r; + fail: + JS_IteratorClose(ctx, this_val, TRUE); + fail_no_close: + JS_FreeValue(ctx, func); + JS_FreeValue(ctx, method); + return JS_EXCEPTION; +} + +static JSValue js_iterator_proto_reduce(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue item, method, ret, func, index_val, acc; + JSValueConst args[3]; + int64_t idx; + int done; + + if (!JS_IsObject(this_val)) + return JS_ThrowTypeErrorNotAnObject(ctx); + acc = JS_UNDEFINED; + func = JS_UNDEFINED; + method = JS_UNDEFINED; + if (check_function(ctx, argv[0])) + goto exception; + func = JS_DupValue(ctx, argv[0]); + method = JS_GetProperty(ctx, this_val, JS_ATOM_next); + if (JS_IsException(method)) + goto exception; + if (argc > 1) { + acc = JS_DupValue(ctx, argv[1]); + idx = 0; + } else { + acc = JS_IteratorNext(ctx, this_val, method, 0, NULL, &done); + if (JS_IsException(acc)) + goto exception_no_close; + if (done) { + JS_ThrowTypeError(ctx, "empty iterator"); + goto exception; + } + idx = 1; + } + for (/* empty */; /*empty*/; idx++) { + item = JS_IteratorNext(ctx, this_val, method, 0, NULL, &done); + if (JS_IsException(item)) + goto exception_no_close; + if (done) + break; + index_val = JS_NewInt64(ctx, idx); + args[0] = acc; + args[1] = item; + args[2] = index_val; + ret = JS_Call(ctx, func, JS_UNDEFINED, countof(args), args); + JS_FreeValue(ctx, item); + JS_FreeValue(ctx, index_val); + if (JS_IsException(ret)) + goto exception; + JS_FreeValue(ctx, acc); + acc = ret; + index_val = JS_UNDEFINED; + ret = JS_UNDEFINED; + item = JS_UNDEFINED; + } + JS_FreeValue(ctx, func); + JS_FreeValue(ctx, method); + return acc; + exception: + JS_IteratorClose(ctx, this_val, TRUE); + exception_no_close: + JS_FreeValue(ctx, acc); + JS_FreeValue(ctx, func); + JS_FreeValue(ctx, method); + return JS_EXCEPTION; +} + +static JSValue js_iterator_proto_toArray(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue item, method, result; + int64_t idx; + int done; + + result = JS_UNDEFINED; + if (!JS_IsObject(this_val)) + return JS_ThrowTypeErrorNotAnObject(ctx); + method = JS_GetProperty(ctx, this_val, JS_ATOM_next); + if (JS_IsException(method)) + return JS_EXCEPTION; + result = JS_NewArray(ctx); + if (JS_IsException(result)) + goto exception; + for (idx = 0; /*empty*/; idx++) { + item = JS_IteratorNext(ctx, this_val, method, 0, NULL, &done); + if (JS_IsException(item)) + goto exception; + if (done) + break; + if (JS_DefinePropertyValueInt64(ctx, result, idx, item, + JS_PROP_C_W_E | JS_PROP_THROW) < 0) + goto exception; + } + if (JS_SetProperty(ctx, result, JS_ATOM_length, JS_NewUint32(ctx, idx)) < 0) + goto exception; + JS_FreeValue(ctx, method); + return result; +exception: + JS_FreeValue(ctx, result); + JS_FreeValue(ctx, method); + return JS_EXCEPTION; +} + static JSValue js_iterator_proto_iterator(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { return JS_DupValue(ctx, this_val); } +static JSValue js_iterator_proto_get_toStringTag(JSContext *ctx, JSValueConst this_val) +{ + return JS_AtomToString(ctx, JS_ATOM_Iterator); +} + +static JSValue js_iterator_proto_set_toStringTag(JSContext *ctx, JSValueConst this_val, JSValueConst val) +{ + int res; + + if (!JS_IsObject(this_val)) + return JS_ThrowTypeErrorNotAnObject(ctx); + if (js_same_value(ctx, this_val, ctx->class_proto[JS_CLASS_ITERATOR])) + return JS_ThrowTypeError(ctx, "Cannot assign to read only property"); + res = JS_GetOwnProperty(ctx, NULL, this_val, JS_ATOM_Symbol_toStringTag); + if (res < 0) + return JS_EXCEPTION; + if (res) { + if (JS_SetProperty(ctx, this_val, JS_ATOM_Symbol_toStringTag, JS_DupValue(ctx, val)) < 0) + return JS_EXCEPTION; + } else { + if (JS_DefinePropertyValue(ctx, this_val, JS_ATOM_Symbol_toStringTag, JS_DupValue(ctx, val), JS_PROP_C_W_E) < 0) + return JS_EXCEPTION; + } + return JS_UNDEFINED; +} + +/* Iterator Helper */ + +static void js_iterator_helper_finalizer(JSRuntime *rt, JSValue val) +{ + JSObject *p = JS_VALUE_GET_OBJ(val); + JSIteratorHelperData *it = p->u.iterator_helper_data; + if (it) { + JS_FreeValueRT(rt, it->obj); + JS_FreeValueRT(rt, it->func); + JS_FreeValueRT(rt, it->next); + JS_FreeValueRT(rt, it->inner); + js_free_rt(rt, it); + } +} + +static void js_iterator_helper_mark(JSRuntime *rt, JSValueConst val, + JS_MarkFunc *mark_func) +{ + JSObject *p = JS_VALUE_GET_OBJ(val); + JSIteratorHelperData *it = p->u.iterator_helper_data; + if (it) { + JS_MarkValue(rt, it->obj, mark_func); + JS_MarkValue(rt, it->func, mark_func); + JS_MarkValue(rt, it->next, mark_func); + JS_MarkValue(rt, it->inner, mark_func); + } +} + +static JSValue js_iterator_helper_next(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv, + int *pdone, int magic) +{ + JSIteratorHelperData *it; + JSValue ret; + + *pdone = FALSE; + + it = JS_GetOpaque2(ctx, this_val, JS_CLASS_ITERATOR_HELPER); + if (!it) + return JS_EXCEPTION; + if (it->executing) + return JS_ThrowTypeError(ctx, "cannot invoke a running iterator"); + if (it->done) { + *pdone = TRUE; + return JS_UNDEFINED; + } + + it->executing = 1; + + switch (it->kind) { + case JS_ITERATOR_HELPER_KIND_DROP: + { + JSValue item, method; + if (magic == GEN_MAGIC_NEXT) { + method = JS_DupValue(ctx, it->next); + } else { + method = JS_GetProperty(ctx, it->obj, JS_ATOM_return); + if (JS_IsException(method)) + goto fail; + } + while (it->count > 0) { + it->count--; + item = JS_IteratorNext(ctx, it->obj, method, 0, NULL, pdone); + if (JS_IsException(item)) { + JS_FreeValue(ctx, method); + goto fail_no_close; + } + JS_FreeValue(ctx, item); + if (magic == GEN_MAGIC_RETURN) + *pdone = TRUE; + if (*pdone) { + JS_FreeValue(ctx, method); + ret = JS_UNDEFINED; + goto done; + } + } + + item = JS_IteratorNext(ctx, it->obj, method, 0, NULL, pdone); + JS_FreeValue(ctx, method); + if (JS_IsException(item)) + goto fail_no_close; + ret = item; + goto done; + } + break; + case JS_ITERATOR_HELPER_KIND_FILTER: + { + JSValue item, method, selected, index_val; + JSValueConst args[2]; + if (magic == GEN_MAGIC_NEXT) { + method = JS_DupValue(ctx, it->next); + } else { + method = JS_GetProperty(ctx, it->obj, JS_ATOM_return); + if (JS_IsException(method)) + goto fail; + } + filter_again: + item = JS_IteratorNext(ctx, it->obj, method, 0, NULL, pdone); + if (JS_IsException(item)) { + JS_FreeValue(ctx, method); + goto fail_no_close; + } + if (*pdone || magic == GEN_MAGIC_RETURN) { + JS_FreeValue(ctx, method); + ret = item; + goto done; + } + index_val = JS_NewInt64(ctx, it->count++); + args[0] = item; + args[1] = index_val; + selected = JS_Call(ctx, it->func, JS_UNDEFINED, countof(args), args); + JS_FreeValue(ctx, index_val); + if (JS_IsException(selected)) { + JS_FreeValue(ctx, item); + JS_FreeValue(ctx, method); + goto fail; + } + if (JS_ToBoolFree(ctx, selected)) { + JS_FreeValue(ctx, method); + ret = item; + goto done; + } + JS_FreeValue(ctx, item); + goto filter_again; + } + break; + case JS_ITERATOR_HELPER_KIND_FLAT_MAP: + { + JSValue item, method, index_val, iter; + JSValueConst args[2]; + flat_map_again: + if (JS_IsUndefined(it->inner)) { + if (magic == GEN_MAGIC_NEXT) { + method = JS_DupValue(ctx, it->next); + } else { + method = JS_GetProperty(ctx, it->obj, JS_ATOM_return); + if (JS_IsException(method)) + goto fail; + } + item = JS_IteratorNext(ctx, it->obj, method, 0, NULL, pdone); + JS_FreeValue(ctx, method); + if (JS_IsException(item)) + goto fail_no_close; + if (*pdone || magic == GEN_MAGIC_RETURN) { + ret = item; + goto done; + } + index_val = JS_NewInt64(ctx, it->count++); + args[0] = item; + args[1] = index_val; + ret = JS_Call(ctx, it->func, JS_UNDEFINED, countof(args), args); + JS_FreeValue(ctx, item); + JS_FreeValue(ctx, index_val); + if (JS_IsException(ret)) + goto fail; + if (!JS_IsObject(ret)) { + JS_FreeValue(ctx, ret); + JS_ThrowTypeError(ctx, "not an object"); + goto fail; + } + method = JS_GetProperty(ctx, ret, JS_ATOM_Symbol_iterator); + if (JS_IsException(method)) { + JS_FreeValue(ctx, ret); + goto fail; + } + if (JS_IsNull(method) || JS_IsUndefined(method)) { + JS_FreeValue(ctx, method); + iter = ret; + } else { + iter = JS_GetIterator2(ctx, ret, method); + JS_FreeValue(ctx, method); + JS_FreeValue(ctx, ret); + if (JS_IsException(iter)) + goto fail; + } + + it->inner = iter; + } + + if (magic == GEN_MAGIC_NEXT) + method = JS_GetProperty(ctx, it->inner, JS_ATOM_next); + else + method = JS_GetProperty(ctx, it->inner, JS_ATOM_return); + if (JS_IsException(method)) { + inner_fail: + JS_IteratorClose(ctx, it->inner, FALSE); + JS_FreeValue(ctx, it->inner); + it->inner = JS_UNDEFINED; + goto fail; + } + if (magic == GEN_MAGIC_RETURN && (JS_IsUndefined(method) || JS_IsNull(method))) { + goto inner_end; + } else { + item = JS_IteratorNext(ctx, it->inner, method, 0, NULL, pdone); + JS_FreeValue(ctx, method); + if (JS_IsException(item)) + goto inner_fail; + } + if (*pdone) { + inner_end: + *pdone = FALSE; // The outer iterator must continue. + JS_IteratorClose(ctx, it->inner, FALSE); + JS_FreeValue(ctx, it->inner); + it->inner = JS_UNDEFINED; + goto flat_map_again; + } + ret = item; + goto done; + } + break; + case JS_ITERATOR_HELPER_KIND_MAP: + { + JSValue item, method, index_val; + JSValueConst args[2]; + if (magic == GEN_MAGIC_NEXT) { + method = JS_DupValue(ctx, it->next); + } else { + method = JS_GetProperty(ctx, it->obj, JS_ATOM_return); + if (JS_IsException(method)) + goto fail; + } + item = JS_IteratorNext(ctx, it->obj, method, 0, NULL, pdone); + JS_FreeValue(ctx, method); + if (JS_IsException(item)) + goto fail_no_close; + if (*pdone || magic == GEN_MAGIC_RETURN) { + ret = item; + goto done; + } + index_val = JS_NewInt64(ctx, it->count++); + args[0] = item; + args[1] = index_val; + ret = JS_Call(ctx, it->func, JS_UNDEFINED, countof(args), args); + JS_FreeValue(ctx, index_val); + if (JS_IsException(ret)) + goto fail; + goto done; + } + break; + case JS_ITERATOR_HELPER_KIND_TAKE: + { + JSValue item, method; + if (it->count > 0) { + if (magic == GEN_MAGIC_NEXT) { + method = JS_DupValue(ctx, it->next); + } else { + method = JS_GetProperty(ctx, it->obj, JS_ATOM_return); + if (JS_IsException(method)) + goto fail; + } + it->count--; + item = JS_IteratorNext(ctx, it->obj, method, 0, NULL, pdone); + JS_FreeValue(ctx, method); + if (JS_IsException(item)) + goto fail_no_close; + ret = item; + goto done; + } + + *pdone = TRUE; + if (JS_IteratorClose(ctx, it->obj, FALSE)) + ret = JS_EXCEPTION; + else + ret = JS_UNDEFINED; + goto done; + } + break; + default: + abort(); + } + + done: + it->done = magic == GEN_MAGIC_NEXT ? *pdone : 1; + it->executing = 0; + return ret; + fail: + /* close the iterator object, preserving pending exception */ + JS_IteratorClose(ctx, it->obj, TRUE); + fail_no_close: + ret = JS_EXCEPTION; + goto done; +} + +static const JSCFunctionListEntry js_iterator_funcs[] = { + JS_CFUNC_DEF("from", 1, js_iterator_from ), +}; + static const JSCFunctionListEntry js_iterator_proto_funcs[] = { + JS_CFUNC_MAGIC_DEF("drop", 1, js_create_iterator_helper, JS_ITERATOR_HELPER_KIND_DROP ), + JS_CFUNC_MAGIC_DEF("filter", 1, js_create_iterator_helper, JS_ITERATOR_HELPER_KIND_FILTER ), + JS_CFUNC_MAGIC_DEF("flatMap", 1, js_create_iterator_helper, JS_ITERATOR_HELPER_KIND_FLAT_MAP ), + JS_CFUNC_MAGIC_DEF("map", 1, js_create_iterator_helper, JS_ITERATOR_HELPER_KIND_MAP ), + JS_CFUNC_MAGIC_DEF("take", 1, js_create_iterator_helper, JS_ITERATOR_HELPER_KIND_TAKE ), + JS_CFUNC_MAGIC_DEF("every", 1, js_iterator_proto_func, JS_ITERATOR_HELPER_KIND_EVERY ), + JS_CFUNC_MAGIC_DEF("find", 1, js_iterator_proto_func, JS_ITERATOR_HELPER_KIND_FIND), + JS_CFUNC_MAGIC_DEF("forEach", 1, js_iterator_proto_func, JS_ITERATOR_HELPER_KIND_FOR_EACH ), + JS_CFUNC_MAGIC_DEF("some", 1, js_iterator_proto_func, JS_ITERATOR_HELPER_KIND_SOME ), + JS_CFUNC_DEF("reduce", 1, js_iterator_proto_reduce ), + JS_CFUNC_DEF("toArray", 0, js_iterator_proto_toArray ), JS_CFUNC_DEF("[Symbol.iterator]", 0, js_iterator_proto_iterator ), + JS_CGETSET_DEF("[Symbol.toStringTag]", js_iterator_proto_get_toStringTag, js_iterator_proto_set_toStringTag), +}; + +static const JSCFunctionListEntry js_iterator_helper_proto_funcs[] = { + JS_ITERATOR_NEXT_DEF("next", 0, js_iterator_helper_next, GEN_MAGIC_NEXT ), + JS_ITERATOR_NEXT_DEF("return", 0, js_iterator_helper_next, GEN_MAGIC_RETURN ), + JS_PROP_STRING_DEF("[Symbol.toStringTag]", "Iterator Helper", JS_PROP_CONFIGURABLE ), +}; + +static const JSCFunctionListEntry js_array_unscopables_funcs[] = { + JS_PROP_BOOL_DEF("at", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("copyWithin", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("entries", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("fill", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("find", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("findIndex", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("findLast", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("findLastIndex", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("flat", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("flatMap", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("includes", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("keys", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("toReversed", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("toSorted", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("toSpliced", TRUE, JS_PROP_C_W_E), + JS_PROP_BOOL_DEF("values", TRUE, JS_PROP_C_W_E), }; static const JSCFunctionListEntry js_array_proto_funcs[] = { @@ -41548,6 +43652,7 @@ static const JSCFunctionListEntry js_array_proto_funcs[] = { JS_ALIAS_DEF("[Symbol.iterator]", "values" ), JS_CFUNC_MAGIC_DEF("keys", 0, js_create_array_iterator, JS_ITERATOR_KIND_KEY ), JS_CFUNC_MAGIC_DEF("entries", 0, js_create_array_iterator, JS_ITERATOR_KIND_KEY_AND_VALUE ), + JS_OBJECT_DEF("[Symbol.unscopables]", js_array_unscopables_funcs, countof(js_array_unscopables_funcs), JS_PROP_CONFIGURABLE ), }; static const JSCFunctionListEntry js_array_iterator_proto_funcs[] = { @@ -43171,12 +45276,6 @@ static JSValue js_string_trim(JSContext *ctx, JSValueConst this_val, return ret; } -static JSValue js_string___quote(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv) -{ - return JS_ToQuotedString(ctx, this_val); -} - /* return 0 if before the first char */ static int string_prevc(JSString *p, int *pidx) { @@ -43664,7 +45763,6 @@ static const JSCFunctionListEntry js_string_proto_funcs[] = { JS_ALIAS_DEF("trimLeft", "trimStart" ), JS_CFUNC_DEF("toString", 0, js_string_toString ), JS_CFUNC_DEF("valueOf", 0, js_string_toString ), - JS_CFUNC_DEF("__quote", 1, js_string___quote ), JS_CFUNC_MAGIC_DEF("toLowerCase", 0, js_string_toLowerCase, 1 ), JS_CFUNC_MAGIC_DEF("toUpperCase", 0, js_string_toLowerCase, 0 ), JS_CFUNC_MAGIC_DEF("toLocaleLowerCase", 0, js_string_toLowerCase, 1 ), @@ -43698,10 +45796,10 @@ static const JSCFunctionListEntry js_string_proto_normalize[] = { JS_CFUNC_DEF("localeCompare", 1, js_string_localeCompare ), }; -void JS_AddIntrinsicStringNormalize(JSContext *ctx) +int JS_AddIntrinsicStringNormalize(JSContext *ctx) { - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_STRING], js_string_proto_normalize, - countof(js_string_proto_normalize)); + return JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_STRING], js_string_proto_normalize, + countof(js_string_proto_normalize)); } /* Math */ @@ -43849,6 +45947,11 @@ static JSValue js_math_hypot(JSContext *ctx, JSValueConst this_val, return JS_NewFloat64(ctx, r); } +static double js_math_f16round(double a) +{ + return fromfp16(tofp16(a)); +} + static double js_math_fround(double a) { return (float)a; @@ -43883,6 +45986,261 @@ static JSValue js_math_clz32(JSContext *ctx, JSValueConst this_val, return JS_NewInt32(ctx, r); } +typedef enum { + SUM_PRECISE_STATE_FINITE, + SUM_PRECISE_STATE_INFINITY, + SUM_PRECISE_STATE_MINUS_INFINITY, /* must be after SUM_PRECISE_STATE_INFINITY */ + SUM_PRECISE_STATE_NAN, /* must be after SUM_PRECISE_STATE_MINUS_INFINITY */ +} SumPreciseStateEnum; + +#define SP_LIMB_BITS 56 +#define SP_RND_BITS (SP_LIMB_BITS - 53) +/* we add one extra limb to avoid having to test for overflows during the sum */ +#define SUM_PRECISE_ACC_LEN 39 + +#define SUM_PRECISE_COUNTER_INIT 250 + +typedef struct { + SumPreciseStateEnum state; + uint32_t counter; + int n_limbs; /* 'acc' contains n_limbs and is not necessarily + acc[n_limb - 1] may be 0. 0 indicates minus zero + result when state = SUM_PRECISE_STATE_FINITE */ + int64_t acc[SUM_PRECISE_ACC_LEN]; +} SumPreciseState; + +static void sum_precise_init(SumPreciseState *s) +{ + memset(s->acc, 0, sizeof(s->acc)); + s->state = SUM_PRECISE_STATE_FINITE; + s->counter = SUM_PRECISE_COUNTER_INIT; + s->n_limbs = 0; +} + +static void sum_precise_renorm(SumPreciseState *s) +{ + int64_t v, carry; + int i; + + carry = 0; + for(i = 0; i < s->n_limbs; i++) { + v = s->acc[i] + carry; + s->acc[i] = v & (((uint64_t)1 << SP_LIMB_BITS) - 1); + carry = v >> SP_LIMB_BITS; + } + /* we add a failsafe but it should be never reached in a + reasonnable amount of time */ + if (carry != 0 && s->n_limbs < SUM_PRECISE_ACC_LEN) + s->acc[s->n_limbs++] = carry; +} + +static void sum_precise_add(SumPreciseState *s, double d) +{ + uint64_t a, m, a0, a1; + int sgn, e, p; + unsigned int shift; + + a = float64_as_uint64(d); + sgn = a >> 63; + e = (a >> 52) & ((1 << 11) - 1); + m = a & (((uint64_t)1 << 52) - 1); + if (unlikely(e == 2047)) { + if (m == 0) { + /* +/- infinity */ + if (s->state == SUM_PRECISE_STATE_NAN || + (s->state == SUM_PRECISE_STATE_MINUS_INFINITY && !sgn) || + (s->state == SUM_PRECISE_STATE_INFINITY && sgn)) { + s->state = SUM_PRECISE_STATE_NAN; + } else { + s->state = SUM_PRECISE_STATE_INFINITY + sgn; + } + } else { + /* NaN */ + s->state = SUM_PRECISE_STATE_NAN; + } + } else if (e == 0) { + if (likely(m == 0)) { + /* zero */ + if (s->n_limbs == 0 && !sgn) + s->n_limbs = 1; + } else { + /* subnormal */ + p = 0; + shift = 0; + goto add; + } + } else { + /* Note: we sum even if state != SUM_PRECISE_STATE_FINITE to + avoid tests */ + m |= (uint64_t)1 << 52; + shift = e - 1; + /* 'p' is the position of a0 in acc. The division is normally + implementation as a multiplication by the compiler. */ + p = shift / SP_LIMB_BITS; + shift %= SP_LIMB_BITS; + add: + a0 = (m << shift) & (((uint64_t)1 << SP_LIMB_BITS) - 1); + a1 = m >> (SP_LIMB_BITS - shift); + if (!sgn) { + s->acc[p] += a0; + s->acc[p + 1] += a1; + } else { + s->acc[p] -= a0; + s->acc[p + 1] -= a1; + } + s->n_limbs = max_int(s->n_limbs, p + 2); + + if (unlikely(--s->counter == 0)) { + s->counter = SUM_PRECISE_COUNTER_INIT; + sum_precise_renorm(s); + } + } +} + +static double sum_precise_get_result(SumPreciseState *s) +{ + int n, shift, e, p, is_neg; + uint64_t m, addend; + + if (s->state != SUM_PRECISE_STATE_FINITE) { + switch(s->state) { + default: + case SUM_PRECISE_STATE_INFINITY: + return INFINITY; + case SUM_PRECISE_STATE_MINUS_INFINITY: + return -INFINITY; + case SUM_PRECISE_STATE_NAN: + return NAN; + } + } + + sum_precise_renorm(s); + + /* extract the sign and absolute value */ +#if 0 + { + int i; + printf("len=%d:", s->n_limbs); + for(i = s->n_limbs - 1; i >= 0; i--) + printf(" %014lx", s->acc[i]); + printf("\n"); + } +#endif + n = s->n_limbs; + /* minus zero result */ + if (n == 0) + return -0.0; + + /* normalize */ + while (n > 0 && s->acc[n - 1] == 0) + n--; + /* zero result. The spec tells it is always positive in the finite case */ + if (n == 0) + return 0.0; + is_neg = (s->acc[n - 1] < 0); + if (is_neg) { + uint64_t v, carry; + int i; + /* negate */ + /* XXX: do it only when needed */ + carry = 1; + for(i = 0; i < n - 1; i++) { + v = (((uint64_t)1 << SP_LIMB_BITS) - 1) - s->acc[i] + carry; + carry = v >> SP_LIMB_BITS; + s->acc[i] = v & (((uint64_t)1 << SP_LIMB_BITS) - 1); + } + s->acc[n - 1] = -s->acc[n - 1] + carry - 1; + while (n > 1 && s->acc[n - 1] == 0) + n--; + } + /* subnormal case */ + if (n == 1 && s->acc[0] < ((uint64_t)1 << 52)) + return uint64_as_float64(((uint64_t)is_neg << 63) | s->acc[0]); + /* normal case */ + e = n * SP_LIMB_BITS; + p = n - 1; + m = s->acc[p]; + shift = clz64(m) - (64 - SP_LIMB_BITS); + e = e - shift - 52; + if (shift != 0) { + m <<= shift; + if (p > 0) { + int shift1; + uint64_t nz; + p--; + shift1 = SP_LIMB_BITS - shift; + nz = s->acc[p] & (((uint64_t)1 << shift1) - 1); + m = m | (s->acc[p] >> shift1) | (nz != 0); + } + } + if ((m & ((1 << SP_RND_BITS) - 1)) == (1 << (SP_RND_BITS - 1))) { + /* see if the LSB part is non zero for the final rounding */ + while (p > 0) { + p--; + if (s->acc[p] != 0) { + m |= 1; + break; + } + } + } + /* rounding to nearest with ties to even */ + addend = (1 << (SP_RND_BITS - 1)) - 1 + ((m >> SP_RND_BITS) & 1); + m = (m + addend) >> SP_RND_BITS; + /* handle overflow in the rounding */ + if (m == ((uint64_t)1 << 53)) + e++; + if (unlikely(e >= 2047)) { + /* infinity */ + return uint64_as_float64(((uint64_t)is_neg << 63) | ((uint64_t)2047 << 52)); + } else { + m &= (((uint64_t)1 << 52) - 1); + return uint64_as_float64(((uint64_t)is_neg << 63) | ((uint64_t)e << 52) | m); + } +} + +static JSValue js_math_sumPrecise(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue iter, next, item, ret; + uint32_t tag; + int done; + double d; + SumPreciseState s_s, *s = &s_s; + + iter = JS_GetIterator(ctx, argv[0], FALSE); + if (JS_IsException(iter)) + return JS_EXCEPTION; + ret = JS_EXCEPTION; + next = JS_GetProperty(ctx, iter, JS_ATOM_next); + if (JS_IsException(next)) + goto fail; + sum_precise_init(s); + for (;;) { + item = JS_IteratorNext(ctx, iter, next, 0, NULL, &done); + if (JS_IsException(item)) + goto fail; + if (done) + break; + tag = JS_VALUE_GET_TAG(item); + if (JS_TAG_IS_FLOAT64(tag)) { + d = JS_VALUE_GET_FLOAT64(item); + } else if (tag == JS_TAG_INT) { + d = JS_VALUE_GET_INT(item); + } else { + JS_FreeValue(ctx, item); + JS_ThrowTypeError(ctx, "not a number"); + JS_IteratorClose(ctx, iter, TRUE); + goto fail; + } + sum_precise_add(s, d); + } + ret = __JS_NewFloat64(ctx, sum_precise_get_result(s)); +fail: + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, next); + return ret; +} + /* xorshift* random number generator by Marsaglia */ static uint64_t xorshift64star(uint64_t *pstate) { @@ -43952,9 +46310,11 @@ static const JSCFunctionListEntry js_math_funcs[] = { JS_CFUNC_SPECIAL_DEF("cbrt", 1, f_f, cbrt ), JS_CFUNC_DEF("hypot", 2, js_math_hypot ), JS_CFUNC_DEF("random", 0, js_math_random ), + JS_CFUNC_SPECIAL_DEF("f16round", 1, f_f, js_math_f16round ), JS_CFUNC_SPECIAL_DEF("fround", 1, f_f, js_math_fround ), JS_CFUNC_DEF("imul", 2, js_math_imul ), JS_CFUNC_DEF("clz32", 1, js_math_clz32 ), + JS_CFUNC_DEF("sumPrecise", 1, js_math_sumPrecise ), JS_PROP_STRING_DEF("[Symbol.toStringTag]", "Math", JS_PROP_CONFIGURABLE ), JS_PROP_DOUBLE_DEF("E", 2.718281828459045, 0 ), JS_PROP_DOUBLE_DEF("LN10", 2.302585092994046, 0 ), @@ -44007,9 +46367,13 @@ static int getTimezoneOffset(int64_t time) time_t gm_ti, loc_ti; tm = gmtime(&ti); + if (!tm) + return 0; gm_ti = mktime(tm); tm = localtime(&ti); + if (!tm) + return 0; loc_ti = mktime(tm); res = (gm_ti - loc_ti) / 60; @@ -44074,8 +46438,10 @@ static void js_regexp_finalizer(JSRuntime *rt, JSValue val) { JSObject *p = JS_VALUE_GET_OBJ(val); JSRegExp *re = &p->u.regexp; - JS_FreeValueRT(rt, JS_MKPTR(JS_TAG_STRING, re->bytecode)); - JS_FreeValueRT(rt, JS_MKPTR(JS_TAG_STRING, re->pattern)); + if (re->bytecode != NULL) + JS_FreeValueRT(rt, JS_MKPTR(JS_TAG_STRING, re->bytecode)); + if (re->pattern != NULL) + JS_FreeValueRT(rt, JS_MKPTR(JS_TAG_STRING, re->pattern)); } /* create a string containing the RegExp bytecode */ @@ -44116,6 +46482,9 @@ static JSValue js_compile_regexp(JSContext *ctx, JSValueConst pattern, case 'u': mask = LRE_FLAG_UNICODE; break; + case 'v': + mask = LRE_FLAG_UNICODE_SETS; + break; case 'y': mask = LRE_FLAG_STICKY; break; @@ -44125,14 +46494,20 @@ static JSValue js_compile_regexp(JSContext *ctx, JSValueConst pattern, if ((re_flags & mask) != 0) { bad_flags: JS_FreeCString(ctx, str); - return JS_ThrowSyntaxError(ctx, "invalid regular expression flags"); + goto bad_flags1; } re_flags |= mask; } JS_FreeCString(ctx, str); } - str = JS_ToCStringLen2(ctx, &len, pattern, !(re_flags & LRE_FLAG_UNICODE)); + /* 'u' and 'v' cannot be both set */ + if ((re_flags & LRE_FLAG_UNICODE_SETS) && (re_flags & LRE_FLAG_UNICODE)) { + bad_flags1: + return JS_ThrowSyntaxError(ctx, "invalid regular expression flags"); + } + + str = JS_ToCStringLen2(ctx, &len, pattern, !(re_flags & (LRE_FLAG_UNICODE | LRE_FLAG_UNICODE_SETS))); if (!str) return JS_EXCEPTION; re_bytecode_buf = lre_compile(&re_bytecode_len, error_msg, @@ -44148,32 +46523,29 @@ static JSValue js_compile_regexp(JSContext *ctx, JSValueConst pattern, return ret; } -/* create a RegExp object from a string containing the RegExp bytecode - and the source pattern */ -static JSValue js_regexp_constructor_internal(JSContext *ctx, JSValueConst ctor, - JSValue pattern, JSValue bc) +/* set the RegExp fields */ +static JSValue js_regexp_set_internal(JSContext *ctx, + JSValue obj, + JSValue pattern, JSValue bc) { - JSValue obj; JSObject *p; JSRegExp *re; /* sanity check */ - if (JS_VALUE_GET_TAG(bc) != JS_TAG_STRING || - JS_VALUE_GET_TAG(pattern) != JS_TAG_STRING) { + if (unlikely(JS_VALUE_GET_TAG(bc) != JS_TAG_STRING || + JS_VALUE_GET_TAG(pattern) != JS_TAG_STRING)) { JS_ThrowTypeError(ctx, "string expected"); - fail: + JS_FreeValue(ctx, obj); JS_FreeValue(ctx, bc); JS_FreeValue(ctx, pattern); return JS_EXCEPTION; } - obj = js_create_from_ctor(ctx, ctor, JS_CLASS_REGEXP); - if (JS_IsException(obj)) - goto fail; p = JS_VALUE_GET_OBJ(obj); re = &p->u.regexp; re->pattern = JS_VALUE_GET_STRING(pattern); re->bytecode = JS_VALUE_GET_STRING(bc); + /* Note: cannot fail because the field is preallocated */ JS_DefinePropertyValue(ctx, obj, JS_ATOM_lastIndex, JS_NewInt32(ctx, 0), JS_PROP_WRITABLE); return obj; @@ -44210,7 +46582,7 @@ static int js_is_regexp(JSContext *ctx, JSValueConst obj) static JSValue js_regexp_constructor(JSContext *ctx, JSValueConst new_target, int argc, JSValueConst *argv) { - JSValue pattern, flags, bc, val; + JSValue pattern, flags, bc, val, obj = JS_UNDEFINED; JSValueConst pat, flags1; JSRegExp *re; int pat_is_regexp; @@ -44236,18 +46608,19 @@ static JSValue js_regexp_constructor(JSContext *ctx, JSValueConst new_target, } } re = js_get_regexp(ctx, pat, FALSE); + flags = JS_UNDEFINED; if (re) { pattern = JS_DupValue(ctx, JS_MKPTR(JS_TAG_STRING, re->pattern)); if (JS_IsUndefined(flags1)) { bc = JS_DupValue(ctx, JS_MKPTR(JS_TAG_STRING, re->bytecode)); + obj = js_create_from_ctor(ctx, new_target, JS_CLASS_REGEXP); + if (JS_IsException(obj)) + goto fail; goto no_compilation; } else { - flags = JS_ToString(ctx, flags1); - if (JS_IsException(flags)) - goto fail; + flags = JS_DupValue(ctx, flags1); } } else { - flags = JS_UNDEFINED; if (pat_is_regexp) { pattern = JS_GetProperty(ctx, pat, JS_ATOM_source); if (JS_IsException(pattern)) @@ -44273,15 +46646,19 @@ static JSValue js_regexp_constructor(JSContext *ctx, JSValueConst new_target, goto fail; } } + obj = js_create_from_ctor(ctx, new_target, JS_CLASS_REGEXP); + if (JS_IsException(obj)) + goto fail; bc = js_compile_regexp(ctx, pattern, flags); if (JS_IsException(bc)) goto fail; JS_FreeValue(ctx, flags); no_compilation: - return js_regexp_constructor_internal(ctx, new_target, pattern, bc); + return js_regexp_set_internal(ctx, obj, pattern, bc); fail: JS_FreeValue(ctx, pattern); JS_FreeValue(ctx, flags); + JS_FreeValue(ctx, obj); return JS_EXCEPTION; } @@ -44436,49 +46813,34 @@ static JSValue js_regexp_get_flag(JSContext *ctx, JSValueConst this_val, int mas return JS_NewBool(ctx, flags & mask); } +#define RE_FLAG_COUNT 8 + static JSValue js_regexp_get_flags(JSContext *ctx, JSValueConst this_val) { - char str[8], *p = str; - int res; - + char str[RE_FLAG_COUNT], *p = str; + int res, i; + static const int flag_atom[RE_FLAG_COUNT] = { + JS_ATOM_hasIndices, + JS_ATOM_global, + JS_ATOM_ignoreCase, + JS_ATOM_multiline, + JS_ATOM_dotAll, + JS_ATOM_unicode, + JS_ATOM_unicodeSets, + JS_ATOM_sticky, + }; + static const char flag_char[RE_FLAG_COUNT] = { 'd', 'g', 'i', 'm', 's', 'u', 'v', 'y' }; + if (JS_VALUE_GET_TAG(this_val) != JS_TAG_OBJECT) return JS_ThrowTypeErrorNotAnObject(ctx); - res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "hasIndices")); - if (res < 0) - goto exception; - if (res) - *p++ = 'd'; - res = JS_ToBoolFree(ctx, JS_GetProperty(ctx, this_val, JS_ATOM_global)); - if (res < 0) - goto exception; - if (res) - *p++ = 'g'; - res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "ignoreCase")); - if (res < 0) - goto exception; - if (res) - *p++ = 'i'; - res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "multiline")); - if (res < 0) - goto exception; - if (res) - *p++ = 'm'; - res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "dotAll")); - if (res < 0) - goto exception; - if (res) - *p++ = 's'; - res = JS_ToBoolFree(ctx, JS_GetProperty(ctx, this_val, JS_ATOM_unicode)); - if (res < 0) - goto exception; - if (res) - *p++ = 'u'; - res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "sticky")); - if (res < 0) - goto exception; - if (res) - *p++ = 'y'; + for(i = 0; i < RE_FLAG_COUNT; i++) { + res = JS_ToBoolFree(ctx, JS_GetProperty(ctx, this_val, flag_atom[i])); + if (res < 0) + goto exception; + if (res) + *p++ = flag_char[i]; + } return JS_NewStringLen(ctx, str, p - str); exception: @@ -44531,6 +46893,58 @@ void *lre_realloc(void *opaque, void *ptr, size_t size) return js_realloc_rt(ctx->rt, ptr, size); } +static JSValue js_regexp_escape(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue str; + StringBuffer b_s, *b = &b_s; + JSString *p; + uint32_t c, i; + char s[16]; + + if (!JS_IsString(argv[0])) + return JS_ThrowTypeError(ctx, "not a string"); + str = JS_ToString(ctx, argv[0]); /* must call it to linearlize ropes */ + if (JS_IsException(str)) + return JS_EXCEPTION; + p = JS_VALUE_GET_STRING(str); + string_buffer_init2(ctx, b, 0, p->is_wide_char); + for (i = 0; i < p->len; i++) { + c = string_get(p, i); + if (c < 33) { + if (c >= 9 && c <= 13) { + string_buffer_putc8(b, '\\'); + string_buffer_putc8(b, "tnvfr"[c - 9]); + } else { + goto hex2; + } + } else if (c < 128) { + if ((c >= '0' && c <= '9') + || (c >= 'A' && c <= 'Z') + || (c >= 'a' && c <= 'z')) { + if (i == 0) + goto hex2; + } else if (strchr(",-=<>#&!%:;@~'`\"", c)) { + goto hex2; + } else if (c != '_') { + string_buffer_putc8(b, '\\'); + } + string_buffer_putc8(b, c); + } else if (c < 256) { + hex2: + snprintf(s, sizeof(s), "\\x%02x", c); + string_buffer_puts8(b, s); + } else if (is_surrogate(c) || lre_is_space(c)) { + snprintf(s, sizeof(s), "\\u%04x", c); + string_buffer_puts8(b, s); + } else { + string_buffer_putc16(b, c); + } + } + JS_FreeValue(ctx, str); + return string_buffer_end(b); +} + static JSValue js_regexp_exec(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { @@ -44911,14 +47325,12 @@ static JSValue js_regexp_Symbol_match(JSContext *ctx, JSValueConst this_val, goto exception; p = JS_VALUE_GET_STRING(flags); - // TODO(bnoordhuis) query 'u' flag the same way? global = (-1 != string_indexof_char(p, 'g', 0)); if (!global) { A = JS_RegExpExec(ctx, rx, S); } else { - fullUnicode = JS_ToBoolFree(ctx, JS_GetProperty(ctx, rx, JS_ATOM_unicode)); - if (fullUnicode < 0) - goto exception; + fullUnicode = (string_indexof_char(p, 'u', 0) >= 0 || + string_indexof_char(p, 'v', 0) >= 0); if (JS_SetProperty(ctx, rx, JS_ATOM_lastIndex, JS_NewInt32(ctx, 0)) < 0) goto exception; @@ -44937,7 +47349,7 @@ static JSValue js_regexp_Symbol_match(JSContext *ctx, JSValueConst this_val, if (JS_IsException(matchStr)) goto exception; isEmpty = JS_IsEmptyString(matchStr); - if (JS_SetPropertyInt64(ctx, A, n++, matchStr) < 0) + if (JS_DefinePropertyValueInt64(ctx, A, n++, matchStr, JS_PROP_C_W_E | JS_PROP_THROW) < 0) goto exception; if (isEmpty) { int64_t thisIndex, nextIndex; @@ -45102,7 +47514,8 @@ static JSValue js_regexp_Symbol_matchAll(JSContext *ctx, JSValueConst this_val, it->iterated_string = S; strp = JS_VALUE_GET_STRING(flags); it->global = string_indexof_char(strp, 'g', 0) >= 0; - it->unicode = string_indexof_char(strp, 'u', 0) >= 0; + it->unicode = (string_indexof_char(strp, 'u', 0) >= 0 || + string_indexof_char(strp, 'v', 0) >= 0); it->done = FALSE; JS_SetOpaque(iter, it); @@ -45249,13 +47662,11 @@ static JSValue js_regexp_Symbol_replace(JSContext *ctx, JSValueConst this_val, goto exception; p = JS_VALUE_GET_STRING(flags); - // TODO(bnoordhuis) query 'u' flag the same way? fullUnicode = 0; is_global = (-1 != string_indexof_char(p, 'g', 0)); if (is_global) { - fullUnicode = JS_ToBoolFree(ctx, JS_GetProperty(ctx, rx, JS_ATOM_unicode)); - if (fullUnicode < 0) - goto exception; + fullUnicode = (string_indexof_char(p, 'u', 0) >= 0 || + string_indexof_char(p, 'v', 0) >= 0); if (JS_SetProperty(ctx, rx, JS_ATOM_lastIndex, JS_NewInt32(ctx, 0)) < 0) goto exception; } @@ -45481,7 +47892,8 @@ static JSValue js_regexp_Symbol_split(JSContext *ctx, JSValueConst this_val, if (JS_IsException(flags)) goto exception; strp = JS_VALUE_GET_STRING(flags); - unicodeMatching = string_indexof_char(strp, 'u', 0) >= 0; + unicodeMatching = (string_indexof_char(strp, 'u', 0) >= 0 || + string_indexof_char(strp, 'v', 0) >= 0); if (string_indexof_char(strp, 'y', 0) < 0) { flags = JS_ConcatString3(ctx, "", flags, "y"); if (JS_IsException(flags)) @@ -45578,6 +47990,7 @@ static JSValue js_regexp_Symbol_split(JSContext *ctx, JSValueConst this_val, } static const JSCFunctionListEntry js_regexp_funcs[] = { + JS_CFUNC_DEF("escape", 1, js_regexp_escape ), JS_CGETSET_DEF("[Symbol.species]", js_get_this, NULL ), //JS_CFUNC_DEF("__RegExpExec", 2, js_regexp___RegExpExec ), //JS_CFUNC_DEF("__RegExpDelete", 2, js_regexp___RegExpDelete ), @@ -45591,6 +48004,7 @@ static const JSCFunctionListEntry js_regexp_proto_funcs[] = { JS_CGETSET_MAGIC_DEF("multiline", js_regexp_get_flag, NULL, LRE_FLAG_MULTILINE ), JS_CGETSET_MAGIC_DEF("dotAll", js_regexp_get_flag, NULL, LRE_FLAG_DOTALL ), JS_CGETSET_MAGIC_DEF("unicode", js_regexp_get_flag, NULL, LRE_FLAG_UNICODE ), + JS_CGETSET_MAGIC_DEF("unicodeSets", js_regexp_get_flag, NULL, LRE_FLAG_UNICODE_SETS ), JS_CGETSET_MAGIC_DEF("sticky", js_regexp_get_flag, NULL, LRE_FLAG_STICKY ), JS_CGETSET_MAGIC_DEF("hasIndices", js_regexp_get_flag, NULL, LRE_FLAG_INDICES ), JS_CFUNC_DEF("exec", 1, js_regexp_exec ), @@ -45616,25 +48030,29 @@ void JS_AddIntrinsicRegExpCompiler(JSContext *ctx) ctx->compile_regexp = js_compile_regexp; } -void JS_AddIntrinsicRegExp(JSContext *ctx) +int JS_AddIntrinsicRegExp(JSContext *ctx) { - JSValueConst obj; + JSValue obj; JS_AddIntrinsicRegExpCompiler(ctx); - ctx->class_proto[JS_CLASS_REGEXP] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_REGEXP], js_regexp_proto_funcs, - countof(js_regexp_proto_funcs)); - obj = JS_NewGlobalCConstructor(ctx, "RegExp", js_regexp_constructor, 2, - ctx->class_proto[JS_CLASS_REGEXP]); - ctx->regexp_ctor = JS_DupValue(ctx, obj); - JS_SetPropertyFunctionList(ctx, obj, js_regexp_funcs, countof(js_regexp_funcs)); - + obj = JS_NewCConstructor(ctx, JS_CLASS_REGEXP, "RegExp", + js_regexp_constructor, 2, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_regexp_funcs, countof(js_regexp_funcs), + js_regexp_proto_funcs, countof(js_regexp_proto_funcs), + 0); + if (JS_IsException(obj)) + return -1; + ctx->regexp_ctor = obj; + ctx->class_proto[JS_CLASS_REGEXP_STRING_ITERATOR] = - JS_NewObjectProto(ctx, ctx->iterator_proto); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_REGEXP_STRING_ITERATOR], - js_regexp_string_iterator_proto_funcs, - countof(js_regexp_string_iterator_proto_funcs)); + JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_ITERATOR], + js_regexp_string_iterator_proto_funcs, + countof(js_regexp_string_iterator_proto_funcs)); + if (JS_IsException(ctx->class_proto[JS_CLASS_REGEXP_STRING_ITERATOR])) + return -1; + return 0; } /* JSON */ @@ -45753,6 +48171,12 @@ static JSValue json_parse_value(JSParseState *s) val = JS_NewBool(ctx, s->token.u.ident.atom == JS_ATOM_true); } else if (s->token.u.ident.atom == JS_ATOM_null) { val = JS_NULL; + } else if (s->token.u.ident.atom == JS_ATOM_NaN && s->ext_json) { + /* Note: json5 identifier handling is ambiguous e.g. is + '{ NaN: 1 }' a valid JSON5 production ? */ + val = JS_NewFloat64(s->ctx, NAN); + } else if (s->token.u.ident.atom == JS_ATOM_Infinity && s->ext_json) { + val = JS_NewFloat64(s->ctx, INFINITY); } else { goto def_token; } @@ -45857,7 +48281,7 @@ static JSValue internalize_json_property(JSContext *ctx, JSValueConst holder, goto fail; } } - js_free_prop_enum(ctx, atoms, len); + JS_FreePropertyEnum(ctx, atoms, len); atoms = NULL; name_val = JS_AtomToValue(ctx, name); if (JS_IsException(name_val)) @@ -45869,7 +48293,7 @@ static JSValue internalize_json_property(JSContext *ctx, JSValueConst holder, JS_FreeValue(ctx, val); return res; fail: - js_free_prop_enum(ctx, atoms, len); + JS_FreePropertyEnum(ctx, atoms, len); JS_FreeValue(ctx, val); return JS_EXCEPTION; } @@ -45917,10 +48341,72 @@ typedef struct JSONStringifyContext { StringBuffer *b; } JSONStringifyContext; -static JSValue JS_ToQuotedStringFree(JSContext *ctx, JSValue val) { - JSValue r = JS_ToQuotedString(ctx, val); +static int JS_ToQuotedString(JSContext *ctx, StringBuffer *b, JSValueConst val1) +{ + JSValue val; + JSString *p; + int i; + uint32_t c; + char buf[16]; + + val = JS_ToStringCheckObject(ctx, val1); + if (JS_IsException(val)) + return -1; + p = JS_VALUE_GET_STRING(val); + + if (string_buffer_putc8(b, '\"')) + goto fail; + for(i = 0; i < p->len; ) { + c = string_getc(p, &i); + switch(c) { + case '\t': + c = 't'; + goto quote; + case '\r': + c = 'r'; + goto quote; + case '\n': + c = 'n'; + goto quote; + case '\b': + c = 'b'; + goto quote; + case '\f': + c = 'f'; + goto quote; + case '\"': + case '\\': + quote: + if (string_buffer_putc8(b, '\\')) + goto fail; + if (string_buffer_putc8(b, c)) + goto fail; + break; + default: + if (c < 32 || is_surrogate(c)) { + snprintf(buf, sizeof(buf), "\\u%04x", c); + if (string_buffer_puts8(b, buf)) + goto fail; + } else { + if (string_buffer_putc(b, c)) + goto fail; + } + break; + } + } + if (string_buffer_putc8(b, '\"')) + goto fail; + JS_FreeValue(ctx, val); + return 0; + fail: + JS_FreeValue(ctx, val); + return -1; +} + +static int JS_ToQuotedStringFree(JSContext *ctx, StringBuffer *b, JSValue val) { + int ret = JS_ToQuotedString(ctx, b, val); JS_FreeValue(ctx, val); - return r; + return ret; } static JSValue js_json_check(JSContext *ctx, JSONStringifyContext *jsc, @@ -46103,13 +48589,11 @@ static int js_json_to_str(JSContext *ctx, JSONStringifyContext *jsc, if (!JS_IsUndefined(v)) { if (has_content) string_buffer_putc8(jsc->b, ','); - prop = JS_ToQuotedStringFree(ctx, prop); - if (JS_IsException(prop)) { + string_buffer_concat_value(jsc->b, sep); + if (JS_ToQuotedString(ctx, jsc->b, prop)) { JS_FreeValue(ctx, v); goto exception; } - string_buffer_concat_value(jsc->b, sep); - string_buffer_concat_value(jsc->b, prop); string_buffer_putc8(jsc->b, ':'); string_buffer_concat_value(jsc->b, sep1); if (js_json_to_str(ctx, jsc, val, v, indent1)) @@ -46137,10 +48621,7 @@ static int js_json_to_str(JSContext *ctx, JSONStringifyContext *jsc, switch (JS_VALUE_GET_NORM_TAG(val)) { case JS_TAG_STRING: case JS_TAG_STRING_ROPE: - val = JS_ToQuotedStringFree(ctx, val); - if (JS_IsException(val)) - goto exception; - goto concat_value; + return JS_ToQuotedStringFree(ctx, jsc->b, val); case JS_TAG_FLOAT64: if (!isfinite(JS_VALUE_GET_FLOAT64(val))) { val = JS_NULL; @@ -46322,10 +48803,10 @@ static const JSCFunctionListEntry js_json_obj[] = { JS_OBJECT_DEF("JSON", js_json_funcs, countof(js_json_funcs), JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE ), }; -void JS_AddIntrinsicJSON(JSContext *ctx) +int JS_AddIntrinsicJSON(JSContext *ctx) { /* add JSON as autoinit object */ - JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_json_obj, countof(js_json_obj)); + return JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_json_obj, countof(js_json_obj)); } /* Reflect */ @@ -46348,7 +48829,7 @@ static JSValue js_reflect_construct(JSContext *ctx, JSValueConst this_val, if (argc > 2) { new_target = argv[2]; if (!JS_IsConstructor(ctx, new_target)) - return JS_ThrowTypeError(ctx, "not a constructor"); + return JS_ThrowTypeErrorNotAConstructor(ctx, new_target); } else { new_target = func; } @@ -46579,7 +49060,7 @@ static JSValue js_proxy_get_prototype(JSContext *ctx, JSValueConst obj) JS_FreeValue(ctx, ret); return JS_EXCEPTION; } - if (JS_VALUE_GET_OBJ(proto1) != JS_VALUE_GET_OBJ(ret)) { + if (!js_same_value(ctx, proto1, ret)) { JS_FreeValue(ctx, proto1); fail: JS_FreeValue(ctx, ret); @@ -46619,7 +49100,7 @@ static int js_proxy_set_prototype(JSContext *ctx, JSValueConst obj, proto1 = JS_GetPrototype(ctx, s->target); if (JS_IsException(proto1)) return -1; - if (JS_VALUE_GET_OBJ(proto_val) != JS_VALUE_GET_OBJ(proto1)) { + if (!js_same_value(ctx, proto_val, proto1)) { JS_FreeValue(ctx, proto1); JS_ThrowTypeError(ctx, "proxy: inconsistent prototype"); return -1; @@ -47243,14 +49724,14 @@ static int js_proxy_get_own_property_names(JSContext *ctx, } } - js_free_prop_enum(ctx, tab2, len2); + JS_FreePropertyEnum(ctx, tab2, len2); JS_FreeValue(ctx, prop_array); *ptab = tab; *plen = len; return 0; fail: - js_free_prop_enum(ctx, tab2, len2); - js_free_prop_enum(ctx, tab, len); + JS_FreePropertyEnum(ctx, tab2, len2); + JS_FreePropertyEnum(ctx, tab, len); JS_FreeValue(ctx, prop_array); return -1; } @@ -47267,7 +49748,7 @@ static JSValue js_proxy_call_constructor(JSContext *ctx, JSValueConst func_obj, if (!s) return JS_EXCEPTION; if (!JS_IsConstructor(ctx, s->target)) - return JS_ThrowTypeError(ctx, "not a constructor"); + return JS_ThrowTypeErrorNotAConstructor(ctx, s->target); if (JS_IsUndefined(method)) return JS_CallConstructor2(ctx, s->target, new_target, argc, argv); arg_array = js_create_array(ctx, argc, argv); @@ -47452,25 +49933,36 @@ static const JSClassShortDef js_proxy_class_def[] = { { JS_ATOM_Object, js_proxy_finalizer, js_proxy_mark }, /* JS_CLASS_PROXY */ }; -void JS_AddIntrinsicProxy(JSContext *ctx) +int JS_AddIntrinsicProxy(JSContext *ctx) { JSRuntime *rt = ctx->rt; JSValue obj1; if (!JS_IsRegisteredClass(rt, JS_CLASS_PROXY)) { - init_class_range(rt, js_proxy_class_def, JS_CLASS_PROXY, - countof(js_proxy_class_def)); + if (init_class_range(rt, js_proxy_class_def, JS_CLASS_PROXY, + countof(js_proxy_class_def))) + return -1; rt->class_array[JS_CLASS_PROXY].exotic = &js_proxy_exotic_methods; rt->class_array[JS_CLASS_PROXY].call = js_proxy_call; } - obj1 = JS_NewCFunction2(ctx, js_proxy_constructor, "Proxy", 2, - JS_CFUNC_constructor, 0); + /* additional fields: name, length */ + obj1 = JS_NewCFunction3(ctx, js_proxy_constructor, "Proxy", 2, + JS_CFUNC_constructor, 0, + ctx->function_proto, countof(js_proxy_funcs) + 2); + if (JS_IsException(obj1)) + return -1; JS_SetConstructorBit(ctx, obj1, TRUE); - JS_SetPropertyFunctionList(ctx, obj1, js_proxy_funcs, - countof(js_proxy_funcs)); - JS_DefinePropertyValueStr(ctx, ctx->global_obj, "Proxy", - obj1, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); + if (JS_SetPropertyFunctionList(ctx, obj1, js_proxy_funcs, + countof(js_proxy_funcs))) + goto fail; + if (JS_DefinePropertyValueStr(ctx, ctx->global_obj, "Proxy", + obj1, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE) < 0) + goto fail; + return 0; + fail: + JS_FreeValue(ctx, obj1); + return -1; } /* Symbol */ @@ -47482,7 +49974,7 @@ static JSValue js_symbol_constructor(JSContext *ctx, JSValueConst new_target, JSString *p; if (!JS_IsUndefined(new_target)) - return JS_ThrowTypeError(ctx, "not a constructor"); + return JS_ThrowTypeErrorNotAConstructor(ctx, new_target); if (argc == 0 || JS_IsUndefined(argv[0])) { p = NULL; } else { @@ -47582,6 +50074,19 @@ static JSValue js_symbol_keyFor(JSContext *ctx, JSValueConst this_val, static const JSCFunctionListEntry js_symbol_funcs[] = { JS_CFUNC_DEF("for", 1, js_symbol_for ), JS_CFUNC_DEF("keyFor", 1, js_symbol_keyFor ), + JS_PROP_ATOM_DEF("toPrimitive", JS_ATOM_Symbol_toPrimitive, 0), + JS_PROP_ATOM_DEF("iterator", JS_ATOM_Symbol_iterator, 0), + JS_PROP_ATOM_DEF("match", JS_ATOM_Symbol_match, 0), + JS_PROP_ATOM_DEF("matchAll", JS_ATOM_Symbol_matchAll, 0), + JS_PROP_ATOM_DEF("replace", JS_ATOM_Symbol_replace, 0), + JS_PROP_ATOM_DEF("search", JS_ATOM_Symbol_search, 0), + JS_PROP_ATOM_DEF("split", JS_ATOM_Symbol_split, 0), + JS_PROP_ATOM_DEF("toStringTag", JS_ATOM_Symbol_toStringTag, 0), + JS_PROP_ATOM_DEF("isConcatSpreadable", JS_ATOM_Symbol_isConcatSpreadable, 0), + JS_PROP_ATOM_DEF("hasInstance", JS_ATOM_Symbol_hasInstance, 0), + JS_PROP_ATOM_DEF("species", JS_ATOM_Symbol_species, 0), + JS_PROP_ATOM_DEF("unscopables", JS_ATOM_Symbol_unscopables, 0), + JS_PROP_ATOM_DEF("asyncIterator", JS_ATOM_Symbol_asyncIterator, 0), }; /* Set/Map/WeakSet/WeakMap */ @@ -47774,7 +50279,7 @@ static JSValue js_map_constructor(JSContext *ctx, JSValueConst new_target, } /* XXX: could normalize strings to speed up comparison */ -static JSValueConst map_normalize_key(JSContext *ctx, JSValueConst key) +static JSValue map_normalize_key(JSContext *ctx, JSValue key) { uint32_t tag = JS_VALUE_GET_TAG(key); /* convert -0.0 to +0.0 */ @@ -47784,6 +50289,11 @@ static JSValueConst map_normalize_key(JSContext *ctx, JSValueConst key) return key; } +static JSValueConst map_normalize_key_const(JSContext *ctx, JSValueConst key) +{ + return (JSValueConst)map_normalize_key(ctx, (JSValue)key); +} + /* hash multipliers, same as the Linux kernel (see Knuth vol 3, section 6.4, exercise 9) */ #define HASH_MUL32 0x61C88647 @@ -47943,8 +50453,19 @@ static JSMapRecord *map_add_record(JSContext *ctx, JSMapState *s, return mr; } +static JSMapRecord *set_add_record(JSContext *ctx, JSMapState *s, + JSValueConst key) +{ + JSMapRecord *mr; + mr = map_add_record(ctx, s, key); + if (!mr) + return NULL; + mr->value = JS_UNDEFINED; + return mr; +} + /* warning: the record must be removed from the hash table before */ -static void map_delete_record(JSRuntime *rt, JSMapState *s, JSMapRecord *mr) +static void map_delete_record_internal(JSRuntime *rt, JSMapState *s, JSMapRecord *mr) { if (mr->empty) return; @@ -48004,7 +50525,7 @@ static void map_delete_weakrefs(JSRuntime *rt, JSWeakRefHeader *wh) /* remove from the hash table */ *pmr = mr1->hash_next; done: - map_delete_record(rt, s, mr); + map_delete_record_internal(rt, s, mr); } } } @@ -48018,7 +50539,7 @@ static JSValue js_map_set(JSContext *ctx, JSValueConst this_val, if (!s) return JS_EXCEPTION; - key = map_normalize_key(ctx, argv[0]); + key = map_normalize_key_const(ctx, argv[0]); if (s->is_weak && !js_weakref_is_target(key)) return JS_ThrowTypeError(ctx, "invalid value used as %s key", (magic & MAGIC_SET) ? "WeakSet" : "WeakMap"); if (magic & MAGIC_SET) @@ -48046,7 +50567,7 @@ static JSValue js_map_get(JSContext *ctx, JSValueConst this_val, if (!s) return JS_EXCEPTION; - key = map_normalize_key(ctx, argv[0]); + key = map_normalize_key_const(ctx, argv[0]); mr = map_find_record(ctx, s, key); if (!mr) return JS_UNDEFINED; @@ -48054,31 +50575,13 @@ static JSValue js_map_get(JSContext *ctx, JSValueConst this_val, return JS_DupValue(ctx, mr->value); } -static JSValue js_map_has(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv, int magic) -{ - JSMapState *s = JS_GetOpaque2(ctx, this_val, JS_CLASS_MAP + magic); - JSMapRecord *mr; - JSValueConst key; - - if (!s) - return JS_EXCEPTION; - key = map_normalize_key(ctx, argv[0]); - mr = map_find_record(ctx, s, key); - return JS_NewBool(ctx, mr != NULL); -} - -static JSValue js_map_delete(JSContext *ctx, JSValueConst this_val, - int argc, JSValueConst *argv, int magic) +/* return JS_TRUE or JS_FALSE */ +static JSValue map_delete_record(JSContext *ctx, JSMapState *s, JSValueConst key) { - JSMapState *s = JS_GetOpaque2(ctx, this_val, JS_CLASS_MAP + magic); JSMapRecord *mr, **pmr; - JSValueConst key; uint32_t h; - if (!s) - return JS_EXCEPTION; - key = map_normalize_key(ctx, argv[0]); + key = map_normalize_key_const(ctx, key); h = map_hash_key(key, s->hash_bits); pmr = &s->hash_table[h]; @@ -48098,10 +50601,70 @@ static JSValue js_map_delete(JSContext *ctx, JSValueConst this_val, /* remove from the hash table */ *pmr = mr->hash_next; - map_delete_record(ctx->rt, s, mr); + map_delete_record_internal(ctx->rt, s, mr); return JS_TRUE; } +static JSValue js_map_getOrInsert(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv, int magic) +{ + BOOL computed = magic & 1; + JSClassID class_id = magic >> 1; + JSMapState *s = JS_GetOpaque2(ctx, this_val, class_id); + JSMapRecord *mr; + JSValueConst key; + JSValue value; + + if (!s) + return JS_EXCEPTION; + if (computed && !JS_IsFunction(ctx, argv[1])) + return JS_ThrowTypeError(ctx, "not a function"); + key = map_normalize_key_const(ctx, argv[0]); + if (s->is_weak && !js_weakref_is_target(key)) + return JS_ThrowTypeError(ctx, "invalid value used as WeakMap key"); + mr = map_find_record(ctx, s, key); + if (!mr) { + if (computed) { + value = JS_Call(ctx, argv[1], JS_UNDEFINED, 1, &key); + if (JS_IsException(value)) + return JS_EXCEPTION; + map_delete_record(ctx, s, key); + } else { + value = JS_DupValue(ctx, argv[1]); + } + mr = map_add_record(ctx, s, key); + if (!mr) { + JS_FreeValue(ctx, value); + return JS_EXCEPTION; + } + mr->value = value; + } + return JS_DupValue(ctx, mr->value); +} + +static JSValue js_map_has(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv, int magic) +{ + JSMapState *s = JS_GetOpaque2(ctx, this_val, JS_CLASS_MAP + magic); + JSMapRecord *mr; + JSValueConst key; + + if (!s) + return JS_EXCEPTION; + key = map_normalize_key_const(ctx, argv[0]); + mr = map_find_record(ctx, s, key); + return JS_NewBool(ctx, mr != NULL); +} + +static JSValue js_map_delete(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv, int magic) +{ + JSMapState *s = JS_GetOpaque2(ctx, this_val, JS_CLASS_MAP + magic); + if (!s) + return JS_EXCEPTION; + return map_delete_record(ctx, s, argv[0]); +} + static JSValue js_map_clear(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, int magic) { @@ -48117,7 +50680,7 @@ static JSValue js_map_clear(JSContext *ctx, JSValueConst this_val, list_for_each_safe(el, el1, &s->records) { mr = list_entry(el, JSMapRecord, link); - map_delete_record(ctx->rt, s, mr); + map_delete_record_internal(ctx->rt, s, mr); } return JS_UNDEFINED; } @@ -48477,6 +51040,560 @@ static JSValue js_map_iterator_next(JSContext *ctx, JSValueConst this_val, } } +static int get_set_record(JSContext *ctx, JSValueConst obj, + int64_t *psize, JSValue *phas, JSValue *pkeys) +{ + JSMapState *s; + int64_t size; + JSValue has = JS_UNDEFINED, keys = JS_UNDEFINED; + + s = JS_GetOpaque(obj, JS_CLASS_SET); + if (s) { + size = s->record_count; + } else { + JSValue v; + double d; + + v = JS_GetProperty(ctx, obj, JS_ATOM_size); + if (JS_IsException(v)) + goto exception; + if (JS_ToFloat64Free(ctx, &d, v) < 0) + goto exception; + if (isnan(d)) { + JS_ThrowTypeError(ctx, ".size is not a number"); + goto exception; + } + if (d < INT64_MIN) + size = INT64_MIN; + else if (d >= 0x1p63) /* must use INT64_MAX + 1 because INT64_MAX cannot be exactly represented as a double */ + size = INT64_MAX; + else + size = (int64_t)d; + if (size < 0) { + JS_ThrowRangeError(ctx, ".size must be positive"); + goto exception; + } + } + + has = JS_GetProperty(ctx, obj, JS_ATOM_has); + if (JS_IsException(has)) + goto exception; + if (JS_IsUndefined(has)) { + JS_ThrowTypeError(ctx, ".has is undefined"); + goto exception; + } + if (!JS_IsFunction(ctx, has)) { + JS_ThrowTypeError(ctx, ".has is not a function"); + goto exception; + } + + keys = JS_GetProperty(ctx, obj, JS_ATOM_keys); + if (JS_IsException(keys)) + goto exception; + if (JS_IsUndefined(keys)) { + JS_ThrowTypeError(ctx, ".keys is undefined"); + goto exception; + } + if (!JS_IsFunction(ctx, keys)) { + JS_ThrowTypeError(ctx, ".keys is not a function"); + goto exception; + } + *psize = size; + *phas = has; + *pkeys = keys; + return 0; + + exception: + JS_FreeValue(ctx, has); + JS_FreeValue(ctx, keys); + *psize = 0; + *phas = JS_UNDEFINED; + *pkeys = JS_UNDEFINED; + return -1; +} + +/* copy 'this_val' in a new set without side effects */ +static JSValue js_copy_set(JSContext *ctx, JSValueConst this_val) +{ + JSValue newset; + JSMapState *s, *t; + struct list_head *el; + JSMapRecord *mr; + + s = JS_GetOpaque2(ctx, this_val, JS_CLASS_SET); + if (!s) + return JS_EXCEPTION; + + newset = js_map_constructor(ctx, JS_UNDEFINED, 0, NULL, MAGIC_SET); + if (JS_IsException(newset)) + return JS_EXCEPTION; + t = JS_GetOpaque(newset, JS_CLASS_SET); + + // can't clone this_val using js_map_constructor(), + // test262 mandates we don't call the .add method + list_for_each(el, &s->records) { + mr = list_entry(el, JSMapRecord, link); + if (mr->empty) + continue; + if (!set_add_record(ctx, t, mr->key)) + goto exception; + } + return newset; + exception: + JS_FreeValue(ctx, newset); + return JS_EXCEPTION; +} + +static JSValue js_set_isDisjointFrom(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue item, iter, keys, has, next, rv, rval; + int done; + BOOL found; + JSMapState *s; + int64_t size; + int ok; + + iter = JS_UNDEFINED; + next = JS_UNDEFINED; + rval = JS_EXCEPTION; + s = JS_GetOpaque2(ctx, this_val, JS_CLASS_SET); + if (!s) + return JS_EXCEPTION; + if (get_set_record(ctx, argv[0], &size, &has, &keys) < 0) + goto exception; + if (s->record_count <= size) { + iter = js_create_map_iterator(ctx, this_val, 0, NULL, MAGIC_SET); + if (JS_IsException(iter)) + goto exception; + found = FALSE; + do { + item = js_map_iterator_next(ctx, iter, 0, NULL, &done, MAGIC_SET); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + rv = JS_Call(ctx, has, argv[0], 1, (JSValueConst *)&item); + JS_FreeValue(ctx, item); + ok = JS_ToBoolFree(ctx, rv); // returns -1 if rv is JS_EXCEPTION + if (ok < 0) + goto exception; + found = (ok > 0); + } while (!found); + } else { + iter = JS_Call(ctx, keys, argv[0], 0, NULL); + if (JS_IsException(iter)) + goto exception; + next = JS_GetProperty(ctx, iter, JS_ATOM_next); + if (JS_IsException(next)) + goto exception; + found = FALSE; + for(;;) { + item = JS_IteratorNext(ctx, iter, next, 0, NULL, &done); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + item = map_normalize_key(ctx, item); + found = (NULL != map_find_record(ctx, s, item)); + JS_FreeValue(ctx, item); + if (found) { + JS_IteratorClose(ctx, iter, FALSE); + break; + } + } + } + rval = !found ? JS_TRUE : JS_FALSE; +exception: + JS_FreeValue(ctx, has); + JS_FreeValue(ctx, keys); + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, next); + return rval; +} + +static JSValue js_set_isSubsetOf(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue item, iter, keys, has, next, rv, rval; + BOOL found; + JSMapState *s; + int64_t size; + int done, ok; + + iter = JS_UNDEFINED; + next = JS_UNDEFINED; + rval = JS_EXCEPTION; + s = JS_GetOpaque2(ctx, this_val, JS_CLASS_SET); + if (!s) + return JS_EXCEPTION; + if (get_set_record(ctx, argv[0], &size, &has, &keys) < 0) + goto exception; + found = FALSE; + if (s->record_count > size) + goto fini; + iter = js_create_map_iterator(ctx, this_val, 0, NULL, MAGIC_SET); + if (JS_IsException(iter)) + goto exception; + found = TRUE; + do { + item = js_map_iterator_next(ctx, iter, 0, NULL, &done, MAGIC_SET); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + rv = JS_Call(ctx, has, argv[0], 1, (JSValueConst *)&item); + JS_FreeValue(ctx, item); + ok = JS_ToBoolFree(ctx, rv); // returns -1 if rv is JS_EXCEPTION + if (ok < 0) + goto exception; + found = (ok > 0); + } while (found); +fini: + rval = found ? JS_TRUE : JS_FALSE; +exception: + JS_FreeValue(ctx, has); + JS_FreeValue(ctx, keys); + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, next); + return rval; +} + +static JSValue js_set_isSupersetOf(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue item, iter, keys, has, next, rval; + int done; + BOOL found; + JSMapState *s; + int64_t size; + + iter = JS_UNDEFINED; + next = JS_UNDEFINED; + rval = JS_EXCEPTION; + s = JS_GetOpaque2(ctx, this_val, JS_CLASS_SET); + if (!s) + return JS_EXCEPTION; + if (get_set_record(ctx, argv[0], &size, &has, &keys) < 0) + goto exception; + found = FALSE; + if (s->record_count < size) + goto fini; + iter = JS_Call(ctx, keys, argv[0], 0, NULL); + if (JS_IsException(iter)) + goto exception; + next = JS_GetProperty(ctx, iter, JS_ATOM_next); + if (JS_IsException(next)) + goto exception; + found = TRUE; + for(;;) { + item = JS_IteratorNext(ctx, iter, next, 0, NULL, &done); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + item = map_normalize_key(ctx, item); + found = (NULL != map_find_record(ctx, s, item)); + JS_FreeValue(ctx, item); + if (!found) { + JS_IteratorClose(ctx, iter, FALSE); + break; + } + } +fini: + rval = found ? JS_TRUE : JS_FALSE; +exception: + JS_FreeValue(ctx, has); + JS_FreeValue(ctx, keys); + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, next); + return rval; +} + +static JSValue js_set_intersection(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue newset, item, iter, keys, has, next, rv; + JSMapState *s, *t; + JSMapRecord *mr; + int64_t size; + int done, ok; + + iter = JS_UNDEFINED; + next = JS_UNDEFINED; + newset = JS_UNDEFINED; + s = JS_GetOpaque2(ctx, this_val, JS_CLASS_SET); + if (!s) + return JS_EXCEPTION; + if (get_set_record(ctx, argv[0], &size, &has, &keys) < 0) + goto exception; + if (s->record_count > size) { + iter = JS_Call(ctx, keys, argv[0], 0, NULL); + if (JS_IsException(iter)) + goto exception; + next = JS_GetProperty(ctx, iter, JS_ATOM_next); + if (JS_IsException(next)) + goto exception; + newset = js_map_constructor(ctx, JS_UNDEFINED, 0, NULL, MAGIC_SET); + if (JS_IsException(newset)) + goto exception; + t = JS_GetOpaque(newset, JS_CLASS_SET); + for (;;) { + item = JS_IteratorNext(ctx, iter, next, 0, NULL, &done); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + item = map_normalize_key(ctx, item); + if (!map_find_record(ctx, s, item)) { + JS_FreeValue(ctx, item); + } else if (map_find_record(ctx, t, item)) { + JS_FreeValue(ctx, item); // no duplicates + } else { + mr = set_add_record(ctx, t, item); + JS_FreeValue(ctx, item); + if (!mr) + goto exception; + } + } + } else { + iter = js_create_map_iterator(ctx, this_val, 0, NULL, MAGIC_SET); + if (JS_IsException(iter)) + goto exception; + newset = js_map_constructor(ctx, JS_UNDEFINED, 0, NULL, MAGIC_SET); + if (JS_IsException(newset)) + goto exception; + t = JS_GetOpaque(newset, JS_CLASS_SET); + for (;;) { + item = js_map_iterator_next(ctx, iter, 0, NULL, &done, MAGIC_SET); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + rv = JS_Call(ctx, has, argv[0], 1, (JSValueConst *)&item); + ok = JS_ToBoolFree(ctx, rv); // returns -1 if rv is JS_EXCEPTION + if (ok > 0) { + item = map_normalize_key(ctx, item); + if (map_find_record(ctx, t, item)) { + JS_FreeValue(ctx, item); // no duplicates + } else { + mr = set_add_record(ctx, t, item); + JS_FreeValue(ctx, item); + if (!mr) + goto exception; + } + } else { + JS_FreeValue(ctx, item); + if (ok < 0) + goto exception; + } + } + } + goto fini; +exception: + JS_FreeValue(ctx, newset); + newset = JS_EXCEPTION; +fini: + JS_FreeValue(ctx, has); + JS_FreeValue(ctx, keys); + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, next); + return newset; +} + +static JSValue js_set_difference(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue newset, item, iter, keys, has, next, rv; + JSMapState *s, *t; + int64_t size; + int done; + int ok; + + iter = JS_UNDEFINED; + next = JS_UNDEFINED; + newset = JS_UNDEFINED; + s = JS_GetOpaque2(ctx, this_val, JS_CLASS_SET); + if (!s) + return JS_EXCEPTION; + if (get_set_record(ctx, argv[0], &size, &has, &keys) < 0) + goto exception; + + newset = js_copy_set(ctx, this_val); + if (JS_IsException(newset)) + goto exception; + t = JS_GetOpaque(newset, JS_CLASS_SET); + + if (s->record_count <= size) { + iter = js_create_map_iterator(ctx, newset, 0, NULL, MAGIC_SET); + if (JS_IsException(iter)) + goto exception; + for (;;) { + item = js_map_iterator_next(ctx, iter, 0, NULL, &done, MAGIC_SET); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + rv = JS_Call(ctx, has, argv[0], 1, (JSValueConst *)&item); + ok = JS_ToBoolFree(ctx, rv); // returns -1 if rv is JS_EXCEPTION + if (ok < 0) { + JS_FreeValue(ctx, item); + goto exception; + } + if (ok) { + map_delete_record(ctx, t, item); + } + JS_FreeValue(ctx, item); + } + } else { + iter = JS_Call(ctx, keys, argv[0], 0, NULL); + if (JS_IsException(iter)) + goto exception; + next = JS_GetProperty(ctx, iter, JS_ATOM_next); + if (JS_IsException(next)) + goto exception; + for (;;) { + item = JS_IteratorNext(ctx, iter, next, 0, NULL, &done); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + map_delete_record(ctx, t, item); + JS_FreeValue(ctx, item); + } + } + goto fini; +exception: + JS_FreeValue(ctx, newset); + newset = JS_EXCEPTION; +fini: + JS_FreeValue(ctx, has); + JS_FreeValue(ctx, keys); + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, next); + return newset; +} + +static JSValue js_set_symmetricDifference(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue newset, item, iter, next, has, keys; + JSMapState *s, *t; + JSMapRecord *mr; + int64_t size; + int done; + BOOL present; + + s = JS_GetOpaque2(ctx, this_val, JS_CLASS_SET); + if (!s) + return JS_EXCEPTION; + if (get_set_record(ctx, argv[0], &size, &has, &keys) < 0) + return JS_EXCEPTION; + JS_FreeValue(ctx, has); + + next = JS_UNDEFINED; + newset = JS_UNDEFINED; + iter = JS_Call(ctx, keys, argv[0], 0, NULL); + if (JS_IsException(iter)) + goto exception; + next = JS_GetProperty(ctx, iter, JS_ATOM_next); + if (JS_IsException(next)) + goto exception; + newset = js_copy_set(ctx, this_val); + if (JS_IsException(newset)) + goto exception; + t = JS_GetOpaque(newset, JS_CLASS_SET); + for (;;) { + item = JS_IteratorNext(ctx, iter, next, 0, NULL, &done); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + // note the subtlety here: due to mutating iterators, it's + // possible for keys to disappear during iteration; test262 + // still expects us to maintain insertion order though, so + // we first check |this|, then |new|; |new| is a copy of |this| + // - if item exists in |this|, delete (if it exists) from |new| + // - if item misses in |this| and |new|, add to |new| + // - if item exists in |new| but misses in |this|, *don't* add it, + // mutating iterator erased it + item = map_normalize_key(ctx, item); + present = (NULL != map_find_record(ctx, s, item)); + mr = map_find_record(ctx, t, item); + if (present) { + map_delete_record(ctx, t, item); + JS_FreeValue(ctx, item); + } else if (mr) { + JS_FreeValue(ctx, item); + } else { + mr = set_add_record(ctx, t, item); + JS_FreeValue(ctx, item); + if (!mr) + goto exception; + } + } + goto fini; +exception: + JS_FreeValue(ctx, newset); + newset = JS_EXCEPTION; +fini: + JS_FreeValue(ctx, next); + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, keys); + return newset; +} + +static JSValue js_set_union(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue newset, item, iter, next, has, keys, rv; + JSMapState *s; + int64_t size; + int done; + + s = JS_GetOpaque2(ctx, this_val, JS_CLASS_SET); + if (!s) + return JS_EXCEPTION; + if (get_set_record(ctx, argv[0], &size, &has, &keys) < 0) + return JS_EXCEPTION; + JS_FreeValue(ctx, has); + + next = JS_UNDEFINED; + newset = JS_UNDEFINED; + iter = JS_Call(ctx, keys, argv[0], 0, NULL); + if (JS_IsException(iter)) + goto exception; + next = JS_GetProperty(ctx, iter, JS_ATOM_next); + if (JS_IsException(next)) + goto exception; + + newset = js_copy_set(ctx, this_val); + if (JS_IsException(newset)) + goto exception; + + for (;;) { + item = JS_IteratorNext(ctx, iter, next, 0, NULL, &done); + if (JS_IsException(item)) + goto exception; + if (done) // item is JS_UNDEFINED + break; + rv = js_map_set(ctx, newset, 1, (JSValueConst *)&item, MAGIC_SET); + JS_FreeValue(ctx, item); + if (JS_IsException(rv)) + goto exception; + JS_FreeValue(ctx, rv); + } + goto fini; +exception: + JS_FreeValue(ctx, newset); + newset = JS_EXCEPTION; +fini: + JS_FreeValue(ctx, next); + JS_FreeValue(ctx, iter); + JS_FreeValue(ctx, keys); + return newset; +} + static const JSCFunctionListEntry js_map_funcs[] = { JS_CFUNC_MAGIC_DEF("groupBy", 2, js_object_groupBy, 1 ), JS_CGETSET_DEF("[Symbol.species]", js_get_this, NULL ), @@ -48485,6 +51602,10 @@ static const JSCFunctionListEntry js_map_funcs[] = { static const JSCFunctionListEntry js_map_proto_funcs[] = { JS_CFUNC_MAGIC_DEF("set", 2, js_map_set, 0 ), JS_CFUNC_MAGIC_DEF("get", 1, js_map_get, 0 ), + JS_CFUNC_MAGIC_DEF("getOrInsert", 2, js_map_getOrInsert, + (JS_CLASS_MAP << 1) | /*computed*/FALSE ), + JS_CFUNC_MAGIC_DEF("getOrInsertComputed", 2, js_map_getOrInsert, + (JS_CLASS_MAP << 1) | /*computed*/TRUE ), JS_CFUNC_MAGIC_DEF("has", 1, js_map_has, 0 ), JS_CFUNC_MAGIC_DEF("delete", 1, js_map_delete, 0 ), JS_CFUNC_MAGIC_DEF("clear", 0, js_map_clear, 0 ), @@ -48509,6 +51630,13 @@ static const JSCFunctionListEntry js_set_proto_funcs[] = { JS_CFUNC_MAGIC_DEF("clear", 0, js_map_clear, MAGIC_SET ), JS_CGETSET_MAGIC_DEF("size", js_map_get_size, NULL, MAGIC_SET ), JS_CFUNC_MAGIC_DEF("forEach", 1, js_map_forEach, MAGIC_SET ), + JS_CFUNC_DEF("isDisjointFrom", 1, js_set_isDisjointFrom ), + JS_CFUNC_DEF("isSubsetOf", 1, js_set_isSubsetOf ), + JS_CFUNC_DEF("isSupersetOf", 1, js_set_isSupersetOf ), + JS_CFUNC_DEF("intersection", 1, js_set_intersection ), + JS_CFUNC_DEF("difference", 1, js_set_difference ), + JS_CFUNC_DEF("symmetricDifference", 1, js_set_symmetricDifference ), + JS_CFUNC_DEF("union", 1, js_set_union ), JS_CFUNC_MAGIC_DEF("values", 0, js_create_map_iterator, (JS_ITERATOR_KIND_KEY << 2) | MAGIC_SET ), JS_ALIAS_DEF("keys", "values" ), JS_ALIAS_DEF("[Symbol.iterator]", "values" ), @@ -48524,6 +51652,10 @@ static const JSCFunctionListEntry js_set_iterator_proto_funcs[] = { static const JSCFunctionListEntry js_weak_map_proto_funcs[] = { JS_CFUNC_MAGIC_DEF("set", 2, js_map_set, MAGIC_WEAK ), JS_CFUNC_MAGIC_DEF("get", 1, js_map_get, MAGIC_WEAK ), + JS_CFUNC_MAGIC_DEF("getOrInsert", 2, js_map_getOrInsert, + (JS_CLASS_WEAKMAP << 1) | /*computed*/FALSE ), + JS_CFUNC_MAGIC_DEF("getOrInsertComputed", 2, js_map_getOrInsert, + (JS_CLASS_WEAKMAP << 1) | /*computed*/TRUE ), JS_CFUNC_MAGIC_DEF("has", 1, js_map_has, MAGIC_WEAK ), JS_CFUNC_MAGIC_DEF("delete", 1, js_map_delete, MAGIC_WEAK ), JS_PROP_STRING_DEF("[Symbol.toStringTag]", "WeakMap", JS_PROP_CONFIGURABLE ), @@ -48554,35 +51686,37 @@ static const uint8_t js_map_proto_funcs_count[6] = { countof(js_set_iterator_proto_funcs), }; -void JS_AddIntrinsicMapSet(JSContext *ctx) +int JS_AddIntrinsicMapSet(JSContext *ctx) { int i; JSValue obj1; char buf[ATOM_GET_STR_BUF_SIZE]; for(i = 0; i < 4; i++) { + JSCFunctionType ft; const char *name = JS_AtomGetStr(ctx, buf, sizeof(buf), JS_ATOM_Map + i); - ctx->class_proto[JS_CLASS_MAP + i] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_MAP + i], - js_map_proto_funcs_ptr[i], - js_map_proto_funcs_count[i]); - obj1 = JS_NewCFunctionMagic(ctx, js_map_constructor, name, 0, - JS_CFUNC_constructor_magic, i); - if (i < 2) { - JS_SetPropertyFunctionList(ctx, obj1, js_map_funcs, - countof(js_map_funcs)); - } - JS_NewGlobalCConstructor2(ctx, obj1, name, ctx->class_proto[JS_CLASS_MAP + i]); + ft.constructor_magic = js_map_constructor; + obj1 = JS_NewCConstructor(ctx, JS_CLASS_MAP + i, name, + ft.generic, 0, JS_CFUNC_constructor_magic, i, + JS_UNDEFINED, + js_map_funcs, i < 2 ? countof(js_map_funcs) : 0, + js_map_proto_funcs_ptr[i], js_map_proto_funcs_count[i], + 0); + if (JS_IsException(obj1)) + return -1; + JS_FreeValue(ctx, obj1); } for(i = 0; i < 2; i++) { ctx->class_proto[JS_CLASS_MAP_ITERATOR + i] = - JS_NewObjectProto(ctx, ctx->iterator_proto); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_MAP_ITERATOR + i], - js_map_proto_funcs_ptr[i + 4], - js_map_proto_funcs_count[i + 4]); + JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_ITERATOR], + js_map_proto_funcs_ptr[i + 4], + js_map_proto_funcs_count[i + 4]); + if (JS_IsException(ctx->class_proto[JS_CLASS_MAP_ITERATOR + i])) + return -1; } + return 0; } /* Generator */ @@ -49095,16 +52229,57 @@ static JSValue js_promise_withResolvers(JSContext *ctx, if (JS_IsException(result_promise)) return result_promise; obj = JS_NewObject(ctx); - if (JS_IsException(obj)) { - JS_FreeValue(ctx, resolving_funcs[0]); - JS_FreeValue(ctx, resolving_funcs[1]); - JS_FreeValue(ctx, result_promise); - return JS_EXCEPTION; + if (JS_IsException(obj)) + goto exception; + if (JS_DefinePropertyValue(ctx, obj, JS_ATOM_promise, result_promise, + JS_PROP_C_W_E) < 0) { + goto exception; + } + result_promise = JS_UNDEFINED; + if (JS_DefinePropertyValue(ctx, obj, JS_ATOM_resolve, resolving_funcs[0], + JS_PROP_C_W_E) < 0) { + goto exception; + } + resolving_funcs[0] = JS_UNDEFINED; + if (JS_DefinePropertyValue(ctx, obj, JS_ATOM_reject, resolving_funcs[1], + JS_PROP_C_W_E) < 0) { + goto exception; } - JS_DefinePropertyValue(ctx, obj, JS_ATOM_promise, result_promise, JS_PROP_C_W_E); - JS_DefinePropertyValue(ctx, obj, JS_ATOM_resolve, resolving_funcs[0], JS_PROP_C_W_E); - JS_DefinePropertyValue(ctx, obj, JS_ATOM_reject, resolving_funcs[1], JS_PROP_C_W_E); return obj; +exception: + JS_FreeValue(ctx, resolving_funcs[0]); + JS_FreeValue(ctx, resolving_funcs[1]); + JS_FreeValue(ctx, result_promise); + JS_FreeValue(ctx, obj); + return JS_EXCEPTION; +} + +static JSValue js_promise_try(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + JSValue result_promise, resolving_funcs[2], ret, ret2; + BOOL is_reject = 0; + + if (!JS_IsObject(this_val)) + return JS_ThrowTypeErrorNotAnObject(ctx); + result_promise = js_new_promise_capability(ctx, resolving_funcs, this_val); + if (JS_IsException(result_promise)) + return result_promise; + ret = JS_Call(ctx, argv[0], JS_UNDEFINED, argc - 1, argv + 1); + if (JS_IsException(ret)) { + is_reject = 1; + ret = JS_GetException(ctx); + } + ret2 = JS_Call(ctx, resolving_funcs[is_reject], JS_UNDEFINED, 1, (JSValueConst *)&ret); + JS_FreeValue(ctx, resolving_funcs[0]); + JS_FreeValue(ctx, resolving_funcs[1]); + JS_FreeValue(ctx, ret); + if (JS_IsException(ret2)) { + JS_FreeValue(ctx, result_promise); + return ret2; + } + JS_FreeValue(ctx, ret2); + return result_promise; } static __exception int remainingElementsCount_add(JSContext *ctx, @@ -49594,6 +52769,7 @@ static const JSCFunctionListEntry js_promise_funcs[] = { JS_CFUNC_MAGIC_DEF("all", 1, js_promise_all, PROMISE_MAGIC_all ), JS_CFUNC_MAGIC_DEF("allSettled", 1, js_promise_all, PROMISE_MAGIC_allSettled ), JS_CFUNC_MAGIC_DEF("any", 1, js_promise_all, PROMISE_MAGIC_any ), + JS_CFUNC_DEF("try", 1, js_promise_try ), JS_CFUNC_DEF("race", 1, js_promise_race ), JS_CFUNC_DEF("withResolvers", 0, js_promise_withResolvers ), JS_CGETSET_DEF("[Symbol.species]", js_get_this, NULL), @@ -49859,14 +53035,16 @@ static JSClassShortDef const js_async_class_def[] = { { JS_ATOM_AsyncGenerator, js_async_generator_finalizer, js_async_generator_mark }, /* JS_CLASS_ASYNC_GENERATOR */ }; -void JS_AddIntrinsicPromise(JSContext *ctx) +int JS_AddIntrinsicPromise(JSContext *ctx) { JSRuntime *rt = ctx->rt; JSValue obj1; + JSCFunctionType ft; if (!JS_IsRegisteredClass(rt, JS_CLASS_PROMISE)) { - init_class_range(rt, js_async_class_def, JS_CLASS_PROMISE, - countof(js_async_class_def)); + if (init_class_range(rt, js_async_class_def, JS_CLASS_PROMISE, + countof(js_async_class_def))) + return -1; rt->class_array[JS_CLASS_PROMISE_RESOLVE_FUNCTION].call = js_promise_resolve_function_call; rt->class_array[JS_CLASS_PROMISE_REJECT_FUNCTION].call = js_promise_resolve_function_call; rt->class_array[JS_CLASS_ASYNC_FUNCTION].call = js_async_function_call; @@ -49876,72 +53054,67 @@ void JS_AddIntrinsicPromise(JSContext *ctx) } /* Promise */ - ctx->class_proto[JS_CLASS_PROMISE] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_PROMISE], - js_promise_proto_funcs, - countof(js_promise_proto_funcs)); - obj1 = JS_NewCFunction2(ctx, js_promise_constructor, "Promise", 1, - JS_CFUNC_constructor, 0); - ctx->promise_ctor = JS_DupValue(ctx, obj1); - JS_SetPropertyFunctionList(ctx, obj1, - js_promise_funcs, - countof(js_promise_funcs)); - JS_NewGlobalCConstructor2(ctx, obj1, "Promise", - ctx->class_proto[JS_CLASS_PROMISE]); - + obj1 = JS_NewCConstructor(ctx, JS_CLASS_PROMISE, "Promise", + js_promise_constructor, 1, JS_CFUNC_constructor, 0, + JS_UNDEFINED, + js_promise_funcs, countof(js_promise_funcs), + js_promise_proto_funcs, countof(js_promise_proto_funcs), + 0); + if (JS_IsException(obj1)) + return -1; + ctx->promise_ctor = obj1; + /* AsyncFunction */ - ctx->class_proto[JS_CLASS_ASYNC_FUNCTION] = JS_NewObjectProto(ctx, ctx->function_proto); - obj1 = JS_NewCFunction3(ctx, (JSCFunction *)js_function_constructor, - "AsyncFunction", 1, - JS_CFUNC_constructor_or_func_magic, JS_FUNC_ASYNC, - ctx->function_ctor); - JS_SetPropertyFunctionList(ctx, - ctx->class_proto[JS_CLASS_ASYNC_FUNCTION], - js_async_function_proto_funcs, - countof(js_async_function_proto_funcs)); - JS_SetConstructor2(ctx, obj1, ctx->class_proto[JS_CLASS_ASYNC_FUNCTION], - 0, JS_PROP_CONFIGURABLE); + ft.generic_magic = js_function_constructor; + obj1 = JS_NewCConstructor(ctx, JS_CLASS_ASYNC_FUNCTION, "AsyncFunction", + ft.generic, 1, JS_CFUNC_constructor_or_func_magic, JS_FUNC_ASYNC, + ctx->function_ctor, + NULL, 0, + js_async_function_proto_funcs, countof(js_async_function_proto_funcs), + JS_NEW_CTOR_NO_GLOBAL | JS_NEW_CTOR_READONLY); + if (JS_IsException(obj1)) + return -1; JS_FreeValue(ctx, obj1); - + /* AsyncIteratorPrototype */ - ctx->async_iterator_proto = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->async_iterator_proto, - js_async_iterator_proto_funcs, - countof(js_async_iterator_proto_funcs)); + ctx->async_iterator_proto = + JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_OBJECT], + js_async_iterator_proto_funcs, + countof(js_async_iterator_proto_funcs)); + if (JS_IsException(ctx->async_iterator_proto)) + return -1; /* AsyncFromSyncIteratorPrototype */ ctx->class_proto[JS_CLASS_ASYNC_FROM_SYNC_ITERATOR] = - JS_NewObjectProto(ctx, ctx->async_iterator_proto); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_ASYNC_FROM_SYNC_ITERATOR], - js_async_from_sync_iterator_proto_funcs, - countof(js_async_from_sync_iterator_proto_funcs)); - + JS_NewObjectProtoList(ctx, ctx->async_iterator_proto, + js_async_from_sync_iterator_proto_funcs, + countof(js_async_from_sync_iterator_proto_funcs)); + if (JS_IsException(ctx->class_proto[JS_CLASS_ASYNC_FROM_SYNC_ITERATOR])) + return -1; + /* AsyncGeneratorPrototype */ ctx->class_proto[JS_CLASS_ASYNC_GENERATOR] = - JS_NewObjectProto(ctx, ctx->async_iterator_proto); - JS_SetPropertyFunctionList(ctx, - ctx->class_proto[JS_CLASS_ASYNC_GENERATOR], - js_async_generator_proto_funcs, - countof(js_async_generator_proto_funcs)); + JS_NewObjectProtoList(ctx, ctx->async_iterator_proto, + js_async_generator_proto_funcs, + countof(js_async_generator_proto_funcs)); + if (JS_IsException(ctx->class_proto[JS_CLASS_ASYNC_GENERATOR])) + return -1; /* AsyncGeneratorFunction */ - ctx->class_proto[JS_CLASS_ASYNC_GENERATOR_FUNCTION] = - JS_NewObjectProto(ctx, ctx->function_proto); - obj1 = JS_NewCFunction3(ctx, (JSCFunction *)js_function_constructor, - "AsyncGeneratorFunction", 1, - JS_CFUNC_constructor_or_func_magic, - JS_FUNC_ASYNC_GENERATOR, - ctx->function_ctor); - JS_SetPropertyFunctionList(ctx, - ctx->class_proto[JS_CLASS_ASYNC_GENERATOR_FUNCTION], - js_async_generator_function_proto_funcs, - countof(js_async_generator_function_proto_funcs)); - JS_SetConstructor2(ctx, ctx->class_proto[JS_CLASS_ASYNC_GENERATOR_FUNCTION], - ctx->class_proto[JS_CLASS_ASYNC_GENERATOR], - JS_PROP_CONFIGURABLE, JS_PROP_CONFIGURABLE); - JS_SetConstructor2(ctx, obj1, ctx->class_proto[JS_CLASS_ASYNC_GENERATOR_FUNCTION], - 0, JS_PROP_CONFIGURABLE); + ft.generic_magic = js_function_constructor; + obj1 = JS_NewCConstructor(ctx, JS_CLASS_ASYNC_GENERATOR_FUNCTION, "AsyncGeneratorFunction", + ft.generic, 1, JS_CFUNC_constructor_or_func_magic, JS_FUNC_ASYNC_GENERATOR, + ctx->function_ctor, + NULL, 0, + js_async_generator_function_proto_funcs, countof(js_async_generator_function_proto_funcs), + JS_NEW_CTOR_NO_GLOBAL | JS_NEW_CTOR_READONLY); + if (JS_IsException(obj1)) + return -1; JS_FreeValue(ctx, obj1); + + return JS_SetConstructor2(ctx, ctx->class_proto[JS_CLASS_ASYNC_GENERATOR_FUNCTION], + ctx->class_proto[JS_CLASS_ASYNC_GENERATOR], + JS_PROP_CONFIGURABLE, JS_PROP_CONFIGURABLE); } /* URI handling */ @@ -50236,6 +53409,7 @@ static const JSCFunctionListEntry js_global_funcs[] = { JS_PROP_DOUBLE_DEF("NaN", NAN, 0 ), JS_PROP_UNDEFINED_DEF("undefined", 0 ), JS_PROP_STRING_DEF("[Symbol.toStringTag]", "global", JS_PROP_CONFIGURABLE ), + JS_CFUNC_DEF("eval", 1, js_global_eval ), }; /* Date */ @@ -50434,6 +53608,21 @@ static double set_date_fields(double fields[minimum_length(7)], int is_local) { return time_clip(tv); } +static double set_date_fields_checked(double fields[minimum_length(7)], int is_local) +{ + int i; + double a; + for(i = 0; i < 7; i++) { + a = fields[i]; + if (!isfinite(a)) + return NAN; + fields[i] = trunc(a); + if (i == 0 && fields[0] >= 0 && fields[0] < 100) + fields[0] += 1900; + } + return set_date_fields(fields, is_local); +} + static JSValue get_date_field(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, int magic) { @@ -50619,7 +53808,7 @@ static JSValue js_date_constructor(JSContext *ctx, JSValueConst new_target, // Date(y, mon, d, h, m, s, ms) JSValue rv; int i, n; - double a, val; + double val; if (JS_IsUndefined(new_target)) { /* invoked as function */ @@ -50657,15 +53846,10 @@ static JSValue js_date_constructor(JSContext *ctx, JSValueConst new_target, if (n > 7) n = 7; for(i = 0; i < n; i++) { - if (JS_ToFloat64(ctx, &a, argv[i])) + if (JS_ToFloat64(ctx, &fields[i], argv[i])) return JS_EXCEPTION; - if (!isfinite(a)) - break; - fields[i] = trunc(a); - if (i == 0 && fields[0] >= 0 && fields[0] < 100) - fields[0] += 1900; } - val = (i == n) ? set_date_fields(fields, 1) : NAN; + val = set_date_fields_checked(fields, 1); } has_val: #if 0 @@ -50695,7 +53879,6 @@ static JSValue js_Date_UTC(JSContext *ctx, JSValueConst this_val, // UTC(y, mon, d, h, m, s, ms) double fields[] = { 0, 0, 1, 0, 0, 0, 0 }; int i, n; - double a; n = argc; if (n == 0) @@ -50703,15 +53886,10 @@ static JSValue js_Date_UTC(JSContext *ctx, JSValueConst this_val, if (n > 7) n = 7; for(i = 0; i < n; i++) { - if (JS_ToFloat64(ctx, &a, argv[i])) + if (JS_ToFloat64(ctx, &fields[i], argv[i])) return JS_EXCEPTION; - if (!isfinite(a)) - return JS_NAN; - fields[i] = trunc(a); - if (i == 0 && fields[0] >= 0 && fields[0] < 100) - fields[0] += 1900; } - return JS_NewFloat64(ctx, set_date_fields(fields, 0)); + return JS_NewFloat64(ctx, set_date_fields_checked(fields, 0)); } /* Date string parsing */ @@ -50822,9 +54000,14 @@ static BOOL string_get_tzoffset(const uint8_t *sp, int *pp, int *tzp, BOOL stric hh = hh / 100; } else { mm = 0; - if (string_skip_char(sp, &p, ':') /* optional separator */ - && !string_get_digits(sp, &p, &mm, 2, 2)) - return FALSE; + if (string_skip_char(sp, &p, ':')) { + /* optional separator */ + if (!string_get_digits(sp, &p, &mm, 2, 2)) + return FALSE; + } else { + if (strict) + return FALSE; /* [+-]HH is not accepted in strict mode */ + } } if (hh > 23 || mm > 59) return FALSE; @@ -51023,12 +54206,16 @@ static BOOL js_date_parse_otherstring(const uint8_t *sp, string_get_milliseconds(sp, &p, &fields[6]); } has_time = TRUE; + if ((sp[p] == '+' || sp[p] == '-') && + string_get_tzoffset(sp, &p, &fields[8], FALSE)) { + *is_local = FALSE; + } } else { - if (p - p_start > 2) { + if (p - p_start > 2 && !has_year) { fields[0] = val; has_year = TRUE; } else - if (val < 1 || val > 31) { + if ((val < 1 || val > 31) && !has_year) { fields[0] = val + (val < 100) * 1900 + (val < 50) * 100; has_year = TRUE; } else { @@ -51367,24 +54554,29 @@ JSValue JS_NewDate(JSContext *ctx, double epoch_ms) return obj; } -void JS_AddIntrinsicDate(JSContext *ctx) +int JS_AddIntrinsicDate(JSContext *ctx) { - JSValueConst obj; + JSValue obj; /* Date */ - ctx->class_proto[JS_CLASS_DATE] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_DATE], js_date_proto_funcs, - countof(js_date_proto_funcs)); - obj = JS_NewGlobalCConstructor(ctx, "Date", js_date_constructor, 7, - ctx->class_proto[JS_CLASS_DATE]); - JS_SetPropertyFunctionList(ctx, obj, js_date_funcs, countof(js_date_funcs)); + obj = JS_NewCConstructor(ctx, JS_CLASS_DATE, "Date", + js_date_constructor, 7, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_date_funcs, countof(js_date_funcs), + js_date_proto_funcs, countof(js_date_proto_funcs), + 0); + if (JS_IsException(obj)) + return -1; + JS_FreeValue(ctx, obj); + return 0; } /* eval */ -void JS_AddIntrinsicEval(JSContext *ctx) +int JS_AddIntrinsicEval(JSContext *ctx) { ctx->eval_internal = __JS_EvalInternal; + return 0; } /* BigInt */ @@ -51444,7 +54636,7 @@ static JSValue js_bigint_constructor(JSContext *ctx, int argc, JSValueConst *argv) { if (!JS_IsUndefined(new_target)) - return JS_ThrowTypeError(ctx, "not a constructor"); + return JS_ThrowTypeErrorNotAConstructor(ctx, new_target); return JS_ToBigIntCtorFree(ctx, JS_DupValue(ctx, argv[0])); } @@ -51568,311 +54760,350 @@ static const JSCFunctionListEntry js_bigint_proto_funcs[] = { JS_PROP_STRING_DEF("[Symbol.toStringTag]", "BigInt", JS_PROP_CONFIGURABLE ), }; -static void JS_AddIntrinsicBigInt(JSContext *ctx) +static int JS_AddIntrinsicBigInt(JSContext *ctx) { - JSValueConst obj1; + JSValue obj1; - ctx->class_proto[JS_CLASS_BIG_INT] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_BIG_INT], - js_bigint_proto_funcs, - countof(js_bigint_proto_funcs)); - obj1 = JS_NewGlobalCConstructor(ctx, "BigInt", js_bigint_constructor, 1, - ctx->class_proto[JS_CLASS_BIG_INT]); - JS_SetPropertyFunctionList(ctx, obj1, js_bigint_funcs, - countof(js_bigint_funcs)); + obj1 = JS_NewCConstructor(ctx, JS_CLASS_BIG_INT, "BigInt", + js_bigint_constructor, 1, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_bigint_funcs, countof(js_bigint_funcs), + js_bigint_proto_funcs, countof(js_bigint_proto_funcs), + 0); + if (JS_IsException(obj1)) + return -1; + JS_FreeValue(ctx, obj1); + return 0; } -static const char * const native_error_name[JS_NATIVE_ERROR_COUNT] = { - "EvalError", "RangeError", "ReferenceError", - "SyntaxError", "TypeError", "URIError", - "InternalError", "AggregateError", -}; - /* Minimum amount of objects to be able to compile code and display - error messages. No JSAtom should be allocated by this function. */ -static void JS_AddIntrinsicBasicObjects(JSContext *ctx) + error messages. */ +static int JS_AddIntrinsicBasicObjects(JSContext *ctx) { - JSValue proto; + JSValue obj; + JSCFunctionType ft; int i; - ctx->class_proto[JS_CLASS_OBJECT] = JS_NewObjectProto(ctx, JS_NULL); + /* warning: ordering is tricky */ + ctx->class_proto[JS_CLASS_OBJECT] = + JS_NewObjectProtoClassAlloc(ctx, JS_NULL, JS_CLASS_OBJECT, + countof(js_object_proto_funcs) + 1); + if (JS_IsException(ctx->class_proto[JS_CLASS_OBJECT])) + return -1; JS_SetImmutablePrototype(ctx, ctx->class_proto[JS_CLASS_OBJECT]); - + + /* 2 more properties: caller and arguments */ ctx->function_proto = JS_NewCFunction3(ctx, js_function_proto, "", 0, JS_CFUNC_generic, 0, - ctx->class_proto[JS_CLASS_OBJECT]); + ctx->class_proto[JS_CLASS_OBJECT], + countof(js_function_proto_funcs) + 3 + 2); + if (JS_IsException(ctx->function_proto)) + return -1; ctx->class_proto[JS_CLASS_BYTECODE_FUNCTION] = JS_DupValue(ctx, ctx->function_proto); - ctx->class_proto[JS_CLASS_ERROR] = JS_NewObject(ctx); -#if 0 - /* these are auto-initialized from js_error_proto_funcs, - but delaying might be a problem */ - JS_DefinePropertyValue(ctx, ctx->class_proto[JS_CLASS_ERROR], JS_ATOM_name, - JS_AtomToString(ctx, JS_ATOM_Error), - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - JS_DefinePropertyValue(ctx, ctx->class_proto[JS_CLASS_ERROR], JS_ATOM_message, - JS_AtomToString(ctx, JS_ATOM_empty_string), - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); -#endif - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_ERROR], - js_error_proto_funcs, - countof(js_error_proto_funcs)); + + ctx->global_obj = JS_NewObjectProtoClassAlloc(ctx, ctx->class_proto[JS_CLASS_OBJECT], + JS_CLASS_GLOBAL_OBJECT, 64); + if (JS_IsException(ctx->global_obj)) + return -1; + { + JSObject *p; + obj = JS_NewObjectProtoClassAlloc(ctx, JS_NULL, JS_CLASS_OBJECT, 4); + p = JS_VALUE_GET_OBJ(ctx->global_obj); + p->u.global_object.uninitialized_vars = obj; + } + ctx->global_var_obj = JS_NewObjectProtoClassAlloc(ctx, JS_NULL, + JS_CLASS_OBJECT, 16); + if (JS_IsException(ctx->global_var_obj)) + return -1; + + /* Error */ + ft.generic_magic = js_error_constructor; + obj = JS_NewCConstructor(ctx, JS_CLASS_ERROR, "Error", + ft.generic, 1, JS_CFUNC_constructor_or_func_magic, -1, + JS_UNDEFINED, + js_error_funcs, countof(js_error_funcs), + js_error_proto_funcs, countof(js_error_proto_funcs), + 0); + if (JS_IsException(obj)) + return -1; for(i = 0; i < JS_NATIVE_ERROR_COUNT; i++) { - proto = JS_NewObjectProto(ctx, ctx->class_proto[JS_CLASS_ERROR]); - JS_DefinePropertyValue(ctx, proto, JS_ATOM_name, - JS_NewAtomString(ctx, native_error_name[i]), - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - JS_DefinePropertyValue(ctx, proto, JS_ATOM_message, - JS_AtomToString(ctx, JS_ATOM_empty_string), - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - ctx->native_error_proto[i] = proto; + JSValue func_obj; + const JSCFunctionListEntry *funcs; + int n_args; + char buf[ATOM_GET_STR_BUF_SIZE]; + const char *name = JS_AtomGetStr(ctx, buf, sizeof(buf), + JS_ATOM_EvalError + i); + n_args = 1 + (i == JS_AGGREGATE_ERROR); + funcs = js_native_error_proto_funcs + 2 * i; + func_obj = JS_NewCConstructor(ctx, -1, name, + ft.generic, n_args, JS_CFUNC_constructor_or_func_magic, i, + obj, + NULL, 0, + funcs, 2, + 0); + if (JS_IsException(func_obj)) { + JS_FreeValue(ctx, obj); + return -1; + } + ctx->native_error_proto[i] = JS_GetProperty(ctx, func_obj, JS_ATOM_prototype); + JS_FreeValue(ctx, func_obj); + if (JS_IsException(ctx->native_error_proto[i])) { + JS_FreeValue(ctx, obj); + return -1; + } } + JS_FreeValue(ctx, obj); - /* the array prototype is an array */ - ctx->class_proto[JS_CLASS_ARRAY] = - JS_NewObjectProtoClass(ctx, ctx->class_proto[JS_CLASS_OBJECT], - JS_CLASS_ARRAY); + /* Array */ + obj = JS_NewCConstructor(ctx, JS_CLASS_ARRAY, "Array", + js_array_constructor, 1, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_array_funcs, countof(js_array_funcs), + js_array_proto_funcs, countof(js_array_proto_funcs), + JS_NEW_CTOR_PROTO_CLASS); + if (JS_IsException(obj)) + return -1; + ctx->array_ctor = obj; ctx->array_shape = js_new_shape2(ctx, get_proto_obj(ctx->class_proto[JS_CLASS_ARRAY]), JS_PROP_INITIAL_HASH_SIZE, 1); - add_shape_property(ctx, &ctx->array_shape, NULL, - JS_ATOM_length, JS_PROP_WRITABLE | JS_PROP_LENGTH); - - /* XXX: could test it on first context creation to ensure that no - new atoms are created in JS_AddIntrinsicBasicObjects(). It is - necessary to avoid useless renumbering of atoms after - JS_EvalBinary() if it is done just after - JS_AddIntrinsicBasicObjects(). */ - // assert(ctx->rt->atom_count == JS_ATOM_END); + if (!ctx->array_shape) + return -1; + if (add_shape_property(ctx, &ctx->array_shape, NULL, + JS_ATOM_length, JS_PROP_WRITABLE | JS_PROP_LENGTH)) + return -1; + ctx->std_array_prototype = TRUE; + + return 0; } -void JS_AddIntrinsicBaseObjects(JSContext *ctx) +int JS_AddIntrinsicBaseObjects(JSContext *ctx) { - int i; - JSValueConst obj, number_obj; - JSValue obj1; + JSValue obj1, obj2; + JSCFunctionType ft; ctx->throw_type_error = JS_NewCFunction(ctx, js_throw_type_error, NULL, 0); - + if (JS_IsException(ctx->throw_type_error)) + return -1; /* add caller and arguments properties to throw a TypeError */ - JS_DefineProperty(ctx, ctx->function_proto, JS_ATOM_caller, JS_UNDEFINED, - ctx->throw_type_error, ctx->throw_type_error, - JS_PROP_HAS_GET | JS_PROP_HAS_SET | - JS_PROP_HAS_CONFIGURABLE | JS_PROP_CONFIGURABLE); - JS_DefineProperty(ctx, ctx->function_proto, JS_ATOM_arguments, JS_UNDEFINED, - ctx->throw_type_error, ctx->throw_type_error, - JS_PROP_HAS_GET | JS_PROP_HAS_SET | - JS_PROP_HAS_CONFIGURABLE | JS_PROP_CONFIGURABLE); + if (JS_DefineProperty(ctx, ctx->function_proto, JS_ATOM_caller, JS_UNDEFINED, + ctx->throw_type_error, ctx->throw_type_error, + JS_PROP_HAS_GET | JS_PROP_HAS_SET | + JS_PROP_HAS_CONFIGURABLE | JS_PROP_CONFIGURABLE) < 0) + return -1; + if (JS_DefineProperty(ctx, ctx->function_proto, JS_ATOM_arguments, JS_UNDEFINED, + ctx->throw_type_error, ctx->throw_type_error, + JS_PROP_HAS_GET | JS_PROP_HAS_SET | + JS_PROP_HAS_CONFIGURABLE | JS_PROP_CONFIGURABLE) < 0) + return -1; JS_FreeValue(ctx, js_object_seal(ctx, JS_UNDEFINED, 1, (JSValueConst *)&ctx->throw_type_error, 1)); - ctx->global_obj = JS_NewObject(ctx); - ctx->global_var_obj = JS_NewObjectProto(ctx, JS_NULL); - /* Object */ - obj = JS_NewGlobalCConstructor(ctx, "Object", js_object_constructor, 1, - ctx->class_proto[JS_CLASS_OBJECT]); - JS_SetPropertyFunctionList(ctx, obj, js_object_funcs, countof(js_object_funcs)); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_OBJECT], - js_object_proto_funcs, countof(js_object_proto_funcs)); - + obj1 = JS_NewCConstructor(ctx, JS_CLASS_OBJECT, "Object", + js_object_constructor, 1, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_object_funcs, countof(js_object_funcs), + js_object_proto_funcs, countof(js_object_proto_funcs), + JS_NEW_CTOR_PROTO_EXIST); + if (JS_IsException(obj1)) + return -1; + JS_FreeValue(ctx, obj1); + /* Function */ - JS_SetPropertyFunctionList(ctx, ctx->function_proto, js_function_proto_funcs, countof(js_function_proto_funcs)); - ctx->function_ctor = JS_NewCFunctionMagic(ctx, js_function_constructor, - "Function", 1, JS_CFUNC_constructor_or_func_magic, - JS_FUNC_NORMAL); - JS_NewGlobalCConstructor2(ctx, JS_DupValue(ctx, ctx->function_ctor), "Function", - ctx->function_proto); - - /* Error */ - obj1 = JS_NewCFunctionMagic(ctx, js_error_constructor, - "Error", 1, JS_CFUNC_constructor_or_func_magic, -1); - JS_NewGlobalCConstructor2(ctx, obj1, - "Error", ctx->class_proto[JS_CLASS_ERROR]); - - /* Used to squelch a -Wcast-function-type warning. */ - JSCFunctionType ft = { .generic_magic = js_error_constructor }; - for(i = 0; i < JS_NATIVE_ERROR_COUNT; i++) { - JSValue func_obj; - int n_args; - n_args = 1 + (i == JS_AGGREGATE_ERROR); - func_obj = JS_NewCFunction3(ctx, ft.generic, - native_error_name[i], n_args, - JS_CFUNC_constructor_or_func_magic, i, obj1); - JS_NewGlobalCConstructor2(ctx, func_obj, native_error_name[i], - ctx->native_error_proto[i]); + ft.generic_magic = js_function_constructor; + obj1 = JS_NewCConstructor(ctx, JS_CLASS_BYTECODE_FUNCTION, "Function", + ft.generic, 1, JS_CFUNC_constructor_or_func_magic, JS_FUNC_NORMAL, + JS_UNDEFINED, + NULL, 0, + js_function_proto_funcs, countof(js_function_proto_funcs), + JS_NEW_CTOR_PROTO_EXIST); + if (JS_IsException(obj1)) + return -1; + ctx->function_ctor = obj1; + + /* Iterator */ + obj2 = JS_NewCConstructor(ctx, JS_CLASS_ITERATOR, "Iterator", + js_iterator_constructor, 0, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_iterator_funcs, countof(js_iterator_funcs), + js_iterator_proto_funcs, countof(js_iterator_proto_funcs), + 0); + if (JS_IsException(obj2)) + return -1; + // quirk: Iterator.prototype.constructor is an accessor property + // TODO(bnoordhuis) mildly inefficient because JS_NewGlobalCConstructor + // first creates a .constructor value property that we then replace with + // an accessor + obj1 = JS_NewCFunctionData(ctx, js_iterator_constructor_getset, + 0, 0, 1, (JSValueConst *)&obj2); + if (JS_IsException(obj1)) { + JS_FreeValue(ctx, obj2); + return -1; } - - /* Iterator prototype */ - ctx->iterator_proto = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->iterator_proto, - js_iterator_proto_funcs, - countof(js_iterator_proto_funcs)); - - /* Array */ - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_ARRAY], - js_array_proto_funcs, - countof(js_array_proto_funcs)); - - obj = JS_NewGlobalCConstructor(ctx, "Array", js_array_constructor, 1, - ctx->class_proto[JS_CLASS_ARRAY]); - ctx->array_ctor = JS_DupValue(ctx, obj); - JS_SetPropertyFunctionList(ctx, obj, js_array_funcs, - countof(js_array_funcs)); - - /* XXX: create auto_initializer */ - { - /* initialize Array.prototype[Symbol.unscopables] */ - static const char unscopables[] = - "at" "\0" - "copyWithin" "\0" - "entries" "\0" - "fill" "\0" - "find" "\0" - "findIndex" "\0" - "findLast" "\0" - "findLastIndex" "\0" - "flat" "\0" - "flatMap" "\0" - "includes" "\0" - "keys" "\0" - "toReversed" "\0" - "toSorted" "\0" - "toSpliced" "\0" - "values" "\0"; - const char *p = unscopables; - obj1 = JS_NewObjectProto(ctx, JS_NULL); - for(p = unscopables; *p; p += strlen(p) + 1) { - JS_DefinePropertyValueStr(ctx, obj1, p, JS_TRUE, JS_PROP_C_W_E); - } - JS_DefinePropertyValue(ctx, ctx->class_proto[JS_CLASS_ARRAY], - JS_ATOM_Symbol_unscopables, obj1, - JS_PROP_CONFIGURABLE); + if (JS_DefineProperty(ctx, ctx->class_proto[JS_CLASS_ITERATOR], + JS_ATOM_constructor, JS_UNDEFINED, + obj1, obj1, + JS_PROP_HAS_GET | JS_PROP_HAS_SET | JS_PROP_CONFIGURABLE) < 0) { + JS_FreeValue(ctx, obj2); + JS_FreeValue(ctx, obj1); + return -1; } + JS_FreeValue(ctx, obj1); + ctx->iterator_ctor = obj2; + + ctx->class_proto[JS_CLASS_ITERATOR_HELPER] = + JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_ITERATOR], + js_iterator_helper_proto_funcs, + countof(js_iterator_helper_proto_funcs)); + if (JS_IsException(ctx->class_proto[JS_CLASS_ITERATOR_HELPER])) + return -1; + + ctx->class_proto[JS_CLASS_ITERATOR_WRAP] = + JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_ITERATOR], + js_iterator_wrap_proto_funcs, + countof(js_iterator_wrap_proto_funcs)); + if (JS_IsException(ctx->class_proto[JS_CLASS_ITERATOR_WRAP])) + return -1; /* needed to initialize arguments[Symbol.iterator] */ ctx->array_proto_values = JS_GetProperty(ctx, ctx->class_proto[JS_CLASS_ARRAY], JS_ATOM_values); + if (JS_IsException(ctx->array_proto_values)) + return -1; - ctx->class_proto[JS_CLASS_ARRAY_ITERATOR] = JS_NewObjectProto(ctx, ctx->iterator_proto); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_ARRAY_ITERATOR], - js_array_iterator_proto_funcs, - countof(js_array_iterator_proto_funcs)); + ctx->class_proto[JS_CLASS_ARRAY_ITERATOR] = + JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_ITERATOR], + js_array_iterator_proto_funcs, + countof(js_array_iterator_proto_funcs)); + if (JS_IsException(ctx->class_proto[JS_CLASS_ARRAY_ITERATOR])) + return -1; /* parseFloat and parseInteger must be defined before Number because of the Number.parseFloat and Number.parseInteger aliases */ - JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_global_funcs, - countof(js_global_funcs)); + if (JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_global_funcs, + countof(js_global_funcs))) + return -1; /* Number */ - ctx->class_proto[JS_CLASS_NUMBER] = JS_NewObjectProtoClass(ctx, ctx->class_proto[JS_CLASS_OBJECT], - JS_CLASS_NUMBER); - JS_SetObjectData(ctx, ctx->class_proto[JS_CLASS_NUMBER], JS_NewInt32(ctx, 0)); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_NUMBER], - js_number_proto_funcs, - countof(js_number_proto_funcs)); - number_obj = JS_NewGlobalCConstructor(ctx, "Number", js_number_constructor, 1, - ctx->class_proto[JS_CLASS_NUMBER]); - JS_SetPropertyFunctionList(ctx, number_obj, js_number_funcs, countof(js_number_funcs)); - + obj1 = JS_NewCConstructor(ctx, JS_CLASS_NUMBER, "Number", + js_number_constructor, 1, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_number_funcs, countof(js_number_funcs), + js_number_proto_funcs, countof(js_number_proto_funcs), + JS_NEW_CTOR_PROTO_CLASS); + if (JS_IsException(obj1)) + return -1; + JS_FreeValue(ctx, obj1); + if (JS_SetObjectData(ctx, ctx->class_proto[JS_CLASS_NUMBER], JS_NewInt32(ctx, 0))) + return -1; + /* Boolean */ - ctx->class_proto[JS_CLASS_BOOLEAN] = JS_NewObjectProtoClass(ctx, ctx->class_proto[JS_CLASS_OBJECT], - JS_CLASS_BOOLEAN); - JS_SetObjectData(ctx, ctx->class_proto[JS_CLASS_BOOLEAN], JS_NewBool(ctx, FALSE)); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_BOOLEAN], js_boolean_proto_funcs, - countof(js_boolean_proto_funcs)); - JS_NewGlobalCConstructor(ctx, "Boolean", js_boolean_constructor, 1, - ctx->class_proto[JS_CLASS_BOOLEAN]); + obj1 = JS_NewCConstructor(ctx, JS_CLASS_BOOLEAN, "Boolean", + js_boolean_constructor, 1, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + NULL, 0, + js_boolean_proto_funcs, countof(js_boolean_proto_funcs), + JS_NEW_CTOR_PROTO_CLASS); + if (JS_IsException(obj1)) + return -1; + JS_FreeValue(ctx, obj1); + if (JS_SetObjectData(ctx, ctx->class_proto[JS_CLASS_BOOLEAN], JS_NewBool(ctx, FALSE))) + return -1; /* String */ - ctx->class_proto[JS_CLASS_STRING] = JS_NewObjectProtoClass(ctx, ctx->class_proto[JS_CLASS_OBJECT], - JS_CLASS_STRING); - JS_SetObjectData(ctx, ctx->class_proto[JS_CLASS_STRING], JS_AtomToString(ctx, JS_ATOM_empty_string)); - obj = JS_NewGlobalCConstructor(ctx, "String", js_string_constructor, 1, - ctx->class_proto[JS_CLASS_STRING]); - JS_SetPropertyFunctionList(ctx, obj, js_string_funcs, - countof(js_string_funcs)); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_STRING], js_string_proto_funcs, - countof(js_string_proto_funcs)); - - ctx->class_proto[JS_CLASS_STRING_ITERATOR] = JS_NewObjectProto(ctx, ctx->iterator_proto); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_STRING_ITERATOR], - js_string_iterator_proto_funcs, - countof(js_string_iterator_proto_funcs)); + obj1 = JS_NewCConstructor(ctx, JS_CLASS_STRING, "String", + js_string_constructor, 1, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_string_funcs, countof(js_string_funcs), + js_string_proto_funcs, countof(js_string_proto_funcs), + JS_NEW_CTOR_PROTO_CLASS); + if (JS_IsException(obj1)) + return -1; + JS_FreeValue(ctx, obj1); + if (JS_SetObjectData(ctx, ctx->class_proto[JS_CLASS_STRING], JS_AtomToString(ctx, JS_ATOM_empty_string))) + return -1; + + ctx->class_proto[JS_CLASS_STRING_ITERATOR] = + JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_ITERATOR], + js_string_iterator_proto_funcs, + countof(js_string_iterator_proto_funcs)); + if (JS_IsException(ctx->class_proto[JS_CLASS_STRING_ITERATOR])) + return -1; /* Math: create as autoinit object */ js_random_init(ctx); - JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_math_obj, countof(js_math_obj)); + if (JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_math_obj, countof(js_math_obj))) + return -1; /* ES6 Reflect: create as autoinit object */ - JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_reflect_obj, countof(js_reflect_obj)); + if (JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_reflect_obj, countof(js_reflect_obj))) + return -1; /* ES6 Symbol */ - ctx->class_proto[JS_CLASS_SYMBOL] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_SYMBOL], js_symbol_proto_funcs, - countof(js_symbol_proto_funcs)); - obj = JS_NewGlobalCConstructor(ctx, "Symbol", js_symbol_constructor, 0, - ctx->class_proto[JS_CLASS_SYMBOL]); - JS_SetPropertyFunctionList(ctx, obj, js_symbol_funcs, - countof(js_symbol_funcs)); - for(i = JS_ATOM_Symbol_toPrimitive; i < JS_ATOM_END; i++) { - char buf[ATOM_GET_STR_BUF_SIZE]; - const char *str, *p; - str = JS_AtomGetStr(ctx, buf, sizeof(buf), i); - /* skip "Symbol." */ - p = strchr(str, '.'); - if (p) - str = p + 1; - JS_DefinePropertyValueStr(ctx, obj, str, JS_AtomToValue(ctx, i), 0); - } - - /* ES6 Generator */ - ctx->class_proto[JS_CLASS_GENERATOR] = JS_NewObjectProto(ctx, ctx->iterator_proto); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_GENERATOR], - js_generator_proto_funcs, - countof(js_generator_proto_funcs)); - - ctx->class_proto[JS_CLASS_GENERATOR_FUNCTION] = JS_NewObjectProto(ctx, ctx->function_proto); - obj1 = JS_NewCFunction3(ctx, (JSCFunction *)js_function_constructor, - "GeneratorFunction", 1, - JS_CFUNC_constructor_or_func_magic, JS_FUNC_GENERATOR, - ctx->function_ctor); - JS_SetPropertyFunctionList(ctx, - ctx->class_proto[JS_CLASS_GENERATOR_FUNCTION], - js_generator_function_proto_funcs, - countof(js_generator_function_proto_funcs)); - JS_SetConstructor2(ctx, ctx->class_proto[JS_CLASS_GENERATOR_FUNCTION], - ctx->class_proto[JS_CLASS_GENERATOR], - JS_PROP_CONFIGURABLE, JS_PROP_CONFIGURABLE); - JS_SetConstructor2(ctx, obj1, ctx->class_proto[JS_CLASS_GENERATOR_FUNCTION], - 0, JS_PROP_CONFIGURABLE); + obj1 = JS_NewCConstructor(ctx, JS_CLASS_SYMBOL, "Symbol", + js_symbol_constructor, 0, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_symbol_funcs, countof(js_symbol_funcs), + js_symbol_proto_funcs, countof(js_symbol_proto_funcs), + 0); + if (JS_IsException(obj1)) + return -1; JS_FreeValue(ctx, obj1); + + /* ES6 Generator */ + ctx->class_proto[JS_CLASS_GENERATOR] = + JS_NewObjectProtoList(ctx, ctx->class_proto[JS_CLASS_ITERATOR], + js_generator_proto_funcs, + countof(js_generator_proto_funcs)); + if (JS_IsException(ctx->class_proto[JS_CLASS_GENERATOR])) + return -1; + ft.generic_magic = js_function_constructor; + obj1 = JS_NewCConstructor(ctx, JS_CLASS_GENERATOR_FUNCTION, "GeneratorFunction", + ft.generic, 1, JS_CFUNC_constructor_or_func_magic, JS_FUNC_GENERATOR, + ctx->function_ctor, + NULL, 0, + js_generator_function_proto_funcs, + countof(js_generator_function_proto_funcs), + JS_NEW_CTOR_NO_GLOBAL | JS_NEW_CTOR_READONLY); + if (JS_IsException(obj1)) + return -1; + JS_FreeValue(ctx, obj1); + if (JS_SetConstructor2(ctx, ctx->class_proto[JS_CLASS_GENERATOR_FUNCTION], + ctx->class_proto[JS_CLASS_GENERATOR], + JS_PROP_CONFIGURABLE, JS_PROP_CONFIGURABLE)) + return -1; + /* global properties */ - ctx->eval_obj = JS_NewCFunction(ctx, js_global_eval, "eval", 1); - JS_DefinePropertyValue(ctx, ctx->global_obj, JS_ATOM_eval, - JS_DupValue(ctx, ctx->eval_obj), - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - - JS_DefinePropertyValue(ctx, ctx->global_obj, JS_ATOM_globalThis, - JS_DupValue(ctx, ctx->global_obj), - JS_PROP_CONFIGURABLE | JS_PROP_WRITABLE); + ctx->eval_obj = JS_GetProperty(ctx, ctx->global_obj, JS_ATOM_eval); + if (JS_IsException(ctx->eval_obj)) + return -1; + + if (JS_DefinePropertyValue(ctx, ctx->global_obj, JS_ATOM_globalThis, + JS_DupValue(ctx, ctx->global_obj), + JS_PROP_CONFIGURABLE | JS_PROP_WRITABLE) < 0) + return -1; /* BigInt */ - JS_AddIntrinsicBigInt(ctx); + if (JS_AddIntrinsicBigInt(ctx)) + return -1; + return 0; } /* Typed Arrays */ static uint8_t const typed_array_size_log2[JS_TYPED_ARRAY_COUNT] = { 0, 0, 0, 1, 1, 2, 2, - 3, 3, /* BigInt64Array, BigUint64Array */ - 2, 3 + 3, 3, // BigInt64Array, BigUint64Array + 1, 2, 3 // Float16Array, Float32Array, Float64Array }; static JSValue js_array_buffer_constructor3(JSContext *ctx, JSValueConst new_target, - uint64_t len, JSClassID class_id, + uint64_t len, uint64_t *max_len, + JSClassID class_id, uint8_t *buf, JSFreeArrayBufferDataFunc *free_func, void *opaque, BOOL alloc_flag) @@ -51880,7 +55111,15 @@ static JSValue js_array_buffer_constructor3(JSContext *ctx, JSRuntime *rt = ctx->rt; JSValue obj; JSArrayBuffer *abuf = NULL; + uint64_t sab_alloc_len; + if (!alloc_flag && buf && max_len && free_func != js_array_buffer_free) { + // not observable from JS land, only through C API misuse; + // JS code cannot create externally managed buffers directly + return JS_ThrowInternalError(ctx, + "resizable ArrayBuffers not supported " + "for externally managed buffers"); + } obj = js_create_from_ctor(ctx, new_target, class_id); if (JS_IsException(obj)) return obj; @@ -51889,18 +55128,26 @@ static JSValue js_array_buffer_constructor3(JSContext *ctx, JS_ThrowRangeError(ctx, "invalid array buffer length"); goto fail; } + if (max_len && *max_len > INT32_MAX) { + JS_ThrowRangeError(ctx, "invalid max array buffer length"); + goto fail; + } abuf = js_malloc(ctx, sizeof(*abuf)); if (!abuf) goto fail; abuf->byte_length = len; + abuf->max_byte_length = max_len ? *max_len : -1; if (alloc_flag) { if (class_id == JS_CLASS_SHARED_ARRAY_BUFFER && rt->sab_funcs.sab_alloc) { + // TOOD(bnoordhuis) resizing backing memory for SABs atomically + // is hard so we cheat and allocate |maxByteLength| bytes upfront + sab_alloc_len = max_len ? *max_len : len; abuf->data = rt->sab_funcs.sab_alloc(rt->sab_funcs.sab_opaque, - max_int(len, 1)); + max_int(sab_alloc_len, 1)); if (!abuf->data) goto fail; - memset(abuf->data, 0, len); + memset(abuf->data, 0, sab_alloc_len); } else { /* the allocation must be done after the object creation */ abuf->data = js_mallocz(ctx, max_int(len, 1)); @@ -51936,18 +55183,19 @@ static void js_array_buffer_free(JSRuntime *rt, void *opaque, void *ptr) static JSValue js_array_buffer_constructor2(JSContext *ctx, JSValueConst new_target, - uint64_t len, JSClassID class_id) + uint64_t len, uint64_t *max_len, + JSClassID class_id) { - return js_array_buffer_constructor3(ctx, new_target, len, class_id, + return js_array_buffer_constructor3(ctx, new_target, len, max_len, class_id, NULL, js_array_buffer_free, NULL, TRUE); } static JSValue js_array_buffer_constructor1(JSContext *ctx, JSValueConst new_target, - uint64_t len) + uint64_t len, uint64_t *max_len) { - return js_array_buffer_constructor2(ctx, new_target, len, + return js_array_buffer_constructor2(ctx, new_target, len, max_len, JS_CLASS_ARRAY_BUFFER); } @@ -51955,39 +55203,70 @@ JSValue JS_NewArrayBuffer(JSContext *ctx, uint8_t *buf, size_t len, JSFreeArrayBufferDataFunc *free_func, void *opaque, BOOL is_shared) { - return js_array_buffer_constructor3(ctx, JS_UNDEFINED, len, - is_shared ? JS_CLASS_SHARED_ARRAY_BUFFER : JS_CLASS_ARRAY_BUFFER, + JSClassID class_id = + is_shared ? JS_CLASS_SHARED_ARRAY_BUFFER : JS_CLASS_ARRAY_BUFFER; + return js_array_buffer_constructor3(ctx, JS_UNDEFINED, len, NULL, class_id, buf, free_func, opaque, FALSE); } /* create a new ArrayBuffer of length 'len' and copy 'buf' to it */ JSValue JS_NewArrayBufferCopy(JSContext *ctx, const uint8_t *buf, size_t len) { - return js_array_buffer_constructor3(ctx, JS_UNDEFINED, len, + return js_array_buffer_constructor3(ctx, JS_UNDEFINED, len, NULL, JS_CLASS_ARRAY_BUFFER, (uint8_t *)buf, js_array_buffer_free, NULL, TRUE); } +static JSValue js_array_buffer_constructor0(JSContext *ctx, JSValueConst new_target, + int argc, JSValueConst *argv, + JSClassID class_id) + { + uint64_t len, max_len, *pmax_len = NULL; + JSValue obj, val; + int64_t i; + + if (JS_ToIndex(ctx, &len, argv[0])) + return JS_EXCEPTION; + if (argc < 2) + goto next; + if (!JS_IsObject(argv[1])) + goto next; + obj = JS_ToObject(ctx, argv[1]); + if (JS_IsException(obj)) + return JS_EXCEPTION; + val = JS_GetProperty(ctx, obj, JS_ATOM_maxByteLength); + JS_FreeValue(ctx, obj); + if (JS_IsException(val)) + return JS_EXCEPTION; + if (JS_IsUndefined(val)) + goto next; + if (JS_ToInt64Free(ctx, &i, val)) + return JS_EXCEPTION; + // don't have to check i < 0 because len >= 0 + if (len > i || i > MAX_SAFE_INTEGER) + return JS_ThrowRangeError(ctx, "invalid array buffer max length"); + max_len = i; + pmax_len = &max_len; +next: + return js_array_buffer_constructor2(ctx, new_target, len, pmax_len, + class_id); +} + static JSValue js_array_buffer_constructor(JSContext *ctx, JSValueConst new_target, int argc, JSValueConst *argv) { - uint64_t len; - if (JS_ToIndex(ctx, &len, argv[0])) - return JS_EXCEPTION; - return js_array_buffer_constructor1(ctx, new_target, len); + return js_array_buffer_constructor0(ctx, new_target, argc, argv, + JS_CLASS_ARRAY_BUFFER); } static JSValue js_shared_array_buffer_constructor(JSContext *ctx, JSValueConst new_target, int argc, JSValueConst *argv) { - uint64_t len; - if (JS_ToIndex(ctx, &len, argv[0])) - return JS_EXCEPTION; - return js_array_buffer_constructor2(ctx, new_target, len, + return js_array_buffer_constructor0(ctx, new_target, argc, argv, JS_CLASS_SHARED_ARRAY_BUFFER); } @@ -52053,6 +55332,23 @@ static JSValue JS_ThrowTypeErrorDetachedArrayBuffer(JSContext *ctx) return JS_ThrowTypeError(ctx, "ArrayBuffer is detached"); } +static JSValue JS_ThrowTypeErrorArrayBufferOOB(JSContext *ctx) +{ + return JS_ThrowTypeError(ctx, "ArrayBuffer is detached or resized"); +} + +// #sec-get-arraybuffer.prototype.detached +static JSValue js_array_buffer_get_detached(JSContext *ctx, + JSValueConst this_val) +{ + JSArrayBuffer *abuf = JS_GetOpaque2(ctx, this_val, JS_CLASS_ARRAY_BUFFER); + if (!abuf) + return JS_EXCEPTION; + if (abuf->shared) + return JS_ThrowTypeError(ctx, "detached called on SharedArrayBuffer"); + return JS_NewBool(ctx, abuf->detached); +} + static JSValue js_array_buffer_get_byteLength(JSContext *ctx, JSValueConst this_val, int class_id) @@ -52064,10 +55360,74 @@ static JSValue js_array_buffer_get_byteLength(JSContext *ctx, return JS_NewUint32(ctx, abuf->byte_length); } +static JSValue js_array_buffer_get_maxByteLength(JSContext *ctx, + JSValueConst this_val, + int class_id) +{ + JSArrayBuffer *abuf = JS_GetOpaque2(ctx, this_val, class_id); + if (!abuf) + return JS_EXCEPTION; + if (array_buffer_is_resizable(abuf)) + return JS_NewUint32(ctx, abuf->max_byte_length); + return JS_NewUint32(ctx, abuf->byte_length); +} + +static JSValue js_array_buffer_get_resizable(JSContext *ctx, + JSValueConst this_val, + int class_id) +{ + JSArrayBuffer *abuf = JS_GetOpaque2(ctx, this_val, class_id); + if (!abuf) + return JS_EXCEPTION; + return JS_NewBool(ctx, array_buffer_is_resizable(abuf)); +} + +static void js_array_buffer_update_typed_arrays(JSArrayBuffer *abuf) +{ + uint32_t size_log2, size_elem; + struct list_head *el; + JSTypedArray *ta; + JSObject *p; + uint8_t *data; + int64_t len; + + len = abuf->byte_length; + data = abuf->data; + // update lengths of all typed arrays backed by this array buffer + list_for_each(el, &abuf->array_list) { + ta = list_entry(el, JSTypedArray, link); + p = ta->obj; + if (p->class_id == JS_CLASS_DATAVIEW) { + if (ta->track_rab) { + if (ta->offset < len) + ta->length = len - ta->offset; + else + ta->length = 0; + } + } else { + p->u.array.count = 0; + p->u.array.u.ptr = NULL; + size_log2 = typed_array_size_log2(p->class_id); + size_elem = 1 << size_log2; + if (ta->track_rab) { + if (len >= (int64_t)ta->offset + size_elem) { + p->u.array.count = (len - ta->offset) >> size_log2; + p->u.array.u.ptr = &data[ta->offset]; + } + } else { + if (len >= (int64_t)ta->offset + ta->length) { + p->u.array.count = ta->length >> size_log2; + p->u.array.u.ptr = &data[ta->offset]; + } + } + } + } + +} + void JS_DetachArrayBuffer(JSContext *ctx, JSValueConst obj) { JSArrayBuffer *abuf = JS_GetOpaque(obj, JS_CLASS_ARRAY_BUFFER); - struct list_head *el; if (!abuf || abuf->detached) return; @@ -52076,19 +55436,7 @@ void JS_DetachArrayBuffer(JSContext *ctx, JSValueConst obj) abuf->data = NULL; abuf->byte_length = 0; abuf->detached = TRUE; - - list_for_each(el, &abuf->array_list) { - JSTypedArray *ta; - JSObject *p; - - ta = list_entry(el, JSTypedArray, link); - p = ta->obj; - /* Note: the typed array length and offset fields are not modified */ - if (p->class_id != JS_CLASS_DATAVIEW) { - p->u.array.count = 0; - p->u.array.u.ptr = NULL; - } - } + js_array_buffer_update_typed_arrays(abuf); } /* get an ArrayBuffer or SharedArrayBuffer */ @@ -52125,6 +55473,142 @@ uint8_t *JS_GetArrayBuffer(JSContext *ctx, size_t *psize, JSValueConst obj) return NULL; } +static BOOL array_buffer_is_resizable(const JSArrayBuffer *abuf) +{ + return abuf->max_byte_length >= 0; +} + +// ES #sec-arraybuffer.prototype.transfer +static JSValue js_array_buffer_transfer(JSContext *ctx, + JSValueConst this_val, + int argc, JSValueConst *argv, + int transfer_to_fixed_length) +{ + JSArrayBuffer *abuf; + uint64_t new_len, *pmax_len, max_len; + + abuf = JS_GetOpaque2(ctx, this_val, JS_CLASS_ARRAY_BUFFER); + if (!abuf) + return JS_EXCEPTION; + if (abuf->shared) + return JS_ThrowTypeError(ctx, "cannot transfer a SharedArrayBuffer"); + if (argc < 1 || JS_IsUndefined(argv[0])) + new_len = abuf->byte_length; + else if (JS_ToIndex(ctx, &new_len, argv[0])) + return JS_EXCEPTION; + if (abuf->detached) + return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + pmax_len = NULL; + if (!transfer_to_fixed_length) { + if (array_buffer_is_resizable(abuf)) { // carry over maxByteLength + max_len = abuf->max_byte_length; + if (new_len > max_len) + return JS_ThrowTypeError(ctx, "invalid array buffer length"); + // TODO(bnoordhuis) support externally managed RABs + if (abuf->free_func == js_array_buffer_free) + pmax_len = &max_len; + } + } + /* create an empty AB */ + if (new_len == 0) { + JS_DetachArrayBuffer(ctx, this_val); + return js_array_buffer_constructor2(ctx, JS_UNDEFINED, 0, pmax_len, JS_CLASS_ARRAY_BUFFER); + } else { + uint64_t old_len; + uint8_t *bs, *new_bs; + JSFreeArrayBufferDataFunc *free_func; + + bs = abuf->data; + old_len = abuf->byte_length; + free_func = abuf->free_func; + + /* if length mismatch, realloc. Otherwise, use the same backing buffer. */ + if (new_len != old_len) { + /* XXX: we are currently limited to 2 GB */ + if (new_len > INT32_MAX) + return JS_ThrowRangeError(ctx, "invalid array buffer length"); + + if (free_func != js_array_buffer_free) { + /* cannot use js_realloc() because the buffer was + allocated with a custom allocator */ + new_bs = js_mallocz(ctx, new_len); + if (!new_bs) + return JS_EXCEPTION; + memcpy(new_bs, bs, min_int(old_len, new_len)); + abuf->free_func(ctx->rt, abuf->opaque, bs); + bs = new_bs; + free_func = js_array_buffer_free; + } else { + new_bs = js_realloc(ctx, bs, new_len); + if (!new_bs) + return JS_EXCEPTION; + bs = new_bs; + if (new_len > old_len) + memset(bs + old_len, 0, new_len - old_len); + } + } + /* neuter the backing buffer */ + abuf->data = NULL; + abuf->byte_length = 0; + abuf->detached = TRUE; + js_array_buffer_update_typed_arrays(abuf); + return js_array_buffer_constructor3(ctx, JS_UNDEFINED, new_len, pmax_len, + JS_CLASS_ARRAY_BUFFER, + bs, free_func, + NULL, FALSE); + } +} + +static JSValue js_array_buffer_resize(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv, int class_id) +{ + JSArrayBuffer *abuf; + uint8_t *data; + int64_t len; + + abuf = JS_GetOpaque2(ctx, this_val, class_id); + if (!abuf) + return JS_EXCEPTION; + if (JS_ToInt64(ctx, &len, argv[0])) + return JS_EXCEPTION; + if (abuf->detached) + return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + if (!array_buffer_is_resizable(abuf)) + return JS_ThrowTypeError(ctx, "array buffer is not resizable"); + // TODO(bnoordhuis) support externally managed RABs + if (abuf->free_func != js_array_buffer_free) + return JS_ThrowTypeError(ctx, "external array buffer is not resizable"); + if (len < 0 || len > abuf->max_byte_length) { + bad_length: + return JS_ThrowRangeError(ctx, "invalid array buffer length"); + } + // SABs can only grow and we don't need to realloc because + // js_array_buffer_constructor3 commits all memory upfront; + // regular RABs are resizable both ways and realloc + if (abuf->shared) { + if (len < abuf->byte_length) + goto bad_length; + // Note this is off-spec; there's supposed to be a single atomic + // |byteLength| property that's shared across SABs but we store + // it per SAB instead. That means when thread A calls sab.grow(2) + // at time t0, and thread B calls sab.grow(1) at time t1, we don't + // throw a TypeError in thread B as the spec says we should, + // instead both threads get their own view of the backing memory, + // 2 bytes big in A, and 1 byte big in B + abuf->byte_length = len; + } else { + data = js_realloc(ctx, abuf->data, max_int(len, 1)); + if (!data) + return JS_EXCEPTION; + if (len > abuf->byte_length) + memset(&data[abuf->byte_length], 0, len - abuf->byte_length); + abuf->byte_length = len; + abuf->data = data; + } + js_array_buffer_update_typed_arrays(abuf); + return JS_UNDEFINED; +} + static JSValue js_array_buffer_slice(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv, int class_id) @@ -52154,7 +55638,7 @@ static JSValue js_array_buffer_slice(JSContext *ctx, return ctor; if (JS_IsUndefined(ctor)) { new_obj = js_array_buffer_constructor2(ctx, JS_UNDEFINED, new_len, - class_id); + NULL, class_id); } else { JSValue args[1]; args[0] = JS_NewInt64(ctx, new_len); @@ -52180,7 +55664,7 @@ static JSValue js_array_buffer_slice(JSContext *ctx, goto fail; } /* must test again because of side effects */ - if (abuf->detached) { + if (abuf->detached || abuf->byte_length < start + new_len) { JS_ThrowTypeErrorDetachedArrayBuffer(ctx); goto fail; } @@ -52193,7 +55677,13 @@ static JSValue js_array_buffer_slice(JSContext *ctx, static const JSCFunctionListEntry js_array_buffer_proto_funcs[] = { JS_CGETSET_MAGIC_DEF("byteLength", js_array_buffer_get_byteLength, NULL, JS_CLASS_ARRAY_BUFFER ), + JS_CGETSET_MAGIC_DEF("maxByteLength", js_array_buffer_get_maxByteLength, NULL, JS_CLASS_ARRAY_BUFFER ), + JS_CGETSET_MAGIC_DEF("resizable", js_array_buffer_get_resizable, NULL, JS_CLASS_ARRAY_BUFFER ), + JS_CGETSET_DEF("detached", js_array_buffer_get_detached, NULL ), + JS_CFUNC_MAGIC_DEF("resize", 1, js_array_buffer_resize, JS_CLASS_ARRAY_BUFFER ), JS_CFUNC_MAGIC_DEF("slice", 2, js_array_buffer_slice, JS_CLASS_ARRAY_BUFFER ), + JS_CFUNC_MAGIC_DEF("transfer", 0, js_array_buffer_transfer, 0 ), + JS_CFUNC_MAGIC_DEF("transferToFixedLength", 0, js_array_buffer_transfer, 1 ), JS_PROP_STRING_DEF("[Symbol.toStringTag]", "ArrayBuffer", JS_PROP_CONFIGURABLE ), }; @@ -52205,59 +55695,85 @@ static const JSCFunctionListEntry js_shared_array_buffer_funcs[] = { static const JSCFunctionListEntry js_shared_array_buffer_proto_funcs[] = { JS_CGETSET_MAGIC_DEF("byteLength", js_array_buffer_get_byteLength, NULL, JS_CLASS_SHARED_ARRAY_BUFFER ), + JS_CGETSET_MAGIC_DEF("maxByteLength", js_array_buffer_get_maxByteLength, NULL, JS_CLASS_SHARED_ARRAY_BUFFER ), + JS_CGETSET_MAGIC_DEF("growable", js_array_buffer_get_resizable, NULL, JS_CLASS_SHARED_ARRAY_BUFFER ), + JS_CFUNC_MAGIC_DEF("grow", 1, js_array_buffer_resize, JS_CLASS_SHARED_ARRAY_BUFFER ), JS_CFUNC_MAGIC_DEF("slice", 2, js_array_buffer_slice, JS_CLASS_SHARED_ARRAY_BUFFER ), JS_PROP_STRING_DEF("[Symbol.toStringTag]", "SharedArrayBuffer", JS_PROP_CONFIGURABLE ), }; -static JSObject *get_typed_array(JSContext *ctx, - JSValueConst this_val, - int is_dataview) +static JSObject *get_typed_array(JSContext *ctx, JSValueConst this_val) { JSObject *p; if (JS_VALUE_GET_TAG(this_val) != JS_TAG_OBJECT) goto fail; p = JS_VALUE_GET_OBJ(this_val); - if (is_dataview) { - if (p->class_id != JS_CLASS_DATAVIEW) - goto fail; - } else { - if (!(p->class_id >= JS_CLASS_UINT8C_ARRAY && - p->class_id <= JS_CLASS_FLOAT64_ARRAY)) { - fail: - JS_ThrowTypeError(ctx, "not a %s", is_dataview ? "DataView" : "TypedArray"); - return NULL; - } + if (!(p->class_id >= JS_CLASS_UINT8C_ARRAY && + p->class_id <= JS_CLASS_FLOAT64_ARRAY)) { + fail: + JS_ThrowTypeError(ctx, "not a TypedArray"); + return NULL; } return p; } -/* WARNING: 'p' must be a typed array */ -static BOOL typed_array_is_detached(JSContext *ctx, JSObject *p) +// is the typed array detached or out of bounds relative to its RAB? +// |p| must be a typed array, *not* a DataView +static BOOL typed_array_is_oob(JSObject *p) { - JSTypedArray *ta = p->u.typed_array; - JSArrayBuffer *abuf = ta->buffer->u.array_buffer; - /* XXX: could simplify test by ensuring that - p->u.array.u.ptr is NULL iff it is detached */ - return abuf->detached; + JSArrayBuffer *abuf; + JSTypedArray *ta; + int len, size_elem; + int64_t end; + + assert(p->class_id >= JS_CLASS_UINT8C_ARRAY); + assert(p->class_id <= JS_CLASS_FLOAT64_ARRAY); + + ta = p->u.typed_array; + abuf = ta->buffer->u.array_buffer; + if (abuf->detached) + return TRUE; + len = abuf->byte_length; + if (ta->offset > len) + return TRUE; + if (ta->track_rab) + return FALSE; + if (len < (int64_t)ta->offset + ta->length) + return TRUE; + size_elem = 1 << typed_array_size_log2(p->class_id); + end = (int64_t)ta->offset + (int64_t)p->u.array.count * size_elem; + return end > len; } -/* WARNING: 'p' must be a typed array. Works even if the array buffer - is detached */ -static uint32_t typed_array_get_length(JSContext *ctx, JSObject *p) +// Be *very* careful if you touch the typed array's memory directly: +// the length is only valid until the next call into JS land because +// JS code can detach or resize the backing array buffer. Functions +// like JS_GetProperty and JS_ToIndex call JS code. +// +// Exclusively reading or writing elements with JS_GetProperty, +// JS_GetPropertyInt64, JS_SetProperty, etc. is safe because they +// perform bounds checks, as does js_get_fast_array_element. +static int js_typed_array_get_length_unsafe(JSContext *ctx, JSValueConst obj) { - JSTypedArray *ta = p->u.typed_array; - int size_log2 = typed_array_size_log2(p->class_id); - return ta->length >> size_log2; + JSObject *p; + p = get_typed_array(ctx, obj); + if (!p) + return -1; + if (typed_array_is_oob(p)) { + JS_ThrowTypeErrorArrayBufferOOB(ctx); + return -1; + } + return p->u.array.count; } static int validate_typed_array(JSContext *ctx, JSValueConst this_val) { JSObject *p; - p = get_typed_array(ctx, this_val, 0); + p = get_typed_array(ctx, this_val); if (!p) return -1; - if (typed_array_is_detached(ctx, p)) { - JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + if (typed_array_is_oob(p)) { + JS_ThrowTypeErrorArrayBufferOOB(ctx); return -1; } return 0; @@ -52267,18 +55783,18 @@ static JSValue js_typed_array_get_length(JSContext *ctx, JSValueConst this_val) { JSObject *p; - p = get_typed_array(ctx, this_val, 0); + p = get_typed_array(ctx, this_val); if (!p) return JS_EXCEPTION; return JS_NewInt32(ctx, p->u.array.count); } static JSValue js_typed_array_get_buffer(JSContext *ctx, - JSValueConst this_val, int is_dataview) + JSValueConst this_val) { JSObject *p; JSTypedArray *ta; - p = get_typed_array(ctx, this_val, is_dataview); + p = get_typed_array(ctx, this_val); if (!p) return JS_EXCEPTION; ta = p->u.typed_array; @@ -52286,43 +55802,36 @@ static JSValue js_typed_array_get_buffer(JSContext *ctx, } static JSValue js_typed_array_get_byteLength(JSContext *ctx, - JSValueConst this_val, - int is_dataview) + JSValueConst this_val) { JSObject *p; JSTypedArray *ta; - p = get_typed_array(ctx, this_val, is_dataview); + int size_log2; + + p = get_typed_array(ctx, this_val); if (!p) return JS_EXCEPTION; - if (typed_array_is_detached(ctx, p)) { - if (is_dataview) { - return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); - } else { - return JS_NewInt32(ctx, 0); - } - } + if (typed_array_is_oob(p)) + return JS_NewInt32(ctx, 0); ta = p->u.typed_array; - return JS_NewInt32(ctx, ta->length); + if (!ta->track_rab) + return JS_NewUint32(ctx, ta->length); + size_log2 = typed_array_size_log2(p->class_id); + return JS_NewInt64(ctx, (int64_t)p->u.array.count << size_log2); } static JSValue js_typed_array_get_byteOffset(JSContext *ctx, - JSValueConst this_val, - int is_dataview) + JSValueConst this_val) { JSObject *p; JSTypedArray *ta; - p = get_typed_array(ctx, this_val, is_dataview); + p = get_typed_array(ctx, this_val); if (!p) return JS_EXCEPTION; - if (typed_array_is_detached(ctx, p)) { - if (is_dataview) { - return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); - } else { - return JS_NewInt32(ctx, 0); - } - } + if (typed_array_is_oob(p)) + return JS_NewInt32(ctx, 0); ta = p->u.typed_array; - return JS_NewInt32(ctx, ta->offset); + return JS_NewUint32(ctx, ta->offset); } JSValue JS_NewTypedArray(JSContext *ctx, int argc, JSValueConst *argv, @@ -52345,11 +55854,11 @@ JSValue JS_GetTypedArrayBuffer(JSContext *ctx, JSValueConst obj, { JSObject *p; JSTypedArray *ta; - p = get_typed_array(ctx, obj, FALSE); + p = get_typed_array(ctx, obj); if (!p) return JS_EXCEPTION; - if (typed_array_is_detached(ctx, p)) - return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); ta = p->u.typed_array; if (pbyte_offset) *pbyte_offset = ta->offset; @@ -52382,21 +55891,22 @@ static JSValue js_typed_array_set_internal(JSContext *ctx, JSObject *p; JSObject *src_p; uint32_t i; - int64_t src_len, offset; + int64_t dst_len, src_len, offset; JSValue val, src_obj = JS_UNDEFINED; - p = get_typed_array(ctx, dst, 0); + p = get_typed_array(ctx, dst); if (!p) goto fail; if (JS_ToInt64Sat(ctx, &offset, off)) goto fail; if (offset < 0) goto range_error; - if (typed_array_is_detached(ctx, p)) { + if (typed_array_is_oob(p)) { detached: - JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + JS_ThrowTypeErrorArrayBufferOOB(ctx); goto fail; } + dst_len = p->u.array.count; src_obj = JS_ToObject(ctx, src); if (JS_IsException(src_obj)) goto fail; @@ -52409,11 +55919,11 @@ static JSValue js_typed_array_set_internal(JSContext *ctx, JSArrayBuffer *src_abuf = src_ta->buffer->u.array_buffer; int shift = typed_array_size_log2(p->class_id); - if (src_abuf->detached) + if (typed_array_is_oob(src_p)) goto detached; src_len = src_p->u.array.count; - if (offset > (int64_t)(p->u.array.count - src_len)) + if (offset > dst_len - src_len) goto range_error; /* copying between typed objects */ @@ -52429,9 +55939,11 @@ static JSValue js_typed_array_set_internal(JSContext *ctx, } /* otherwise, default behavior is slow but correct */ } else { + // can change |dst| as a side effect; per spec, + // perform the range check against its old length if (js_get_length64(ctx, &src_len, src_obj)) goto fail; - if (offset > (int64_t)(p->u.array.count - src_len)) { + if (offset > dst_len - src_len) { range_error: JS_ThrowRangeError(ctx, "invalid array length"); goto fail; @@ -52458,21 +55970,22 @@ static JSValue js_typed_array_at(JSContext *ctx, JSValueConst this_val, JSObject *p; int64_t idx, len; - p = get_typed_array(ctx, this_val, 0); + p = get_typed_array(ctx, this_val); if (!p) return JS_EXCEPTION; - if (typed_array_is_detached(ctx, p)) { - JS_ThrowTypeErrorDetachedArrayBuffer(ctx); - return JS_EXCEPTION; - } + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + len = p->u.array.count; + // note: can change p->u.array.count if (JS_ToInt64Sat(ctx, &idx, argv[0])) return JS_EXCEPTION; - len = p->u.array.count; if (idx < 0) idx = len + idx; + + len = p->u.array.count; if (idx < 0 || idx >= len) return JS_UNDEFINED; return JS_GetPropertyInt64(ctx, this_val, idx); @@ -52485,16 +55998,16 @@ static JSValue js_typed_array_with(JSContext *ctx, JSValueConst this_val, JSObject *p; int64_t idx, len; - p = get_typed_array(ctx, this_val, /*is_dataview*/0); + p = get_typed_array(ctx, this_val); if (!p) return JS_EXCEPTION; - if (typed_array_is_detached(ctx, p)) + if (typed_array_is_oob(p)) return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + len = p->u.array.count; if (JS_ToInt64Sat(ctx, &idx, argv[0])) return JS_EXCEPTION; - len = p->u.array.count; if (idx < 0) idx = len + idx; @@ -52502,11 +56015,11 @@ static JSValue js_typed_array_with(JSContext *ctx, JSValueConst this_val, if (JS_IsException(val)) return JS_EXCEPTION; - if (typed_array_is_detached(ctx, p) || idx < 0 || idx >= len) + if (typed_array_is_oob(p) || idx < 0 || idx >= p->u.array.count) return JS_ThrowRangeError(ctx, "invalid array index"); arr = js_typed_array_constructor_ta(ctx, JS_UNDEFINED, this_val, - p->class_id); + p->class_id, len); if (JS_IsException(arr)) { JS_FreeValue(ctx, val); return JS_EXCEPTION; @@ -52537,41 +56050,6 @@ static JSValue js_create_typed_array_iterator(JSContext *ctx, JSValueConst this_ return js_create_array_iterator(ctx, this_val, argc, argv, magic); } -/* return < 0 if exception */ -static int js_typed_array_get_length_internal(JSContext *ctx, - JSValueConst obj) -{ - JSObject *p; - p = get_typed_array(ctx, obj, 0); - if (!p) - return -1; - if (typed_array_is_detached(ctx, p)) { - JS_ThrowTypeErrorDetachedArrayBuffer(ctx); - return -1; - } - return p->u.array.count; -} - -#if 0 -/* validate a typed array and return its length */ -static JSValue js_typed_array___getLength(JSContext *ctx, - JSValueConst this_val, - int argc, JSValueConst *argv) -{ - BOOL ignore_detached = JS_ToBool(ctx, argv[1]); - - if (ignore_detached) { - return js_typed_array_get_length(ctx, argv[0]); - } else { - int len; - len = js_typed_array_get_length_internal(ctx, argv[0]); - if (len < 0) - return JS_EXCEPTION; - return JS_NewInt32(ctx, len); - } -} -#endif - static JSValue js_typed_array_create(JSContext *ctx, JSValueConst ctor, int argc, JSValueConst *argv) { @@ -52583,7 +56061,7 @@ static JSValue js_typed_array_create(JSContext *ctx, JSValueConst ctor, if (JS_IsException(ret)) return ret; /* validate the typed array */ - new_len = js_typed_array_get_length_internal(ctx, ret); + new_len = js_typed_array_get_length_unsafe(ctx, ret); if (new_len < 0) goto fail; if (argc == 1) { @@ -52619,7 +56097,7 @@ static JSValue js_typed_array___speciesCreate(JSContext *ctx, int argc1; obj = argv[0]; - p = get_typed_array(ctx, obj, 0); + p = get_typed_array(ctx, obj); if (!p) return JS_EXCEPTION; ctor = JS_SpeciesConstructor(ctx, obj, JS_UNDEFINED); @@ -52740,11 +56218,14 @@ static JSValue js_typed_array_copyWithin(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { JSObject *p; - int len, to, from, final, count, shift; + int len, to, from, final, count, shift, space; - len = js_typed_array_get_length_internal(ctx, this_val); - if (len < 0) + p = get_typed_array(ctx, this_val); + if (!p) return JS_EXCEPTION; + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + len = p->u.array.count; if (JS_ToInt32Clamp(ctx, &to, argv[0], 0, len, len)) return JS_EXCEPTION; @@ -52758,11 +56239,14 @@ static JSValue js_typed_array_copyWithin(JSContext *ctx, JSValueConst this_val, return JS_EXCEPTION; } + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + + // RAB may have been resized by evil .valueOf method + space = p->u.array.count - max_int(to, from); count = min_int(final - from, len - to); + count = min_int(count, space); if (count > 0) { - p = JS_VALUE_GET_OBJ(this_val); - if (typed_array_is_detached(ctx, p)) - return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); shift = typed_array_size_log2(p->class_id); memmove(p->u.array.u.uint8_ptr + (to << shift), p->u.array.u.uint8_ptr + (from << shift), @@ -52778,10 +56262,12 @@ static JSValue js_typed_array_fill(JSContext *ctx, JSValueConst this_val, int len, k, final, shift; uint64_t v64; - len = js_typed_array_get_length_internal(ctx, this_val); - if (len < 0) + p = get_typed_array(ctx, this_val); + if (!p) return JS_EXCEPTION; - p = JS_VALUE_GET_OBJ(this_val); + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + len = p->u.array.count; if (p->class_id == JS_CLASS_UINT8C_ARRAY) { int32_t v; @@ -52800,7 +56286,9 @@ static JSValue js_typed_array_fill(JSContext *ctx, JSValueConst this_val, double d; if (JS_ToFloat64(ctx, &d, argv[0])) return JS_EXCEPTION; - if (p->class_id == JS_CLASS_FLOAT32_ARRAY) { + if (p->class_id == JS_CLASS_FLOAT16_ARRAY) { + v64 = tofp16(d); + } else if (p->class_id == JS_CLASS_FLOAT32_ARRAY) { union { float f; uint32_t u32; @@ -52826,9 +56314,11 @@ static JSValue js_typed_array_fill(JSContext *ctx, JSValueConst this_val, return JS_EXCEPTION; } - if (typed_array_is_detached(ctx, p)) - return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + // RAB may have been resized by evil .valueOf method + final = min_int(final, p->u.array.count); shift = typed_array_size_log2(p->class_id); switch(shift) { case 0: @@ -52867,7 +56357,7 @@ static JSValue js_typed_array_find(JSContext *ctx, JSValueConst this_val, int dir; val = JS_UNDEFINED; - len = js_typed_array_get_length_internal(ctx, this_val); + len = js_typed_array_get_length_unsafe(ctx, this_val); if (len < 0) goto exception; @@ -52931,32 +56421,27 @@ static JSValue js_typed_array_indexOf(JSContext *ctx, JSValueConst this_val, int64_t v64; double d; float f; + uint16_t hf; + + p = get_typed_array(ctx, this_val); + if (!p) + return JS_EXCEPTION; + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + len = p->u.array.count; - len = js_typed_array_get_length_internal(ctx, this_val); - if (len < 0) - goto exception; if (len == 0) goto done; if (special == special_lastIndexOf) { k = len - 1; if (argc > 1) { - if (JS_ToFloat64(ctx, &d, argv[1])) + int64_t k1; + if (JS_ToInt64Clamp(ctx, &k1, argv[1], -1, len - 1, len)) goto exception; - if (isnan(d)) { - k = 0; - } else { - if (d >= 0) { - if (d < k) { - k = d; - } - } else { - d += len; - if (d < 0) - goto done; - k = d; - } - } + k = k1; + if (k < 0) + goto done; } stop = -1; inc = -1; @@ -52970,16 +56455,23 @@ static JSValue js_typed_array_indexOf(JSContext *ctx, JSValueConst this_val, inc = 1; } - p = JS_VALUE_GET_OBJ(this_val); - /* if the array was detached, no need to go further (but no - exception is raised) */ - if (typed_array_is_detached(ctx, p)) { - /* "includes" scans all the properties, so "undefined" can match */ - if (special == special_includes && JS_IsUndefined(argv[0]) && len > 0) - res = 0; + /* includes function: 'undefined' can be found if searching out of bounds */ + if (len > p->u.array.count && special == special_includes && + JS_IsUndefined(argv[0]) && k < len) { + res = 0; goto done; } + // RAB may have been resized by evil .valueOf method + len = min_int(len, p->u.array.count); + if (len == 0) + goto done; + if (special == special_lastIndexOf) + k = min_int(k, len - 1); + else + k = min_int(k, len); + stop = min_int(stop, len); + is_bigint = 0; is_int = 0; /* avoid warning */ v64 = 0; /* avoid warning */ @@ -53042,7 +56534,9 @@ static JSValue js_typed_array_indexOf(JSContext *ctx, JSValueConst this_val, pv = p->u.array.u.uint8_ptr; v = v64; if (inc > 0) { - pp = memchr(pv + k, v, len - k); + pp = NULL; + if (pv) + pp = memchr(pv + k, v, len - k); if (pp) res = pp - pv; } else { @@ -53093,6 +56587,39 @@ static JSValue js_typed_array_indexOf(JSContext *ctx, JSValueConst this_val, } } break; + case JS_CLASS_FLOAT16_ARRAY: + if (is_bigint) + break; + if (isnan(d)) { + const uint16_t *pv = p->u.array.u.fp16_ptr; + /* special case: indexOf returns -1, includes finds NaN */ + if (special != special_includes) + goto done; + for (; k != stop; k += inc) { + if (isfp16nan(pv[k])) { + res = k; + break; + } + } + } else if (d == 0) { + // special case: includes also finds negative zero + const uint16_t *pv = p->u.array.u.fp16_ptr; + for (; k != stop; k += inc) { + if (isfp16zero(pv[k])) { + res = k; + break; + } + } + } else if (hf = tofp16(d), d == fromfp16(hf)) { + const uint16_t *pv = p->u.array.u.fp16_ptr; + for (; k != stop; k += inc) { + if (pv[k] == hf) { + res = k; + break; + } + } + } + break; case JS_CLASS_FLOAT32_ARRAY: if (is_bigint) break; @@ -53178,35 +56705,42 @@ static JSValue js_typed_array_join(JSContext *ctx, JSValueConst this_val, { JSValue sep = JS_UNDEFINED, el; StringBuffer b_s, *b = &b_s; - JSString *p = NULL; - int i, n; + JSString *s = NULL; + JSObject *p; + int i, len, oldlen, newlen; int c; - n = js_typed_array_get_length_internal(ctx, this_val); - if (n < 0) - goto exception; + p = get_typed_array(ctx, this_val); + if (!p) + return JS_EXCEPTION; + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + len = oldlen = newlen = p->u.array.count; c = ','; /* default separator */ if (!toLocaleString && argc > 0 && !JS_IsUndefined(argv[0])) { sep = JS_ToString(ctx, argv[0]); if (JS_IsException(sep)) goto exception; - p = JS_VALUE_GET_STRING(sep); - if (p->len == 1 && !p->is_wide_char) - c = p->u.str8[0]; + s = JS_VALUE_GET_STRING(sep); + if (s->len == 1 && !s->is_wide_char) + c = s->u.str8[0]; else c = -1; + // ToString(sep) can detach or resize the arraybuffer as a side effect + newlen = p->u.array.count; + len = min_int(len, newlen); } string_buffer_init(ctx, b, 0); /* XXX: optimize with direct access */ - for(i = 0; i < n; i++) { + for(i = 0; i < len; i++) { if (i > 0) { if (c >= 0) { if (string_buffer_putc8(b, c)) goto fail; } else { - if (string_buffer_concat(b, p, 0, p->len)) + if (string_buffer_concat(b, s, 0, s->len)) goto fail; } } @@ -53222,6 +56756,19 @@ static JSValue js_typed_array_join(JSContext *ctx, JSValueConst this_val, goto fail; } } + + // add extra separators in case RAB was resized by evil .valueOf method + i = max_int(1, newlen); + for(/*empty*/; i < oldlen; i++) { + if (c >= 0) { + if (string_buffer_putc8(b, c)) + goto fail; + } else { + if (string_buffer_concat(b, s, 0, s->len)) + goto fail; + } + } + JS_FreeValue(ctx, sep); return string_buffer_end(b); @@ -53238,7 +56785,7 @@ static JSValue js_typed_array_reverse(JSContext *ctx, JSValueConst this_val, JSObject *p; int len; - len = js_typed_array_get_length_internal(ctx, this_val); + len = js_typed_array_get_length_unsafe(ctx, this_val); if (len < 0) return JS_EXCEPTION; if (len > 0) { @@ -53301,11 +56848,11 @@ static JSValue js_typed_array_toReversed(JSContext *ctx, JSValueConst this_val, JSValue arr, ret; JSObject *p; - p = get_typed_array(ctx, this_val, /*is_dataview*/0); + p = get_typed_array(ctx, this_val); if (!p) return JS_EXCEPTION; arr = js_typed_array_constructor_ta(ctx, JS_UNDEFINED, this_val, - p->class_id); + p->class_id, p->u.array.count); if (JS_IsException(arr)) return JS_EXCEPTION; ret = js_typed_array_reverse(ctx, arr, argc, argv); @@ -53331,12 +56878,15 @@ static JSValue js_typed_array_slice(JSContext *ctx, JSValueConst this_val, JSValueConst args[2]; JSValue arr, val; JSObject *p, *p1; - int n, len, start, final, count, shift; + int n, len, start, final, count, shift, space; arr = JS_UNDEFINED; - len = js_typed_array_get_length_internal(ctx, this_val); - if (len < 0) + p = get_typed_array(ctx, this_val); + if (!p) goto exception; + if (typed_array_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + len = p->u.array.count; if (JS_ToInt32Clamp(ctx, &start, argv[0], 0, len, len)) goto exception; @@ -53347,9 +56897,6 @@ static JSValue js_typed_array_slice(JSContext *ctx, JSValueConst this_val, } count = max_int(final - start, 0); - p = get_typed_array(ctx, this_val, 0); - if (p == NULL) - goto exception; shift = typed_array_size_log2(p->class_id); args[0] = this_val; @@ -53363,10 +56910,10 @@ static JSValue js_typed_array_slice(JSContext *ctx, JSValueConst this_val, || validate_typed_array(ctx, arr)) goto exception; - p1 = get_typed_array(ctx, arr, 0); - if (p1 != NULL && p->class_id == p1->class_id && - typed_array_get_length(ctx, p1) >= count && - typed_array_get_length(ctx, p) >= start + count) { + p1 = get_typed_array(ctx, arr); + space = max_int(0, p->u.array.count - start); + count = min_int(count, space); + if (p1 != NULL && p->class_id == p1->class_id) { slice_memcpy(p1->u.array.u.uint8_ptr, p->u.array.u.uint8_ptr + (start << shift), count << shift); @@ -53392,37 +56939,41 @@ static JSValue js_typed_array_subarray(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { JSValueConst args[4]; - JSValue arr, byteOffset, ta_buffer; + JSValue arr, ta_buffer; + JSTypedArray *ta; JSObject *p; int len, start, final, count, shift, offset; - - p = get_typed_array(ctx, this_val, 0); + BOOL is_auto; + + p = get_typed_array(ctx, this_val); if (!p) goto exception; len = p->u.array.count; if (JS_ToInt32Clamp(ctx, &start, argv[0], 0, len, len)) goto exception; + shift = typed_array_size_log2(p->class_id); + ta = p->u.typed_array; + /* Read byteOffset (ta->offset) even if detached */ + offset = ta->offset + (start << shift); + final = len; - if (!JS_IsUndefined(argv[1])) { + if (JS_IsUndefined(argv[1])) { + is_auto = ta->track_rab; + } else { + is_auto = FALSE; if (JS_ToInt32Clamp(ctx, &final, argv[1], 0, len, len)) goto exception; - } + } count = max_int(final - start, 0); - byteOffset = js_typed_array_get_byteOffset(ctx, this_val, 0); - if (JS_IsException(byteOffset)) - goto exception; - shift = typed_array_size_log2(p->class_id); - offset = JS_VALUE_GET_INT(byteOffset) + (start << shift); - JS_FreeValue(ctx, byteOffset); - ta_buffer = js_typed_array_get_buffer(ctx, this_val, 0); + ta_buffer = js_typed_array_get_buffer(ctx, this_val); if (JS_IsException(ta_buffer)) goto exception; args[0] = this_val; args[1] = ta_buffer; args[2] = JS_NewInt32(ctx, offset); args[3] = JS_NewInt32(ctx, count); - arr = js_typed_array___speciesCreate(ctx, JS_UNDEFINED, 4, args); + arr = js_typed_array___speciesCreate(ctx, JS_UNDEFINED, is_auto ? 3 : 4, args); JS_FreeValue(ctx, ta_buffer); return arr; @@ -53483,6 +57034,11 @@ static int js_TA_cmp_uint64(const void *a, const void *b, void *opaque) { return (y < x) - (y > x); } +static int js_TA_cmp_float16(const void *a, const void *b, void *opaque) { + return js_cmp_doubles(fromfp16(*(const uint16_t *)a), + fromfp16(*(const uint16_t *)b)); +} + static int js_TA_cmp_float32(const void *a, const void *b, void *opaque) { return js_cmp_doubles(*(const float *)a, *(const float *)b); } @@ -53523,6 +57079,10 @@ static JSValue js_TA_get_uint64(JSContext *ctx, const void *a) { return JS_NewBigUint64(ctx, *(uint64_t *)a); } +static JSValue js_TA_get_float16(JSContext *ctx, const void *a) { + return __JS_NewFloat64(ctx, fromfp16(*(const uint16_t *)a)); +} + static JSValue js_TA_get_float32(JSContext *ctx, const void *a) { return __JS_NewFloat64(ctx, *(const float *)a); } @@ -53537,7 +57097,6 @@ struct TA_sort_context { JSValueConst arr; JSValueConst cmp; JSValue (*getfun)(JSContext *ctx, const void *a); - uint8_t *array_ptr; /* cannot change unless the array is detached */ int elt_size; }; @@ -53548,16 +57107,23 @@ static int js_TA_cmp_generic(const void *a, const void *b, void *opaque) { JSValueConst argv[2]; JSValue res; int cmp; - + JSObject *p; + cmp = 0; if (!psc->exception) { /* Note: the typed array can be detached without causing an error */ a_idx = *(uint32_t *)a; b_idx = *(uint32_t *)b; - argv[0] = psc->getfun(ctx, psc->array_ptr + + p = JS_VALUE_GET_PTR(psc->arr); + if (a_idx >= p->u.array.count || b_idx >= p->u.array.count) { + /* OOB case */ + psc->exception = 2; + return 0; + } + argv[0] = psc->getfun(ctx, p->u.array.u.uint8_ptr + a_idx * (size_t)psc->elt_size); - argv[1] = psc->getfun(ctx, psc->array_ptr + + argv[1] = psc->getfun(ctx, p->u.array.u.uint8_ptr + b_idx * (size_t)(psc->elt_size)); res = JS_Call(ctx, psc->cmp, JS_UNDEFINED, 2, argv); if (JS_IsException(res)) { @@ -53580,10 +57146,6 @@ static int js_TA_cmp_generic(const void *a, const void *b, void *opaque) { /* make sort stable: compare array offsets */ cmp = (a_idx > b_idx) - (a_idx < b_idx); } - if (unlikely(typed_array_is_detached(ctx, - JS_VALUE_GET_PTR(psc->arr)))) { - psc->exception = 2; - } done: JS_FreeValue(ctx, (JSValue)argv[0]); JS_FreeValue(ctx, (JSValue)argv[1]); @@ -53598,7 +57160,6 @@ static JSValue js_typed_array_sort(JSContext *ctx, JSValueConst this_val, int len; size_t elt_size; struct TA_sort_context tsc; - void *array_ptr; int (*cmpfun)(const void *a, const void *b, void *opaque); tsc.ctx = ctx; @@ -53608,7 +57169,7 @@ static JSValue js_typed_array_sort(JSContext *ctx, JSValueConst this_val, if (!JS_IsUndefined(tsc.cmp) && check_function(ctx, tsc.cmp)) return JS_EXCEPTION; - len = js_typed_array_get_length_internal(ctx, this_val); + len = js_typed_array_get_length_unsafe(ctx, this_val); if (len < 0) return JS_EXCEPTION; @@ -53648,6 +57209,10 @@ static JSValue js_typed_array_sort(JSContext *ctx, JSValueConst this_val, tsc.getfun = js_TA_get_uint64; cmpfun = js_TA_cmp_uint64; break; + case JS_CLASS_FLOAT16_ARRAY: + tsc.getfun = js_TA_get_float16; + cmpfun = js_TA_cmp_float16; + break; case JS_CLASS_FLOAT32_ARRAY: tsc.getfun = js_TA_get_float32; cmpfun = js_TA_cmp_float32; @@ -53659,7 +57224,6 @@ static JSValue js_typed_array_sort(JSContext *ctx, JSValueConst this_val, default: abort(); } - array_ptr = p->u.array.u.ptr; elt_size = 1 << typed_array_size_log2(p->class_id); if (!JS_IsUndefined(tsc.cmp)) { uint32_t *array_idx; @@ -53672,7 +57236,6 @@ static JSValue js_typed_array_sort(JSContext *ctx, JSValueConst this_val, return JS_EXCEPTION; for(i = 0; i < len; i++) array_idx[i] = i; - tsc.array_ptr = array_ptr; tsc.elt_size = elt_size; rqsort(array_idx, len, sizeof(array_idx[0]), js_TA_cmp_generic, &tsc); @@ -53681,46 +57244,50 @@ static JSValue js_typed_array_sort(JSContext *ctx, JSValueConst this_val, goto fail; /* detached typed array during the sort: no error */ } else { - array_tmp = js_malloc(ctx, len * elt_size); - if (!array_tmp) { - fail: - js_free(ctx, array_idx); - return JS_EXCEPTION; - } - memcpy(array_tmp, array_ptr, len * elt_size); - switch(elt_size) { - case 1: - for(i = 0; i < len; i++) { - j = array_idx[i]; - ((uint8_t *)array_ptr)[i] = ((uint8_t *)array_tmp)[j]; - } - break; - case 2: - for(i = 0; i < len; i++) { - j = array_idx[i]; - ((uint16_t *)array_ptr)[i] = ((uint16_t *)array_tmp)[j]; - } - break; - case 4: - for(i = 0; i < len; i++) { - j = array_idx[i]; - ((uint32_t *)array_ptr)[i] = ((uint32_t *)array_tmp)[j]; + void *array_ptr = p->u.array.u.ptr; + len = min_int(len, p->u.array.count); + if (len != 0) { + array_tmp = js_malloc(ctx, len * elt_size); + if (!array_tmp) { + fail: + js_free(ctx, array_idx); + return JS_EXCEPTION; } - break; - case 8: - for(i = 0; i < len; i++) { - j = array_idx[i]; - ((uint64_t *)array_ptr)[i] = ((uint64_t *)array_tmp)[j]; + memcpy(array_tmp, array_ptr, len * elt_size); + switch(elt_size) { + case 1: + for(i = 0; i < len; i++) { + j = array_idx[i]; + ((uint8_t *)array_ptr)[i] = ((uint8_t *)array_tmp)[j]; + } + break; + case 2: + for(i = 0; i < len; i++) { + j = array_idx[i]; + ((uint16_t *)array_ptr)[i] = ((uint16_t *)array_tmp)[j]; + } + break; + case 4: + for(i = 0; i < len; i++) { + j = array_idx[i]; + ((uint32_t *)array_ptr)[i] = ((uint32_t *)array_tmp)[j]; + } + break; + case 8: + for(i = 0; i < len; i++) { + j = array_idx[i]; + ((uint64_t *)array_ptr)[i] = ((uint64_t *)array_tmp)[j]; + } + break; + default: + abort(); } - break; - default: - abort(); + js_free(ctx, array_tmp); } - js_free(ctx, array_tmp); } js_free(ctx, array_idx); } else { - rqsort(array_ptr, len, elt_size, cmpfun, &tsc); + rqsort(p->u.array.u.ptr, len, elt_size, cmpfun, &tsc); if (tsc.exception) return JS_EXCEPTION; } @@ -53734,11 +57301,11 @@ static JSValue js_typed_array_toSorted(JSContext *ctx, JSValueConst this_val, JSValue arr, ret; JSObject *p; - p = get_typed_array(ctx, this_val, /*is_dataview*/0); + p = get_typed_array(ctx, this_val); if (!p) return JS_EXCEPTION; arr = js_typed_array_constructor_ta(ctx, JS_UNDEFINED, this_val, - p->class_id); + p->class_id, p->u.array.count); if (JS_IsException(arr)) return JS_EXCEPTION; ret = js_typed_array_sort(ctx, arr, argc, argv); @@ -53750,18 +57317,15 @@ static const JSCFunctionListEntry js_typed_array_base_funcs[] = { JS_CFUNC_DEF("from", 1, js_typed_array_from ), JS_CFUNC_DEF("of", 0, js_typed_array_of ), JS_CGETSET_DEF("[Symbol.species]", js_get_this, NULL ), - //JS_CFUNC_DEF("__getLength", 2, js_typed_array___getLength ), - //JS_CFUNC_DEF("__create", 2, js_typed_array___create ), - //JS_CFUNC_DEF("__speciesCreate", 2, js_typed_array___speciesCreate ), }; static const JSCFunctionListEntry js_typed_array_base_proto_funcs[] = { JS_CGETSET_DEF("length", js_typed_array_get_length, NULL ), JS_CFUNC_DEF("at", 1, js_typed_array_at ), JS_CFUNC_DEF("with", 2, js_typed_array_with ), - JS_CGETSET_MAGIC_DEF("buffer", js_typed_array_get_buffer, NULL, 0 ), - JS_CGETSET_MAGIC_DEF("byteLength", js_typed_array_get_byteLength, NULL, 0 ), - JS_CGETSET_MAGIC_DEF("byteOffset", js_typed_array_get_byteOffset, NULL, 0 ), + JS_CGETSET_DEF("buffer", js_typed_array_get_buffer, NULL ), + JS_CGETSET_DEF("byteLength", js_typed_array_get_byteLength, NULL ), + JS_CGETSET_DEF("byteOffset", js_typed_array_get_byteOffset, NULL ), JS_CFUNC_DEF("set", 1, js_typed_array_set ), JS_CFUNC_MAGIC_DEF("values", 0, js_create_typed_array_iterator, JS_ITERATOR_KIND_VALUE ), JS_ALIAS_DEF("[Symbol.iterator]", "values" ), @@ -53795,6 +57359,13 @@ static const JSCFunctionListEntry js_typed_array_base_proto_funcs[] = { //JS_ALIAS_BASE_DEF("toString", "toString", 2 /* Array.prototype. */), @@@ }; +static const JSCFunctionListEntry js_typed_array_funcs[] = { + JS_PROP_INT32_DEF("BYTES_PER_ELEMENT", 1, 0), + JS_PROP_INT32_DEF("BYTES_PER_ELEMENT", 2, 0), + JS_PROP_INT32_DEF("BYTES_PER_ELEMENT", 4, 0), + JS_PROP_INT32_DEF("BYTES_PER_ELEMENT", 8, 0), +}; + static JSValue js_typed_array_base_constructor(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) @@ -53804,7 +57375,8 @@ static JSValue js_typed_array_base_constructor(JSContext *ctx, /* 'obj' must be an allocated typed array object */ static int typed_array_init(JSContext *ctx, JSValueConst obj, - JSValue buffer, uint64_t offset, uint64_t len) + JSValue buffer, uint64_t offset, uint64_t len, + BOOL track_rab) { JSTypedArray *ta; JSObject *p, *pbuffer; @@ -53824,6 +57396,7 @@ static int typed_array_init(JSContext *ctx, JSValueConst obj, ta->buffer = pbuffer; ta->offset = offset; ta->length = len << size_log2; + ta->track_rab = track_rab; list_add_tail(&ta->link, &abuf->array_list); p->u.typed_array = ta; p->u.array.count = len; @@ -53903,10 +57476,11 @@ static JSValue js_typed_array_constructor_obj(JSContext *ctx, } buffer = js_array_buffer_constructor1(ctx, JS_UNDEFINED, - len << size_log2); + len << size_log2, + NULL); if (JS_IsException(buffer)) goto fail; - if (typed_array_init(ctx, ret, buffer, 0, len)) + if (typed_array_init(ctx, ret, buffer, 0, len, /*track_rab*/FALSE)) goto fail; for(i = 0; i < len; i++) { @@ -53927,12 +57501,12 @@ static JSValue js_typed_array_constructor_obj(JSContext *ctx, static JSValue js_typed_array_constructor_ta(JSContext *ctx, JSValueConst new_target, JSValueConst src_obj, - int classid) + int classid, uint32_t len) { JSObject *p, *src_buffer; JSTypedArray *ta; JSValue obj, buffer; - uint32_t len, i; + uint32_t i; int size_log2; JSArrayBuffer *src_abuf, *abuf; @@ -53940,27 +57514,27 @@ static JSValue js_typed_array_constructor_ta(JSContext *ctx, if (JS_IsException(obj)) return obj; p = JS_VALUE_GET_OBJ(src_obj); - if (typed_array_is_detached(ctx, p)) { - JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + if (typed_array_is_oob(p)) { + JS_ThrowTypeErrorArrayBufferOOB(ctx); goto fail; } ta = p->u.typed_array; - len = p->u.array.count; src_buffer = ta->buffer; src_abuf = src_buffer->u.array_buffer; size_log2 = typed_array_size_log2(classid); buffer = js_array_buffer_constructor1(ctx, JS_UNDEFINED, - (uint64_t)len << size_log2); + (uint64_t)len << size_log2, + NULL); if (JS_IsException(buffer)) goto fail; /* necessary because it could have been detached */ - if (typed_array_is_detached(ctx, p)) { + if (typed_array_is_oob(p)) { JS_FreeValue(ctx, buffer); - JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + JS_ThrowTypeErrorArrayBufferOOB(ctx); goto fail; } abuf = JS_GetOpaque(buffer, JS_CLASS_ARRAY_BUFFER); - if (typed_array_init(ctx, obj, buffer, 0, len)) + if (typed_array_init(ctx, obj, buffer, 0, len, /*track_rab*/FALSE)) goto fail; if (p->class_id == classid) { /* same type: copy the content */ @@ -53986,6 +57560,7 @@ static JSValue js_typed_array_constructor(JSContext *ctx, int argc, JSValueConst *argv, int classid) { + BOOL track_rab = FALSE; JSValue buffer, obj; JSArrayBuffer *abuf; int size_log2; @@ -53996,7 +57571,8 @@ static JSValue js_typed_array_constructor(JSContext *ctx, if (JS_ToIndex(ctx, &len, argv[0])) return JS_EXCEPTION; buffer = js_array_buffer_constructor1(ctx, JS_UNDEFINED, - len << size_log2); + len << size_log2, + NULL); if (JS_IsException(buffer)) return JS_EXCEPTION; offset = 0; @@ -54013,8 +57589,10 @@ static JSValue js_typed_array_constructor(JSContext *ctx, offset > abuf->byte_length) return JS_ThrowRangeError(ctx, "invalid offset"); if (JS_IsUndefined(argv[2])) { - if ((abuf->byte_length & ((1 << size_log2) - 1)) != 0) - goto invalid_length; + track_rab = array_buffer_is_resizable(abuf); + if (!track_rab) + if ((abuf->byte_length & ((1 << size_log2) - 1)) != 0) + goto invalid_length; len = (abuf->byte_length - offset) >> size_log2; } else { if (JS_ToIndex(ctx, &len, argv[2])) @@ -54030,7 +57608,8 @@ static JSValue js_typed_array_constructor(JSContext *ctx, } else { if (p->class_id >= JS_CLASS_UINT8C_ARRAY && p->class_id <= JS_CLASS_FLOAT64_ARRAY) { - return js_typed_array_constructor_ta(ctx, new_target, argv[0], classid); + return js_typed_array_constructor_ta(ctx, new_target, argv[0], + classid, p->u.array.count); } else { return js_typed_array_constructor_obj(ctx, new_target, argv[0], classid); } @@ -54042,7 +57621,7 @@ static JSValue js_typed_array_constructor(JSContext *ctx, JS_FreeValue(ctx, buffer); return JS_EXCEPTION; } - if (typed_array_init(ctx, obj, buffer, offset, len)) { + if (typed_array_init(ctx, obj, buffer, offset, len, track_rab)) { JS_FreeValue(ctx, obj); return JS_EXCEPTION; } @@ -54078,6 +57657,8 @@ static JSValue js_dataview_constructor(JSContext *ctx, JSValueConst new_target, int argc, JSValueConst *argv) { + BOOL recompute_len = FALSE; + BOOL track_rab = FALSE; JSArrayBuffer *abuf; uint64_t offset; uint32_t len; @@ -54107,6 +57688,9 @@ static JSValue js_dataview_constructor(JSContext *ctx, if (l > len) return JS_ThrowRangeError(ctx, "invalid byteLength"); len = l; + } else { + recompute_len = TRUE; + track_rab = array_buffer_is_resizable(abuf); } obj = js_create_from_ctor(ctx, new_target, JS_CLASS_DATAVIEW); @@ -54117,6 +57701,16 @@ static JSValue js_dataview_constructor(JSContext *ctx, JS_ThrowTypeErrorDetachedArrayBuffer(ctx); goto fail; } + // RAB could have been resized in js_create_from_ctor() + if (offset > abuf->byte_length) { + goto out_of_bound; + } else if (recompute_len) { + len = abuf->byte_length - offset; + } else if (offset + len > abuf->byte_length) { + out_of_bound: + JS_ThrowRangeError(ctx, "invalid byteOffset or byteLength"); + goto fail; + } ta = js_malloc(ctx, sizeof(*ta)); if (!ta) { fail: @@ -54128,11 +57722,88 @@ static JSValue js_dataview_constructor(JSContext *ctx, ta->buffer = JS_VALUE_GET_OBJ(JS_DupValue(ctx, buffer)); ta->offset = offset; ta->length = len; + ta->track_rab = track_rab; list_add_tail(&ta->link, &abuf->array_list); p->u.typed_array = ta; return obj; } +// is the DataView out of bounds relative to its parent arraybuffer? +static BOOL dataview_is_oob(JSObject *p) +{ + JSArrayBuffer *abuf; + JSTypedArray *ta; + + assert(p->class_id == JS_CLASS_DATAVIEW); + ta = p->u.typed_array; + abuf = ta->buffer->u.array_buffer; + if (abuf->detached) + return TRUE; + if (ta->offset > abuf->byte_length) + return TRUE; + if (ta->track_rab) + return FALSE; + return (int64_t)ta->offset + ta->length > abuf->byte_length; +} + +static JSObject *get_dataview(JSContext *ctx, JSValueConst this_val) +{ + JSObject *p; + if (JS_VALUE_GET_TAG(this_val) != JS_TAG_OBJECT) + goto fail; + p = JS_VALUE_GET_OBJ(this_val); + if (p->class_id != JS_CLASS_DATAVIEW) { + fail: + JS_ThrowTypeError(ctx, "not a DataView"); + return NULL; + } + return p; +} + +static JSValue js_dataview_get_buffer(JSContext *ctx, JSValueConst this_val) +{ + JSObject *p; + JSTypedArray *ta; + p = get_dataview(ctx, this_val); + if (!p) + return JS_EXCEPTION; + ta = p->u.typed_array; + return JS_DupValue(ctx, JS_MKPTR(JS_TAG_OBJECT, ta->buffer)); +} + +static JSValue js_dataview_get_byteLength(JSContext *ctx, JSValueConst this_val) +{ + JSArrayBuffer *abuf; + JSTypedArray *ta; + JSObject *p; + + p = get_dataview(ctx, this_val); + if (!p) + return JS_EXCEPTION; + if (dataview_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + ta = p->u.typed_array; + if (ta->track_rab) { + abuf = ta->buffer->u.array_buffer; + return JS_NewUint32(ctx, abuf->byte_length - ta->offset); + } + return JS_NewUint32(ctx, ta->length); +} + +static JSValue js_dataview_get_byteOffset(JSContext *ctx, JSValueConst this_val) +{ + JSTypedArray *ta; + JSObject *p; + + p = get_dataview(ctx, this_val); + if (!p) + return JS_EXCEPTION; + if (dataview_is_oob(p)) + return JS_ThrowTypeErrorArrayBufferOOB(ctx); + ta = p->u.typed_array; + return JS_NewUint32(ctx, ta->offset); +} + static JSValue js_dataview_getValue(JSContext *ctx, JSValueConst this_obj, int argc, JSValueConst *argv, int class_id) @@ -54156,8 +57827,14 @@ static JSValue js_dataview_getValue(JSContext *ctx, abuf = ta->buffer->u.array_buffer; if (abuf->detached) return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + // order matters: this check should come before the next one if ((pos + size) > ta->length) return JS_ThrowRangeError(ctx, "out of bound"); + // test262 expects a TypeError for this and V8, in its infinite wisdom, + // throws a "detached array buffer" exception, but IMO that doesn't make + // sense because the buffer is not in fact detached, it's still there + if ((int64_t)ta->offset + ta->length > abuf->byte_length) + return JS_ThrowTypeError(ctx, "out of bound"); ptr = abuf->data + ta->offset + pos; switch(class_id) { @@ -54203,6 +57880,14 @@ static JSValue js_dataview_getValue(JSContext *ctx, return JS_NewBigUint64(ctx, v); } break; + case JS_CLASS_FLOAT16_ARRAY: + { + uint16_t v; + v = get_u16(ptr); + if (is_swap) + v = bswap16(v); + return __JS_NewFloat64(ctx, fromfp16(v)); + } case JS_CLASS_FLOAT32_ARRAY: { union { @@ -54264,7 +57949,9 @@ static JSValue js_dataview_setValue(JSContext *ctx, double d; if (JS_ToFloat64(ctx, &d, val)) return JS_EXCEPTION; - if (class_id == JS_CLASS_FLOAT32_ARRAY) { + if (class_id == JS_CLASS_FLOAT16_ARRAY) { + v = tofp16(d); + } else if (class_id == JS_CLASS_FLOAT32_ARRAY) { union { float f; uint32_t i; @@ -54282,8 +57969,14 @@ static JSValue js_dataview_setValue(JSContext *ctx, abuf = ta->buffer->u.array_buffer; if (abuf->detached) return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); + // order matters: this check should come before the next one if ((pos + size) > ta->length) return JS_ThrowRangeError(ctx, "out of bound"); + // test262 expects a TypeError for this and V8, in its infinite wisdom, + // throws a "detached array buffer" exception, but IMO that doesn't make + // sense because the buffer is not in fact detached, it's still there + if ((int64_t)ta->offset + ta->length > abuf->byte_length) + return JS_ThrowTypeError(ctx, "out of bound"); ptr = abuf->data + ta->offset + pos; switch(class_id) { @@ -54293,6 +57986,7 @@ static JSValue js_dataview_setValue(JSContext *ctx, break; case JS_CLASS_INT16_ARRAY: case JS_CLASS_UINT16_ARRAY: + case JS_CLASS_FLOAT16_ARRAY: if (is_swap) v = bswap16(v); put_u16(ptr, v); @@ -54318,9 +58012,9 @@ static JSValue js_dataview_setValue(JSContext *ctx, } static const JSCFunctionListEntry js_dataview_proto_funcs[] = { - JS_CGETSET_MAGIC_DEF("buffer", js_typed_array_get_buffer, NULL, 1 ), - JS_CGETSET_MAGIC_DEF("byteLength", js_typed_array_get_byteLength, NULL, 1 ), - JS_CGETSET_MAGIC_DEF("byteOffset", js_typed_array_get_byteOffset, NULL, 1 ), + JS_CGETSET_DEF("buffer", js_dataview_get_buffer, NULL ), + JS_CGETSET_DEF("byteLength", js_dataview_get_byteLength, NULL ), + JS_CGETSET_DEF("byteOffset", js_dataview_get_byteOffset, NULL ), JS_CFUNC_MAGIC_DEF("getInt8", 1, js_dataview_getValue, JS_CLASS_INT8_ARRAY ), JS_CFUNC_MAGIC_DEF("getUint8", 1, js_dataview_getValue, JS_CLASS_UINT8_ARRAY ), JS_CFUNC_MAGIC_DEF("getInt16", 1, js_dataview_getValue, JS_CLASS_INT16_ARRAY ), @@ -54329,6 +58023,7 @@ static const JSCFunctionListEntry js_dataview_proto_funcs[] = { JS_CFUNC_MAGIC_DEF("getUint32", 1, js_dataview_getValue, JS_CLASS_UINT32_ARRAY ), JS_CFUNC_MAGIC_DEF("getBigInt64", 1, js_dataview_getValue, JS_CLASS_BIG_INT64_ARRAY ), JS_CFUNC_MAGIC_DEF("getBigUint64", 1, js_dataview_getValue, JS_CLASS_BIG_UINT64_ARRAY ), + JS_CFUNC_MAGIC_DEF("getFloat16", 1, js_dataview_getValue, JS_CLASS_FLOAT16_ARRAY ), JS_CFUNC_MAGIC_DEF("getFloat32", 1, js_dataview_getValue, JS_CLASS_FLOAT32_ARRAY ), JS_CFUNC_MAGIC_DEF("getFloat64", 1, js_dataview_getValue, JS_CLASS_FLOAT64_ARRAY ), JS_CFUNC_MAGIC_DEF("setInt8", 2, js_dataview_setValue, JS_CLASS_INT8_ARRAY ), @@ -54339,6 +58034,7 @@ static const JSCFunctionListEntry js_dataview_proto_funcs[] = { JS_CFUNC_MAGIC_DEF("setUint32", 2, js_dataview_setValue, JS_CLASS_UINT32_ARRAY ), JS_CFUNC_MAGIC_DEF("setBigInt64", 2, js_dataview_setValue, JS_CLASS_BIG_INT64_ARRAY ), JS_CFUNC_MAGIC_DEF("setBigUint64", 2, js_dataview_setValue, JS_CLASS_BIG_UINT64_ARRAY ), + JS_CFUNC_MAGIC_DEF("setFloat16", 2, js_dataview_setValue, JS_CLASS_FLOAT16_ARRAY ), JS_CFUNC_MAGIC_DEF("setFloat32", 2, js_dataview_setValue, JS_CLASS_FLOAT32_ARRAY ), JS_CFUNC_MAGIC_DEF("setFloat64", 2, js_dataview_setValue, JS_CLASS_FLOAT64_ARRAY ), JS_PROP_STRING_DEF("[Symbol.toStringTag]", "DataView", JS_PROP_CONFIGURABLE ), @@ -54358,11 +58054,12 @@ typedef enum AtomicsOpEnum { ATOMICS_OP_LOAD, } AtomicsOpEnum; -static void *js_atomics_get_ptr(JSContext *ctx, - JSArrayBuffer **pabuf, - int *psize_log2, JSClassID *pclass_id, - JSValueConst obj, JSValueConst idx_val, - int is_waitable) +static int js_atomics_get_ptr(JSContext *ctx, + void **pptr, + JSArrayBuffer **pabuf, + int *psize_log2, JSClassID *pclass_id, + JSValueConst obj, JSValueConst idx_val, + int is_waitable) { JSObject *p; JSTypedArray *ta; @@ -54370,7 +58067,7 @@ static void *js_atomics_get_ptr(JSContext *ctx, void *ptr; uint64_t idx; BOOL err; - int size_log2; + int size_log2, old_len; if (JS_VALUE_GET_TAG(obj) != JS_TAG_OBJECT) goto fail; @@ -54384,33 +58081,46 @@ static void *js_atomics_get_ptr(JSContext *ctx, if (err) { fail: JS_ThrowTypeError(ctx, "integer TypedArray expected"); - return NULL; + return -1; } ta = p->u.typed_array; abuf = ta->buffer->u.array_buffer; if (!abuf->shared) { if (is_waitable == 2) { JS_ThrowTypeError(ctx, "not a SharedArrayBuffer TypedArray"); - return NULL; + return -1; } if (abuf->detached) { JS_ThrowTypeErrorDetachedArrayBuffer(ctx); - return NULL; + return -1; } } + old_len = p->u.array.count; + if (JS_ToIndex(ctx, &idx, idx_val)) { - return NULL; - } - /* RevalidateAtomicAccess(): must test again detached after JS_ToIndex() */ - if (abuf->detached) { - JS_ThrowTypeErrorDetachedArrayBuffer(ctx); - return NULL; + return -1; } - /* if the array buffer is detached, p->u.array.count = 0 */ - if (idx >= p->u.array.count) { - JS_ThrowRangeError(ctx, "out-of-bound access"); - return NULL; + + if (idx >= old_len) + goto oob; + + if (is_waitable == 1) { + /* notify(): just avoid having an invalid pointer if overflow */ + if (idx >= p->u.array.count) + ptr = NULL; + } else { + /* RevalidateAtomicAccess() */ + if (typed_array_is_oob(p)) { + JS_ThrowTypeErrorArrayBufferOOB(ctx); + return -1; + } + if (idx >= p->u.array.count) { + oob: + JS_ThrowRangeError(ctx, "out-of-bound access"); + return -1; + } } + size_log2 = typed_array_size_log2(p->class_id); ptr = p->u.array.u.uint8_ptr + ((uintptr_t)idx << size_log2); if (pabuf) @@ -54419,7 +58129,8 @@ static void *js_atomics_get_ptr(JSContext *ctx, *psize_log2 = size_log2; if (pclass_id) *pclass_id = p->class_id; - return ptr; + *pptr = ptr; + return 0; } static JSValue js_atomics_op(JSContext *ctx, @@ -54433,9 +58144,8 @@ static JSValue js_atomics_op(JSContext *ctx, JSClassID class_id; JSArrayBuffer *abuf; - ptr = js_atomics_get_ptr(ctx, &abuf, &size_log2, &class_id, - argv[0], argv[1], 0); - if (!ptr) + if (js_atomics_get_ptr(ctx, &ptr, &abuf, &size_log2, &class_id, + argv[0], argv[1], 0)) return JS_EXCEPTION; rep_val = 0; if (op == ATOMICS_OP_LOAD) { @@ -54576,9 +58286,8 @@ static JSValue js_atomics_store(JSContext *ctx, JSValue ret; JSArrayBuffer *abuf; - ptr = js_atomics_get_ptr(ctx, &abuf, &size_log2, NULL, - argv[0], argv[1], 0); - if (!ptr) + if (js_atomics_get_ptr(ctx, &ptr, &abuf, &size_log2, NULL, + argv[0], argv[1], 0)) return JS_EXCEPTION; if (size_log2 == 3) { int64_t v64; @@ -54643,6 +58352,49 @@ static pthread_mutex_t js_atomics_mutex = PTHREAD_MUTEX_INITIALIZER; static struct list_head js_atomics_waiter_list = LIST_HEAD_INIT(js_atomics_waiter_list); +#if defined(__aarch64__) +static inline void cpu_pause(void) +{ + asm volatile("yield" ::: "memory"); +} +#elif defined(__x86_64) || defined(__i386__) +static inline void cpu_pause(void) +{ + asm volatile("pause" ::: "memory"); +} +#else +static inline void cpu_pause(void) +{ +} +#endif + +// no-op: Atomics.pause() is not allowed to block or yield to another +// thread, only to hint the CPU that it should back off for a bit; +// the amount of work we do here is a good enough substitute +static JSValue js_atomics_pause(JSContext *ctx, JSValueConst this_obj, + int argc, JSValueConst *argv) +{ + double d; + + if (argc > 0) { + switch (JS_VALUE_GET_NORM_TAG(argv[0])) { + case JS_TAG_FLOAT64: // accepted if and only if fraction == 0.0 + d = JS_VALUE_GET_FLOAT64(argv[0]); + if (isfinite(d)) + if (0 == modf(d, &d)) + break; + // fallthru + default: + return JS_ThrowTypeError(ctx, "not an integral number"); + case JS_TAG_UNDEFINED: + case JS_TAG_INT: + break; + } + } + cpu_pause(); + return JS_UNDEFINED; +} + static JSValue js_atomics_wait(JSContext *ctx, JSValueConst this_obj, int argc, JSValueConst *argv) @@ -54656,9 +58408,8 @@ static JSValue js_atomics_wait(JSContext *ctx, int ret, size_log2, res; double d; - ptr = js_atomics_get_ptr(ctx, NULL, &size_log2, NULL, - argv[0], argv[1], 2); - if (!ptr) + if (js_atomics_get_ptr(ctx, &ptr, NULL, &size_log2, NULL, + argv[0], argv[1], 2)) return JS_EXCEPTION; if (size_log2 == 3) { if (JS_ToBigInt64(ctx, &v, argv[2])) @@ -54736,18 +58487,15 @@ static JSValue js_atomics_notify(JSContext *ctx, JSAtomicsWaiter *waiter; JSArrayBuffer *abuf; - ptr = js_atomics_get_ptr(ctx, &abuf, NULL, NULL, argv[0], argv[1], 1); - if (!ptr) + if (js_atomics_get_ptr(ctx, &ptr, &abuf, NULL, NULL, argv[0], argv[1], 1)) return JS_EXCEPTION; - + if (JS_IsUndefined(argv[2])) { count = INT32_MAX; } else { if (JS_ToInt32Clamp(ctx, &count, argv[2], 0, INT32_MAX, 0)) return JS_EXCEPTION; } - if (abuf->detached) - return JS_ThrowTypeErrorDetachedArrayBuffer(ctx); n = 0; if (abuf->shared && count > 0) { @@ -54784,6 +58532,7 @@ static const JSCFunctionListEntry js_atomics_funcs[] = { JS_CFUNC_MAGIC_DEF("load", 2, js_atomics_op, ATOMICS_OP_LOAD ), JS_CFUNC_DEF("store", 3, js_atomics_store ), JS_CFUNC_DEF("isLockFree", 1, js_atomics_isLockFree ), + JS_CFUNC_DEF("pause", 0, js_atomics_pause ), JS_CFUNC_DEF("wait", 4, js_atomics_wait ), JS_CFUNC_DEF("notify", 3, js_atomics_notify ), JS_PROP_STRING_DEF("[Symbol.toStringTag]", "Atomics", JS_PROP_CONFIGURABLE ), @@ -54793,100 +58542,106 @@ static const JSCFunctionListEntry js_atomics_obj[] = { JS_OBJECT_DEF("Atomics", js_atomics_funcs, countof(js_atomics_funcs), JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE ), }; -void JS_AddIntrinsicAtomics(JSContext *ctx) +static int JS_AddIntrinsicAtomics(JSContext *ctx) { /* add Atomics as autoinit object */ - JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_atomics_obj, countof(js_atomics_obj)); + return JS_SetPropertyFunctionList(ctx, ctx->global_obj, js_atomics_obj, countof(js_atomics_obj)); } #endif /* CONFIG_ATOMICS */ -void JS_AddIntrinsicTypedArrays(JSContext *ctx) +int JS_AddIntrinsicTypedArrays(JSContext *ctx) { - JSValue typed_array_base_proto, typed_array_base_func; - JSValueConst array_buffer_func, shared_array_buffer_func; - int i; + JSValue typed_array_base_func, typed_array_base_proto, obj; + int i, ret; - ctx->class_proto[JS_CLASS_ARRAY_BUFFER] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_ARRAY_BUFFER], - js_array_buffer_proto_funcs, - countof(js_array_buffer_proto_funcs)); - - array_buffer_func = JS_NewGlobalCConstructorOnly(ctx, "ArrayBuffer", - js_array_buffer_constructor, 1, - ctx->class_proto[JS_CLASS_ARRAY_BUFFER]); - JS_SetPropertyFunctionList(ctx, array_buffer_func, - js_array_buffer_funcs, - countof(js_array_buffer_funcs)); - - ctx->class_proto[JS_CLASS_SHARED_ARRAY_BUFFER] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_SHARED_ARRAY_BUFFER], - js_shared_array_buffer_proto_funcs, - countof(js_shared_array_buffer_proto_funcs)); - - shared_array_buffer_func = JS_NewGlobalCConstructorOnly(ctx, "SharedArrayBuffer", - js_shared_array_buffer_constructor, 1, - ctx->class_proto[JS_CLASS_SHARED_ARRAY_BUFFER]); - JS_SetPropertyFunctionList(ctx, shared_array_buffer_func, - js_shared_array_buffer_funcs, - countof(js_shared_array_buffer_funcs)); - - typed_array_base_proto = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, typed_array_base_proto, - js_typed_array_base_proto_funcs, - countof(js_typed_array_base_proto_funcs)); + obj = JS_NewCConstructor(ctx, JS_CLASS_ARRAY_BUFFER, "ArrayBuffer", + js_array_buffer_constructor, 1, JS_CFUNC_constructor, 0, + JS_UNDEFINED, + js_array_buffer_funcs, countof(js_array_buffer_funcs), + js_array_buffer_proto_funcs, countof(js_array_buffer_proto_funcs), + 0); + if (JS_IsException(obj)) + return -1; + JS_FreeValue(ctx, obj); + + obj = JS_NewCConstructor(ctx, JS_CLASS_SHARED_ARRAY_BUFFER, "SharedArrayBuffer", + js_shared_array_buffer_constructor, 1, JS_CFUNC_constructor, 0, + JS_UNDEFINED, + js_shared_array_buffer_funcs, countof(js_shared_array_buffer_funcs), + js_shared_array_buffer_proto_funcs, countof(js_shared_array_buffer_proto_funcs), + 0); + if (JS_IsException(obj)) + return -1; + JS_FreeValue(ctx, obj); - /* TypedArray.prototype.toString must be the same object as Array.prototype.toString */ - JSValue obj = JS_GetProperty(ctx, ctx->class_proto[JS_CLASS_ARRAY], JS_ATOM_toString); - /* XXX: should use alias method in JSCFunctionListEntry */ //@@@ - JS_DefinePropertyValue(ctx, typed_array_base_proto, JS_ATOM_toString, obj, - JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); - typed_array_base_func = JS_NewCFunction2(ctx, js_typed_array_base_constructor, - "TypedArray", 0, JS_CFUNC_constructor_or_func, 0); - JS_SetPropertyFunctionList(ctx, typed_array_base_func, - js_typed_array_base_funcs, - countof(js_typed_array_base_funcs)); - JS_SetConstructor(ctx, typed_array_base_func, typed_array_base_proto); + typed_array_base_func = + JS_NewCConstructor(ctx, -1, "TypedArray", + js_typed_array_base_constructor, 0, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + js_typed_array_base_funcs, countof(js_typed_array_base_funcs), + js_typed_array_base_proto_funcs, countof(js_typed_array_base_proto_funcs), + JS_NEW_CTOR_NO_GLOBAL); + if (JS_IsException(typed_array_base_func)) + return -1; + /* TypedArray.prototype.toString must be the same object as Array.prototype.toString */ + obj = JS_GetProperty(ctx, ctx->class_proto[JS_CLASS_ARRAY], JS_ATOM_toString); + if (JS_IsException(obj)) + goto fail; + /* XXX: should use alias method in JSCFunctionListEntry */ //@@@ + typed_array_base_proto = JS_GetProperty(ctx, typed_array_base_func, JS_ATOM_prototype); + if (JS_IsException(typed_array_base_proto)) + goto fail; + ret = JS_DefinePropertyValue(ctx, typed_array_base_proto, JS_ATOM_toString, obj, + JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE); + JS_FreeValue(ctx, typed_array_base_proto); + if (ret < 0) + goto fail; + /* Used to squelch a -Wcast-function-type warning. */ JSCFunctionType ft = { .generic_magic = js_typed_array_constructor }; for(i = JS_CLASS_UINT8C_ARRAY; i < JS_CLASS_UINT8C_ARRAY + JS_TYPED_ARRAY_COUNT; i++) { - JSValue func_obj; char buf[ATOM_GET_STR_BUF_SIZE]; const char *name; - - ctx->class_proto[i] = JS_NewObjectProto(ctx, typed_array_base_proto); - JS_DefinePropertyValueStr(ctx, ctx->class_proto[i], - "BYTES_PER_ELEMENT", - JS_NewInt32(ctx, 1 << typed_array_size_log2(i)), - 0); + const JSCFunctionListEntry *bpe; + name = JS_AtomGetStr(ctx, buf, sizeof(buf), JS_ATOM_Uint8ClampedArray + i - JS_CLASS_UINT8C_ARRAY); - func_obj = JS_NewCFunction3(ctx, ft.generic, - name, 3, JS_CFUNC_constructor_magic, i, - typed_array_base_func); - JS_NewGlobalCConstructor2(ctx, func_obj, name, ctx->class_proto[i]); - JS_DefinePropertyValueStr(ctx, func_obj, - "BYTES_PER_ELEMENT", - JS_NewInt32(ctx, 1 << typed_array_size_log2(i)), - 0); + bpe = js_typed_array_funcs + typed_array_size_log2(i); + obj = JS_NewCConstructor(ctx, i, name, + ft.generic, 3, JS_CFUNC_constructor_magic, i, + typed_array_base_func, + bpe, 1, + bpe, 1, + 0); + if (JS_IsException(obj)) { + fail: + JS_FreeValue(ctx, typed_array_base_func); + return -1; + } + JS_FreeValue(ctx, obj); } - JS_FreeValue(ctx, typed_array_base_proto); JS_FreeValue(ctx, typed_array_base_func); /* DataView */ - ctx->class_proto[JS_CLASS_DATAVIEW] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_DATAVIEW], - js_dataview_proto_funcs, - countof(js_dataview_proto_funcs)); - JS_NewGlobalCConstructorOnly(ctx, "DataView", - js_dataview_constructor, 1, - ctx->class_proto[JS_CLASS_DATAVIEW]); + obj = JS_NewCConstructor(ctx, JS_CLASS_DATAVIEW, "DataView", + js_dataview_constructor, 1, JS_CFUNC_constructor, 0, + JS_UNDEFINED, + NULL, 0, + js_dataview_proto_funcs, countof(js_dataview_proto_funcs), + 0); + if (JS_IsException(obj)) + return -1; + JS_FreeValue(ctx, obj); + /* Atomics */ #ifdef CONFIG_ATOMICS - JS_AddIntrinsicAtomics(ctx); + if (JS_AddIntrinsicAtomics(ctx)) + return -1; #endif + return 0; } /* WeakRef */ @@ -54972,7 +58727,7 @@ typedef struct JSFinRecEntry { typedef struct JSFinalizationRegistryData { JSWeakRefHeader weakref_header; struct list_head entries; /* list of JSFinRecEntry.link */ - JSContext *ctx; + JSContext *realm; JSValue cb; } JSFinalizationRegistryData; @@ -54989,6 +58744,7 @@ static void js_finrec_finalizer(JSRuntime *rt, JSValue val) js_free_rt(rt, fre); } JS_FreeValueRT(rt, frd->cb); + JS_FreeContext(frd->realm); list_del(&frd->weakref_header.link); js_free_rt(rt, frd); } @@ -55005,6 +58761,7 @@ static void js_finrec_mark(JSRuntime *rt, JSValueConst val, JS_MarkValue(rt, fre->held_val, mark_func); } JS_MarkValue(rt, frd->cb, mark_func); + mark_func(rt, &frd->realm->header); } } @@ -55030,7 +58787,7 @@ static void finrec_delete_weakref(JSRuntime *rt, JSWeakRefHeader *wh) JSValueConst args[2]; args[0] = frd->cb; args[1] = fre->held_val; - JS_EnqueueJob(frd->ctx, js_finrec_job, 2, args); + JS_EnqueueJob(frd->realm, js_finrec_job, 2, args); js_weakref_free(rt, fre->target); js_weakref_free(rt, fre->token); @@ -55065,7 +58822,7 @@ static JSValue js_finrec_constructor(JSContext *ctx, JSValueConst new_target, frd->weakref_header.weakref_type = JS_WEAKREF_TYPE_FINREC; list_add_tail(&frd->weakref_header.link, &ctx->rt->weakref_list); init_list_head(&frd->entries); - frd->ctx = ctx; /* XXX: JS_DupContext() ? */ + frd->realm = JS_DupContext(ctx); frd->cb = JS_DupValue(ctx, cb); JS_SetOpaque(obj, frd); return obj; @@ -55139,29 +58896,42 @@ static const JSClassShortDef js_finrec_class_def[] = { { JS_ATOM_FinalizationRegistry, js_finrec_finalizer, js_finrec_mark }, /* JS_CLASS_FINALIZATION_REGISTRY */ }; -void JS_AddIntrinsicWeakRef(JSContext *ctx) +int JS_AddIntrinsicWeakRef(JSContext *ctx) { JSRuntime *rt = ctx->rt; - + JSValue obj; + /* WeakRef */ if (!JS_IsRegisteredClass(rt, JS_CLASS_WEAK_REF)) { - init_class_range(rt, js_weakref_class_def, JS_CLASS_WEAK_REF, - countof(js_weakref_class_def)); + if (init_class_range(rt, js_weakref_class_def, JS_CLASS_WEAK_REF, + countof(js_weakref_class_def))) + return -1; } - ctx->class_proto[JS_CLASS_WEAK_REF] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_WEAK_REF], - js_weakref_proto_funcs, - countof(js_weakref_proto_funcs)); - JS_NewGlobalCConstructor(ctx, "WeakRef", js_weakref_constructor, 1, ctx->class_proto[JS_CLASS_WEAK_REF]); + obj = JS_NewCConstructor(ctx, JS_CLASS_WEAK_REF, "WeakRef", + js_weakref_constructor, 1, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + NULL, 0, + js_weakref_proto_funcs, countof(js_weakref_proto_funcs), + 0); + if (JS_IsException(obj)) + return -1; + JS_FreeValue(ctx, obj); /* FinalizationRegistry */ if (!JS_IsRegisteredClass(rt, JS_CLASS_FINALIZATION_REGISTRY)) { - init_class_range(rt, js_finrec_class_def, JS_CLASS_FINALIZATION_REGISTRY, - countof(js_finrec_class_def)); - } - ctx->class_proto[JS_CLASS_FINALIZATION_REGISTRY] = JS_NewObject(ctx); - JS_SetPropertyFunctionList(ctx, ctx->class_proto[JS_CLASS_FINALIZATION_REGISTRY], - js_finrec_proto_funcs, - countof(js_finrec_proto_funcs)); - JS_NewGlobalCConstructor(ctx, "FinalizationRegistry", js_finrec_constructor, 1, ctx->class_proto[JS_CLASS_FINALIZATION_REGISTRY]); + if (init_class_range(rt, js_finrec_class_def, JS_CLASS_FINALIZATION_REGISTRY, + countof(js_finrec_class_def))) + return -1; + } + + obj = JS_NewCConstructor(ctx, JS_CLASS_FINALIZATION_REGISTRY, "FinalizationRegistry", + js_finrec_constructor, 1, JS_CFUNC_constructor_or_func, 0, + JS_UNDEFINED, + NULL, 0, + js_finrec_proto_funcs, countof(js_finrec_proto_funcs), + 0); + if (JS_IsException(obj)) + return -1; + JS_FreeValue(ctx, obj); + return 0; } diff --git a/src/couch_quickjs/quickjs/quickjs.h b/src/couch_quickjs/quickjs/quickjs.h index b851cd9ca7..92cc000d0a 100644 --- a/src/couch_quickjs/quickjs/quickjs.h +++ b/src/couch_quickjs/quickjs/quickjs.h @@ -319,8 +319,7 @@ static inline JSValue __JS_NewShortBigInt(JSContext *ctx, int64_t d) (JS_SetProperty) */ #define JS_PROP_THROW_STRICT (1 << 15) -#define JS_PROP_NO_ADD (1 << 16) /* internal use */ -#define JS_PROP_NO_EXOTIC (1 << 17) /* internal use */ +#define JS_PROP_NO_EXOTIC (1 << 16) /* internal use */ #ifndef JS_DEFAULT_STACK_SIZE #define JS_DEFAULT_STACK_SIZE (1024 * 1024) @@ -395,18 +394,18 @@ JSValue JS_GetClassProto(JSContext *ctx, JSClassID class_id); /* the following functions are used to select the intrinsic object to save memory */ JSContext *JS_NewContextRaw(JSRuntime *rt); -void JS_AddIntrinsicBaseObjects(JSContext *ctx); -void JS_AddIntrinsicDate(JSContext *ctx); -void JS_AddIntrinsicEval(JSContext *ctx); -void JS_AddIntrinsicStringNormalize(JSContext *ctx); +int JS_AddIntrinsicBaseObjects(JSContext *ctx); +int JS_AddIntrinsicDate(JSContext *ctx); +int JS_AddIntrinsicEval(JSContext *ctx); +int JS_AddIntrinsicStringNormalize(JSContext *ctx); void JS_AddIntrinsicRegExpCompiler(JSContext *ctx); -void JS_AddIntrinsicRegExp(JSContext *ctx); -void JS_AddIntrinsicJSON(JSContext *ctx); -void JS_AddIntrinsicProxy(JSContext *ctx); -void JS_AddIntrinsicMapSet(JSContext *ctx); -void JS_AddIntrinsicTypedArrays(JSContext *ctx); -void JS_AddIntrinsicPromise(JSContext *ctx); -void JS_AddIntrinsicWeakRef(JSContext *ctx); +int JS_AddIntrinsicRegExp(JSContext *ctx); +int JS_AddIntrinsicJSON(JSContext *ctx); +int JS_AddIntrinsicProxy(JSContext *ctx); +int JS_AddIntrinsicMapSet(JSContext *ctx); +int JS_AddIntrinsicTypedArrays(JSContext *ctx); +int JS_AddIntrinsicPromise(JSContext *ctx); +int JS_AddIntrinsicWeakRef(JSContext *ctx); JSValue js_string_codePointRange(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv); @@ -456,7 +455,11 @@ void JS_FreeAtom(JSContext *ctx, JSAtom v); void JS_FreeAtomRT(JSRuntime *rt, JSAtom v); JSValue JS_AtomToValue(JSContext *ctx, JSAtom atom); JSValue JS_AtomToString(JSContext *ctx, JSAtom atom); -const char *JS_AtomToCString(JSContext *ctx, JSAtom atom); +const char *JS_AtomToCStringLen(JSContext *ctx, size_t *plen, JSAtom atom); +static inline const char *JS_AtomToCString(JSContext *ctx, JSAtom atom) +{ + return JS_AtomToCStringLen(ctx, NULL, atom); +} JSAtom JS_ValueToAtom(JSContext *ctx, JSValueConst val); /* object class support */ @@ -805,6 +808,8 @@ JSValue JS_GetPrototype(JSContext *ctx, JSValueConst val); int JS_GetOwnPropertyNames(JSContext *ctx, JSPropertyEnum **ptab, uint32_t *plen, JSValueConst obj, int flags); +void JS_FreePropertyEnum(JSContext *ctx, JSPropertyEnum *tab, + uint32_t len); int JS_GetOwnProperty(JSContext *ctx, JSPropertyDescriptor *desc, JSValueConst obj, JSAtom prop); @@ -871,6 +876,7 @@ typedef enum JSTypedArrayEnum { JS_TYPED_ARRAY_UINT32, JS_TYPED_ARRAY_BIG_INT64, JS_TYPED_ARRAY_BIG_UINT64, + JS_TYPED_ARRAY_FLOAT16, JS_TYPED_ARRAY_FLOAT32, JS_TYPED_ARRAY_FLOAT64, } JSTypedArrayEnum; @@ -929,12 +935,25 @@ typedef char *JSModuleNormalizeFunc(JSContext *ctx, const char *module_name, void *opaque); typedef JSModuleDef *JSModuleLoaderFunc(JSContext *ctx, const char *module_name, void *opaque); - +typedef JSModuleDef *JSModuleLoaderFunc2(JSContext *ctx, + const char *module_name, void *opaque, + JSValueConst attributes); +/* return -1 if exception, 0 if OK */ +typedef int JSModuleCheckSupportedImportAttributes(JSContext *ctx, void *opaque, + JSValueConst attributes); + /* module_normalize = NULL is allowed and invokes the default module filename normalizer */ void JS_SetModuleLoaderFunc(JSRuntime *rt, JSModuleNormalizeFunc *module_normalize, JSModuleLoaderFunc *module_loader, void *opaque); +/* same as JS_SetModuleLoaderFunc but with attributes. if + module_check_attrs = NULL, no attribute checking is done. */ +void JS_SetModuleLoaderFunc2(JSRuntime *rt, + JSModuleNormalizeFunc *module_normalize, + JSModuleLoaderFunc2 *module_loader, + JSModuleCheckSupportedImportAttributes *module_check_attrs, + void *opaque); /* return the import.meta object of a module */ JSValue JS_GetImportMeta(JSContext *ctx, JSModuleDef *m); JSAtom JS_GetModuleName(JSContext *ctx, JSModuleDef *m); @@ -1029,10 +1048,12 @@ static inline JSValue JS_NewCFunctionMagic(JSContext *ctx, JSCFunctionMagic *fun const char *name, int length, JSCFunctionEnum cproto, int magic) { - return JS_NewCFunction2(ctx, (JSCFunction *)func, name, length, cproto, magic); + /* Used to squelch a -Wcast-function-type warning. */ + JSCFunctionType ft = { .generic_magic = func }; + return JS_NewCFunction2(ctx, ft.generic, name, length, cproto, magic); } -void JS_SetConstructor(JSContext *ctx, JSValueConst func_obj, - JSValueConst proto); +int JS_SetConstructor(JSContext *ctx, JSValueConst func_obj, + JSValueConst proto); /* C property definition */ @@ -1076,6 +1097,8 @@ typedef struct JSCFunctionListEntry { #define JS_DEF_PROP_UNDEFINED 7 #define JS_DEF_OBJECT 8 #define JS_DEF_ALIAS 9 +#define JS_DEF_PROP_ATOM 10 +#define JS_DEF_PROP_BOOL 11 /* Note: c++ does not like nested designators */ #define JS_CFUNC_DEF(name, length, func1) { name, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE, JS_DEF_CFUNC, 0, .u = { .func = { length, JS_CFUNC_generic, { .generic = func1 } } } } @@ -1089,13 +1112,15 @@ typedef struct JSCFunctionListEntry { #define JS_PROP_INT64_DEF(name, val, prop_flags) { name, prop_flags, JS_DEF_PROP_INT64, 0, .u = { .i64 = val } } #define JS_PROP_DOUBLE_DEF(name, val, prop_flags) { name, prop_flags, JS_DEF_PROP_DOUBLE, 0, .u = { .f64 = val } } #define JS_PROP_UNDEFINED_DEF(name, prop_flags) { name, prop_flags, JS_DEF_PROP_UNDEFINED, 0, .u = { .i32 = 0 } } +#define JS_PROP_ATOM_DEF(name, val, prop_flags) { name, prop_flags, JS_DEF_PROP_ATOM, 0, .u = { .i32 = val } } +#define JS_PROP_BOOL_DEF(name, val, prop_flags) { name, prop_flags, JS_DEF_PROP_BOOL, 0, .u = { .i32 = val } } #define JS_OBJECT_DEF(name, tab, len, prop_flags) { name, prop_flags, JS_DEF_OBJECT, 0, .u = { .prop_list = { tab, len } } } #define JS_ALIAS_DEF(name, from) { name, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE, JS_DEF_ALIAS, 0, .u = { .alias = { from, -1 } } } #define JS_ALIAS_BASE_DEF(name, from, base) { name, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE, JS_DEF_ALIAS, 0, .u = { .alias = { from, base } } } -void JS_SetPropertyFunctionList(JSContext *ctx, JSValueConst obj, - const JSCFunctionListEntry *tab, - int len); +int JS_SetPropertyFunctionList(JSContext *ctx, JSValueConst obj, + const JSCFunctionListEntry *tab, + int len); /* C module definition */ @@ -1112,12 +1137,14 @@ int JS_SetModuleExport(JSContext *ctx, JSModuleDef *m, const char *export_name, JSValue val); int JS_SetModuleExportList(JSContext *ctx, JSModuleDef *m, const JSCFunctionListEntry *tab, int len); - +/* associate a JSValue to a C module */ +int JS_SetModulePrivateValue(JSContext *ctx, JSModuleDef *m, JSValue val); +JSValue JS_GetModulePrivateValue(JSContext *ctx, JSModuleDef *m); + /* debug value output */ typedef struct { JS_BOOL show_hidden : 8; /* only show enumerable properties */ - JS_BOOL show_closure : 8; /* show closure variables */ JS_BOOL raw_dump : 8; /* avoid doing autoinit and avoid any malloc() call (for internal use) */ uint32_t max_depth; /* recurse up to this depth, 0 = no limit */ uint32_t max_string_length; /* print no more than this length for @@ -1126,9 +1153,13 @@ typedef struct { arrays or objects, 0 = no limit */ } JSPrintValueOptions; +typedef void JSPrintValueWrite(void *opaque, const char *buf, size_t len); + void JS_PrintValueSetDefaultOptions(JSPrintValueOptions *options); -void JS_PrintValueRT(JSRuntime *rt, FILE *fo, JSValueConst val, const JSPrintValueOptions *options); -void JS_PrintValue(JSContext *ctx, FILE *fo, JSValueConst val, const JSPrintValueOptions *options); +void JS_PrintValueRT(JSRuntime *rt, JSPrintValueWrite *write_func, void *write_opaque, + JSValueConst val, const JSPrintValueOptions *options); +void JS_PrintValue(JSContext *ctx, JSPrintValueWrite *write_func, void *write_opaque, + JSValueConst val, const JSPrintValueOptions *options); #undef js_unlikely #undef js_force_inline diff --git a/src/couch_quickjs/quickjs/run-test262.c b/src/couch_quickjs/quickjs/run-test262.c index d1fa4f30a9..100ed134a9 100644 --- a/src/couch_quickjs/quickjs/run-test262.c +++ b/src/couch_quickjs/quickjs/run-test262.c @@ -78,6 +78,7 @@ char *harness_dir; char *harness_exclude; char *harness_features; char *harness_skip_features; +int *harness_skip_features_count; char *error_filename; char *error_file; FILE *error_out; @@ -372,6 +373,12 @@ static void enumerate_tests(const char *path) namelist_cmp_indirect); } +static void js_print_value_write(void *opaque, const char *buf, size_t len) +{ + FILE *fo = opaque; + fwrite(buf, 1, len, fo); +} + static JSValue js_print(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { @@ -397,7 +404,7 @@ static JSValue js_print(JSContext *ctx, JSValueConst this_val, fwrite(str, 1, len, outfile); JS_FreeCString(ctx, str); } else { - JS_PrintValue(ctx, outfile, v, NULL); + JS_PrintValue(ctx, js_print_value_write, outfile, v, NULL); } } fputc('\n', outfile); @@ -490,8 +497,7 @@ static void *agent_start(void *arg) JS_FreeValue(ctx, ret_val); for(;;) { - JSContext *ctx1; - ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1); + ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), NULL); if (ret < 0) { js_std_dump_error(ctx); break; @@ -830,13 +836,21 @@ static char *load_file(const char *filename, size_t *lenp) return buf; } +static int json_module_init_test(JSContext *ctx, JSModuleDef *m) +{ + JSValue val; + val = JS_GetModulePrivateValue(ctx, m); + JS_SetModuleExport(ctx, m, "default", val); + return 0; +} + static JSModuleDef *js_module_loader_test(JSContext *ctx, - const char *module_name, void *opaque) + const char *module_name, void *opaque, + JSValueConst attributes) { size_t buf_len; uint8_t *buf; JSModuleDef *m; - JSValue func_val; char *filename, *slash, path[1024]; // interpret import("bar.js") from path/to/foo.js as @@ -858,15 +872,33 @@ static JSModuleDef *js_module_loader_test(JSContext *ctx, return NULL; } - /* compile the module */ - func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name, - JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); - js_free(ctx, buf); - if (JS_IsException(func_val)) - return NULL; - /* the module is already referenced, so we must free it */ - m = JS_VALUE_GET_PTR(func_val); - JS_FreeValue(ctx, func_val); + if (js_module_test_json(ctx, attributes) == 1) { + /* compile as JSON */ + JSValue val; + val = JS_ParseJSON(ctx, (char *)buf, buf_len, module_name); + js_free(ctx, buf); + if (JS_IsException(val)) + return NULL; + m = JS_NewCModule(ctx, module_name, json_module_init_test); + if (!m) { + JS_FreeValue(ctx, val); + return NULL; + } + /* only export the "default" symbol which will contain the JSON object */ + JS_AddModuleExport(ctx, m, "default"); + JS_SetModulePrivateValue(ctx, m, val); + } else { + JSValue func_val; + /* compile the module */ + func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name, + JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + js_free(ctx, buf); + if (JS_IsException(func_val)) + return NULL; + /* the module is already referenced, so we must free it */ + m = JS_VALUE_GET_PTR(func_val); + JS_FreeValue(ctx, func_val); + } return m; } @@ -1238,8 +1270,7 @@ static int eval_buf(JSContext *ctx, const char *buf, size_t buf_len, JS_FreeValue(ctx, res_val); } for(;;) { - JSContext *ctx1; - ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1); + ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), NULL); if (ret < 0) { res_val = JS_EXCEPTION; break; @@ -1581,7 +1612,7 @@ int run_test_buf(const char *filename, const char *harness, namelist_t *ip, JS_SetCanBlock(rt, can_block); /* loader for ES6 modules */ - JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, (void *)filename); + JS_SetModuleLoaderFunc2(rt, NULL, js_module_loader_test, NULL, (void *)filename); add_helpers(ctx); @@ -1706,10 +1737,13 @@ int run_test(const char *filename, int index) p = find_tag(desc, "features:", &state); if (p) { while ((option = get_option(&p, &state)) != NULL) { + char *p1; if (find_word(harness_features, option)) { /* feature is enabled */ - } else if (find_word(harness_skip_features, option)) { + } else if ((p1 = find_word(harness_skip_features, option)) != NULL) { /* skip disabled feature */ + if (harness_skip_features_count) + harness_skip_features_count[p1 - harness_skip_features]++; skip |= 1; } else { /* feature is not listed: skip and warn */ @@ -1882,7 +1916,7 @@ int run_test262_harness_test(const char *filename, BOOL is_module) JS_SetCanBlock(rt, can_block); /* loader for ES6 modules */ - JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, (void *)filename); + JS_SetModuleLoaderFunc2(rt, NULL, js_module_loader_test, NULL, (void *)filename); add_helpers(ctx); @@ -1906,10 +1940,9 @@ int run_test262_harness_test(const char *filename, BOOL is_module) JS_FreeValue(ctx, res_val); } for(;;) { - JSContext *ctx1; - ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1); + ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), NULL); if (ret < 0) { - js_std_dump_error(ctx1); + js_std_dump_error(ctx); ret_code = 1; } else if (ret == 0) { break; @@ -2043,6 +2076,7 @@ int main(int argc, char **argv) const char *ignore = ""; BOOL is_test262_harness = FALSE; BOOL is_module = FALSE; + BOOL count_skipped_features = FALSE; clock_t clocks; #if !defined(_WIN32) @@ -2110,6 +2144,8 @@ int main(int argc, char **argv) is_test262_harness = TRUE; } else if (str_equal(arg, "--module")) { is_module = TRUE; + } else if (str_equal(arg, "--count_skipped_features")) { + count_skipped_features = TRUE; } else { fatal(1, "unknown option: %s", arg); break; @@ -2144,6 +2180,14 @@ int main(int argc, char **argv) clocks = clock(); + if (count_skipped_features) { + /* not storage efficient but it is simple */ + size_t size; + size = sizeof(harness_skip_features_count[0]) * strlen(harness_skip_features); + harness_skip_features_count = malloc(size); + memset(harness_skip_features_count, 0, size); + } + if (is_dir_list) { if (optind < argc && !isdigit((unsigned char)argv[optind][0])) { filename = argv[optind++]; @@ -2194,6 +2238,30 @@ int main(int argc, char **argv) printf("\n"); } + if (count_skipped_features) { + size_t i, n, len = strlen(harness_skip_features); + BOOL disp = FALSE; + int c; + for(i = 0; i < len; i++) { + if (harness_skip_features_count[i] != 0) { + if (!disp) { + disp = TRUE; + printf("%-30s %7s\n", "SKIPPED FEATURE", "COUNT"); + } + for(n = 0; n < 30; n++) { + c = harness_skip_features[i + n]; + if (is_word_sep(c)) + break; + putchar(c); + } + for(; n < 30; n++) + putchar(' '); + printf(" %7d\n", harness_skip_features_count[i]); + } + } + printf("\n"); + } + if (is_dir_list) { fprintf(stderr, "Result: %d/%d error%s", test_failed, test_count, test_count != 1 ? "s" : ""); @@ -2223,6 +2291,8 @@ int main(int argc, char **argv) namelist_free(&exclude_list); namelist_free(&exclude_dir_list); free(harness_dir); + free(harness_skip_features); + free(harness_skip_features_count); free(harness_features); free(harness_exclude); free(error_file); diff --git a/src/couch_quickjs/quickjs/test262.conf b/src/couch_quickjs/quickjs/test262.conf index 68906820ca..cf8b91d539 100644 --- a/src/couch_quickjs/quickjs/test262.conf +++ b/src/couch_quickjs/quickjs/test262.conf @@ -64,12 +64,12 @@ Array.prototype.flatMap Array.prototype.includes Array.prototype.values ArrayBuffer -arraybuffer-transfer=skip +arraybuffer-transfer arrow-function async-functions async-iteration Atomics -Atomics.pause=skip +Atomics.pause Atomics.waitAsync=skip BigInt caller @@ -103,12 +103,12 @@ destructuring-assignment destructuring-binding dynamic-import error-cause -Error.isError=skip +Error.isError explicit-resource-management=skip exponentiation export-star-as-namespace-from-module FinalizationRegistry -Float16Array=skip +Float16Array Float32Array Float64Array for-in-order @@ -116,9 +116,9 @@ for-of generators globalThis hashbang -host-gc-required=skip -import-assertions=skip -import-attributes=skip +host-gc-required +immutable-arraybuffer=skip +import-attributes import-defer=skip import.meta Int16Array @@ -142,17 +142,18 @@ Intl.NumberFormat-v3=skip Intl.RelativeTimeFormat=skip Intl.Segmenter=skip IsHTMLDDA -iterator-helpers=skip +iterator-helpers iterator-sequencing=skip -json-modules=skip +json-modules json-parse-with-source=skip json-superset legacy-regexp=skip let logical-assignment-operators Map -Math.sumPrecise=skip +Math.sumPrecise new.target +nonextensible-applies-to-private=skip numeric-separator-literal object-rest object-spread @@ -162,7 +163,7 @@ Object.is optional-catch-binding optional-chaining Promise -promise-try=skip +promise-try promise-with-resolvers Promise.allSettled Promise.any @@ -177,19 +178,21 @@ regexp-dotall regexp-duplicate-named-groups=skip regexp-lookbehind regexp-match-indices -regexp-modifiers=skip +regexp-modifiers regexp-named-groups regexp-unicode-property-escapes -regexp-v-flag=skip -RegExp.escape=skip -resizable-arraybuffer=skip +regexp-v-flag +RegExp.escape +resizable-arraybuffer rest-parameters Set -set-methods=skip +set-methods ShadowRealm=skip SharedArrayBuffer source-phase-imports-module-source=skip source-phase-imports=skip +stable-array-sort +stable-typedarray-sort string-trimming String.fromCodePoint String.prototype.at @@ -230,6 +233,7 @@ Uint32Array Uint8Array uint8array-base64=skip Uint8ClampedArray +upsert WeakMap WeakRef WeakSet @@ -250,32 +254,6 @@ test262/test/built-ins/ThrowTypeError/unique-per-realm-function-proto.js #test262/test/built-ins/RegExp/CharacterClassEscapes/ #test262/test/built-ins/RegExp/property-escapes/ -# feature regexp-v-flag is missing in the tests -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-digit-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-digit-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-digit-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-digit-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-digit-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-digit-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-digit-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-digit-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-whitespace-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-whitespace-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-whitespace-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-whitespace-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-word-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-word-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-word-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-word-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-whitespace-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-whitespace-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-whitespace-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-whitespace-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-word-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-word-class-escape-negative-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-word-class-escape-positive-cases.js -test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-word-class-escape-positive-cases.js - # not yet in official specification test262/test/built-ins/String/prototype/match/cstm-matcher-on-bigint-primitive.js test262/test/built-ins/String/prototype/match/cstm-matcher-on-bigint-primitive.js @@ -331,24 +309,12 @@ test262/test/built-ins/String/prototype/split/cstm-split-on-string-primitive.js # spec updates it in this case) test262/test/staging/sm/Array/frozen-dense-array.js +# does not match spec +test262/test/staging/sm/Iterator/from/wrap-next-not-object-throws.js + # not supported -test262/test/staging/sm/Set/difference.js -test262/test/staging/sm/Set/intersection.js -test262/test/staging/sm/Set/is-disjoint-from.js -test262/test/staging/sm/Set/is-subset-of.js -test262/test/staging/sm/Set/is-superset-of.js -test262/test/staging/sm/Set/symmetric-difference.js -test262/test/staging/sm/Set/union.js test262/test/staging/sm/extensions/censor-strict-caller.js test262/test/staging/sm/JSON/parse-with-source.js -test262/test/staging/sm/RegExp/flags.js -test262/test/staging/sm/RegExp/prototype.js - -# no f16 -test262/test/staging/sm/Math/f16round.js -test262/test/staging/sm/TypedArray/sort_small.js -test262/test/staging/sm/extensions/dataview.js -test262/test/staging/sm/TypedArray/toString.js # not standard test262/test/staging/sm/Function/builtin-no-construct.js @@ -357,12 +323,25 @@ test262/test/staging/sm/Function/function-toString-builtin-name.js test262/test/staging/sm/extensions/arguments-property-access-in-function.js test262/test/staging/sm/extensions/function-caller-skips-eval-frames.js test262/test/staging/sm/extensions/function-properties.js +test262/test/staging/sm/regress/regress-577648-1.js +test262/test/staging/sm/regress/regress-577648-2.js +test262/test/staging/sm/regress/regress-584355.js +test262/test/staging/sm/regress/regress-586482-1.js +test262/test/staging/sm/regress/regress-586482-2.js +test262/test/staging/sm/regress/regress-586482-3.js +test262/test/staging/sm/regress/regress-586482-4.js +test262/test/staging/sm/regress/regress-699682.js + # RegExp toSource not fully compliant test262/test/staging/sm/RegExp/toString.js test262/test/staging/sm/RegExp/source.js test262/test/staging/sm/RegExp/escape.js +# RegExp.lastMatch not supported +test262/test/staging/sm/statements/regress-642975.js # source directives are not standard yet test262/test/staging/sm/syntax/syntax-parsed-arrow-then-directive.js +# returning "bound fn" as initialName for a function is permitted by the spec +test262/test/staging/sm/Function/function-toString-builtin.js [tests] # list test files or use config.testdir diff --git a/src/couch_quickjs/quickjs/test262_errors.txt b/src/couch_quickjs/quickjs/test262_errors.txt index 5fb832d6ba..cc49b2f322 100644 --- a/src/couch_quickjs/quickjs/test262_errors.txt +++ b/src/couch_quickjs/quickjs/test262_errors.txt @@ -1,67 +1,68 @@ -test262/test/language/module-code/top-level-await/module-graphs-does-not-hang.js:10: TypeError: $DONE() not called -test262/test/staging/sm/Date/UTC-convert-all-arguments.js:75: Test262Error: index 1: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true -test262/test/staging/sm/Date/constructor-convert-all-arguments.js:75: Test262Error: index undefined: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true -test262/test/staging/sm/Date/non-iso.js:76: Test262Error: Expected SameValue(«NaN», «-40071559730000») to be true -test262/test/staging/sm/Date/two-digit-years.js:76: Test262Error: Expected SameValue(«915177600000», «NaN») to be true -test262/test/staging/sm/Function/arguments-parameter-shadowing.js:15: Test262Error: Expected SameValue(«true», «false») to be true -test262/test/staging/sm/Function/constructor-binding.js:12: Test262Error: Expected SameValue(«"function"», «"undefined"») to be true -test262/test/staging/sm/Function/function-bind.js:14: Test262Error: Conforms to NativeFunction Syntax: "function bound unbound() {\n [native code]\n}" -test262/test/staging/sm/Function/function-name-for.js:12: Test262Error: Expected SameValue(«""», «"forInHead"») to be true -test262/test/staging/sm/Function/function-toString-builtin.js:14: Test262Error: Expected match to '/^\s*function\s*(get|set)?\s*(\w+|(?:'[^']*')|(?:"[^"]*")|\d+|(?:\[[^\]]+\]))?\s*\(\s*\)\s*\{\s*\[native code\]\s*\}\s*$/', Actual value 'function bound fn() { - [native code] -}' Expected SameValue(«null», «null») to be false -test262/test/staging/sm/Function/implicit-this-in-parameter-expression.js:13: Test262Error: Expected SameValue(«[object Object]», «undefined») to be true -test262/test/staging/sm/Function/invalid-parameter-list.js:35: Error: Assertion failed: expected exception SyntaxError, no exception thrown -test262/test/staging/sm/JSON/parse-number-syntax.js:39: Test262Error: parsing string <1.> threw a non-SyntaxError exception: Test262Error: string <1.> shouldn't have parsed as JSON Expected SameValue(«false», «true») to be true Expected SameValue(«true», «false») to be true -test262/test/staging/sm/JSON/parse-syntax-errors-02.js:51: Test262Error: parsing string <["Illegal backslash escape: \x15"]> threw a non-SyntaxError exception: Test262Error: string <["Illegal backslash escape: \x15"]> shouldn't have parsed as JSON Expected SameValue(«false», «true») to be true Expected SameValue(«true», «false») to be true -test262/test/staging/sm/Math/cbrt-approx.js:26: Error: got 1.39561242508609, expected a number near 1.3956124250860895 (relative error: 2) -test262/test/staging/sm/RegExp/constructor-ordering-2.js:15: Test262Error: Expected SameValue(«false», «true») to be true -test262/test/staging/sm/RegExp/match-trace.js:13: Test262Error: Expected SameValue(«"get:flags,get:unicode,set:lastIndex,get:exec,call:exec,get:result[0],get:exec,call:exec,get:result[0],get:exec,call:exec,"», «"get:flags,set:lastIndex,get:exec,call:exec,get:result[0],get:exec,call:exec,get:result[0],get:exec,call:exec,"») to be true -test262/test/staging/sm/RegExp/regress-613820-1.js:13: Test262Error: Expected SameValue(«"aaa"», «"aa"») to be true -test262/test/staging/sm/RegExp/regress-613820-2.js:13: Test262Error: Expected SameValue(«"f"», «undefined») to be true -test262/test/staging/sm/RegExp/regress-613820-3.js:13: Test262Error: Expected SameValue(«"aab"», «"aa"») to be true -test262/test/staging/sm/RegExp/replace-trace.js:13: Test262Error: Expected SameValue(«"get:flags,get:unicode,set:lastIndex,get:exec,call:exec,get:result[0],get:exec,call:exec,get:result[length],get:result[0],get:result[index],get:result[groups],"», «"get:flags,set:lastIndex,get:exec,call:exec,get:result[0],get:exec,call:exec,get:result[length],get:result[0],get:result[index],get:result[groups],"») to be true -test262/test/staging/sm/RegExp/unicode-ignoreCase-escape.js:22: Test262Error: Actual argument shouldn't be nullish. -test262/test/staging/sm/RegExp/unicode-ignoreCase-word-boundary.js:13: Test262Error: Expected SameValue(«false», «true») to be true -test262/test/staging/sm/String/match-defines-match-elements.js:52: Test262Error: Expected SameValue(«true», «false») to be true -test262/test/staging/sm/TypedArray/constructor-buffer-sequence.js:73: Error: Assertion failed: expected exception ExpectedError, got Error: Poisoned Value +test262/test/annexB/language/expressions/assignmenttargettype/callexpression-as-for-in-lhs.js:27: SyntaxError: invalid for in/of left hand-side +test262/test/annexB/language/expressions/assignmenttargettype/callexpression-as-for-of-lhs.js:27: SyntaxError: invalid for in/of left hand-side +test262/test/annexB/language/expressions/assignmenttargettype/callexpression-in-compound-assignment.js:33: SyntaxError: invalid assignment left-hand side +test262/test/annexB/language/expressions/assignmenttargettype/callexpression-in-postfix-update.js:27: SyntaxError: invalid increment/decrement operand +test262/test/annexB/language/expressions/assignmenttargettype/callexpression-in-prefix-update.js:27: SyntaxError: invalid increment/decrement operand +test262/test/annexB/language/expressions/assignmenttargettype/callexpression.js:33: SyntaxError: invalid assignment left-hand side +test262/test/annexB/language/expressions/assignmenttargettype/cover-callexpression-and-asyncarrowhead.js:20: SyntaxError: invalid assignment left-hand side +test262/test/language/expressions/assignment/S11.13.1_A6_T1.js:23: Test262Error: #1: innerX === undefined. Actual: 1 +test262/test/language/expressions/assignment/S11.13.1_A6_T2.js:23: Test262Error: #1: innerX === 2. Actual: 1 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.1_T1.js:24: Test262Error: #1: innerX === 2. Actual: 12 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.2_T1.js:24: Test262Error: #1: innerX === 2. Actual: 5 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.3_T1.js:24: Test262Error: #1: innerX === 2. Actual: 3 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.4_T1.js:24: Test262Error: #1: innerX === 2. Actual: 4 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.5_T1.js:24: Test262Error: #1: innerX === 2. Actual: 4 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.6_T1.js:24: Test262Error: #1: innerX === 2. Actual: 8 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.7_T1.js:24: Test262Error: #1: innerX === 2. Actual: 4 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.8_T1.js:24: Test262Error: #1: innerX === 2. Actual: 4 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.9_T1.js:24: Test262Error: #1: innerX === 2. Actual: 1 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.10_T1.js:24: Test262Error: #1: innerX === 2. Actual: 5 +test262/test/language/expressions/compound-assignment/S11.13.2_A6.11_T1.js:24: Test262Error: #1: innerX === 2. Actual: 5 +test262/test/language/identifier-resolution/assign-to-global-undefined.js:20: strict mode: expected error +test262/test/language/statements/expression/S12.4_A1.js:15: unexpected error type: Test262: This statement should not be evaluated. +test262/test/language/statements/expression/S12.4_A1.js:15: strict mode: unexpected error type: Test262: This statement should not be evaluated. +test262/test/staging/sm/Function/arguments-parameter-shadowing.js:14: Test262Error: Expected SameValue(«true», «false») to be true +test262/test/staging/sm/Function/constructor-binding.js:11: Test262Error: Expected SameValue(«"function"», «"undefined"») to be true +test262/test/staging/sm/Function/constructor-binding.js:11: strict mode: Test262Error: Expected SameValue(«"function"», «"undefined"») to be true +test262/test/staging/sm/Function/function-bind.js:24: Test262Error: Conforms to NativeFunction Syntax: "function bound unbound() {\n [native code]\n}" +test262/test/staging/sm/Function/function-name-for.js:13: Test262Error: Expected SameValue(«""», «"forInHead"») to be true +test262/test/staging/sm/Function/implicit-this-in-parameter-expression.js:12: Test262Error: Expected SameValue(«[object Object]», «undefined») to be true +test262/test/staging/sm/Function/invalid-parameter-list.js:13: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/Function/invalid-parameter-list.js:13: strict mode: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/RegExp/regress-613820-1.js:12: Test262Error: Actual [aaa, aa, a] and expected [aa, a, a] should have the same contents. +test262/test/staging/sm/RegExp/regress-613820-1.js:12: strict mode: Test262Error: Actual [aaa, aa, a] and expected [aa, a, a] should have the same contents. +test262/test/staging/sm/RegExp/regress-613820-2.js:12: Test262Error: Actual [foobar, f, o, o, b, a, r] and expected [foobar, undefined, undefined, undefined, b, a, r] should have the same contents. +test262/test/staging/sm/RegExp/regress-613820-2.js:12: strict mode: Test262Error: Actual [foobar, f, o, o, b, a, r] and expected [foobar, undefined, undefined, undefined, b, a, r] should have the same contents. +test262/test/staging/sm/RegExp/regress-613820-3.js:12: Test262Error: Actual [aab, a, undefined, ab] and expected [aa, undefined, a, undefined] should have the same contents. +test262/test/staging/sm/RegExp/regress-613820-3.js:12: strict mode: Test262Error: Actual [aab, a, undefined, ab] and expected [aa, undefined, a, undefined] should have the same contents. +test262/test/staging/sm/TypedArray/constructor-buffer-sequence.js:29: Test262Error: Expected a ExpectedError but got a Error +test262/test/staging/sm/TypedArray/constructor-buffer-sequence.js:29: strict mode: Test262Error: Expected a ExpectedError but got a Error test262/test/staging/sm/TypedArray/prototype-constructor-identity.js:17: Test262Error: Expected SameValue(«2», «6») to be true -test262/test/staging/sm/TypedArray/set-detached-bigint.js:27: Error: Assertion failed: expected exception SyntaxError, got RangeError: invalid array length -test262/test/staging/sm/TypedArray/set-detached.js:112: RangeError: invalid array length -test262/test/staging/sm/TypedArray/sort-negative-nan.js:102: TypeError: cannot read property 'name' of undefined -test262/test/staging/sm/TypedArray/sort_modifications.js:12: Test262Error: Int8Array at index 0 for size 4 Expected SameValue(«0», «1») to be true -test262/test/staging/sm/TypedArray/subarray.js:15: Test262Error: Expected SameValue(«0», «1») to be true -test262/test/staging/sm/async-functions/async-contains-unicode-escape.js:45: Error: Assertion failed: expected exception SyntaxError, no exception thrown -test262/test/staging/sm/async-functions/await-error.js:12: Test262Error: Expected SameValue(«false», «true») to be true -test262/test/staging/sm/async-functions/await-in-arrow-parameters.js:33: Error: Assertion failed: expected exception SyntaxError, no exception thrown - AsyncFunction:(a = (b = await/r/g) => {}) => {} -test262/test/staging/sm/class/boundFunctionSubclassing.js:12: Test262Error: Expected SameValue(«false», «true») to be true -test262/test/staging/sm/class/compPropNames.js:26: Error: Expected syntax error: ({[1, 2]: 3}) -test262/test/staging/sm/class/methDefn.js:26: Error: Expected syntax error: b = {a() => 0} -test262/test/staging/sm/class/strictExecution.js:32: Error: Assertion failed: expected exception TypeError, no exception thrown -test262/test/staging/sm/class/superPropOrdering.js:83: Error: Assertion failed: expected exception TypeError, no exception thrown -test262/test/staging/sm/expressions/optional-chain.js:25: Error: Assertion failed: expected exception SyntaxError, no exception thrown -test262/test/staging/sm/expressions/short-circuit-compound-assignment-const.js:97: TypeError: 'a' is read-only -test262/test/staging/sm/expressions/short-circuit-compound-assignment-tdz.js:23: Error: Assertion failed: expected exception ReferenceError, got TypeError: 'a' is read-only -test262/test/staging/sm/extensions/TypedArray-set-object-funky-length-detaches.js:55: RangeError: invalid array length -test262/test/staging/sm/extensions/regress-469625-01.js:16: Test262Error: TM: Array prototype and expression closures Expected SameValue(«"TypeError: [].__proto__ is not a function"», «"TypeError: not a function"») to be true -test262/test/staging/sm/generators/syntax.js:30: Error: Assertion failed: expected SyntaxError, but no exception thrown - function* g() { (function* yield() {}); } -test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-arguments.js:14: Test262Error: Expected SameValue(«"object"», «"function"») to be true -test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-eval.js:12: Test262Error: Expected SameValue(«"outer-gouter-geval-gtruefalseq"», «"outer-geval-gwith-gtruefalseq"») to be true -test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-if.js:20: TypeError: not a function -test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-notapplicable.js:15: Test262Error: Expected SameValue(«function x() {2}», «function x() {1}») to be true +test262/test/staging/sm/TypedArray/prototype-constructor-identity.js:17: strict mode: Test262Error: Expected SameValue(«2», «6») to be true +test262/test/staging/sm/TypedArray/sort_modifications.js:9: Test262Error: Int8Array at index 0 for size 4 Expected SameValue(«0», «1») to be true +test262/test/staging/sm/TypedArray/sort_modifications.js:9: strict mode: Test262Error: Int8Array at index 0 for size 4 Expected SameValue(«0», «1») to be true +test262/test/staging/sm/async-functions/async-contains-unicode-escape.js:11: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/async-functions/async-contains-unicode-escape.js:11: strict mode: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/async-functions/await-in-arrow-parameters.js:10: Test262Error: AsyncFunction:(a = (b = await/r/g) => {}) => {} Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/async-functions/await-in-arrow-parameters.js:10: strict mode: Test262Error: AsyncFunction:(a = (b = await/r/g) => {}) => {} Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/class/boundFunctionSubclassing.js:9: Test262Error: Expected SameValue(«false», «true») to be true +test262/test/staging/sm/class/boundFunctionSubclassing.js:9: strict mode: Test262Error: Expected SameValue(«false», «true») to be true +test262/test/staging/sm/class/strictExecution.js:13: Test262Error: Expected a TypeError to be thrown but no exception was thrown at all +test262/test/staging/sm/class/superPropOrdering.js:17: Test262Error: Expected a TypeError to be thrown but no exception was thrown at all +test262/test/staging/sm/class/superPropOrdering.js:17: strict mode: Test262Error: Expected a TypeError to be thrown but no exception was thrown at all +test262/test/staging/sm/generators/syntax.js:50: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-arguments.js:13: Test262Error: Expected SameValue(«"object"», «"function"») to be true +test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-eval.js:11: Test262Error: Expected SameValue(«"outer-gouter-geval-gtruefalseq"», «"outer-geval-gwith-gtruefalseq"») to be true +test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-if.js:19: TypeError: not a function +test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-notapplicable.js:14: Test262Error: Expected SameValue(«function x() {2}», «function x() {1}») to be true test262/test/staging/sm/lexical-environment/block-scoped-functions-deprecated-redecl.js:23: Test262Error: Expected SameValue(«3», «4») to be true -test262/test/staging/sm/lexical-environment/var-in-catch-body-annex-b-eval.js:17: Test262Error: Expected SameValue(«"g"», «"global-x"») to be true -test262/test/staging/sm/object/defineProperties-order.js:14: Test262Error: Expected SameValue(«"ownKeys,getOwnPropertyDescriptor,getOwnPropertyDescriptor,get,get"», «"ownKeys,getOwnPropertyDescriptor,get,getOwnPropertyDescriptor,get"») to be true -test262/test/staging/sm/regress/regress-577648-1.js:21: Test262Error: 1 Expected SameValue(«true», «false») to be true -test262/test/staging/sm/regress/regress-577648-2.js:14: Test262Error: Expected SameValue(«true», «false») to be true -test262/test/staging/sm/regress/regress-584355.js:12: Test262Error: Expected SameValue(«"function f () { ff (); }"», «"undefined"») to be true -test262/test/staging/sm/regress/regress-586482-1.js:19: Test262Error: ok Expected SameValue(«true», «false») to be true -test262/test/staging/sm/regress/regress-586482-2.js:19: Test262Error: ok Expected SameValue(«true», «false») to be true -test262/test/staging/sm/regress/regress-586482-3.js:18: Test262Error: ok Expected SameValue(«true», «false») to be true -test262/test/staging/sm/regress/regress-586482-4.js:14: Test262Error: ok Expected SameValue(«function() { this.f(); }», «undefined») to be true -test262/test/staging/sm/regress/regress-602621.js:14: Test262Error: function sub-statement must override arguments Expected SameValue(«"function"», «"object"») to be true -test262/test/staging/sm/regress/regress-699682.js:15: Test262Error: Expected SameValue(«false», «true») to be true -test262/test/staging/sm/regress/regress-1383630.js:30: Error: Assertion failed: expected exception TypeError, no exception thrown -test262/test/staging/sm/statements/arrow-function-in-for-statement-head.js:15: Test262Error: expected syntax error, got Error: didn't throw Expected SameValue(«false», «true») to be true -test262/test/staging/sm/statements/regress-642975.js:14: Test262Error: Expected SameValue(«undefined», «"y"») to be true -test262/test/staging/sm/statements/try-completion.js:17: Test262Error: Expected SameValue(«"try"», «undefined») to be true +test262/test/staging/sm/lexical-environment/var-in-catch-body-annex-b-eval.js:16: Test262Error: Expected SameValue(«"g"», «"global-x"») to be true +test262/test/staging/sm/object/defineProperties-order.js:11: Test262Error: Expected SameValue(«"ownKeys,getOwnPropertyDescriptor,getOwnPropertyDescriptor,get,get"», «"ownKeys,getOwnPropertyDescriptor,get,getOwnPropertyDescriptor,get"») to be true +test262/test/staging/sm/object/defineProperties-order.js:11: strict mode: Test262Error: Expected SameValue(«"ownKeys,getOwnPropertyDescriptor,getOwnPropertyDescriptor,get,get"», «"ownKeys,getOwnPropertyDescriptor,get,getOwnPropertyDescriptor,get"») to be true +test262/test/staging/sm/regress/regress-602621.js:13: Test262Error: function sub-statement must override arguments Expected SameValue(«"function"», «"object"») to be true +test262/test/staging/sm/regress/regress-1383630.js:28: Test262Error: proxy must report the same value for the non-writable, non-configurable property Expected a TypeError to be thrown but no exception was thrown at all +test262/test/staging/sm/regress/regress-1383630.js:28: strict mode: Test262Error: proxy must report the same value for the non-writable, non-configurable property Expected a TypeError to be thrown but no exception was thrown at all +test262/test/staging/sm/statements/arrow-function-in-for-statement-head.js:13: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/statements/arrow-function-in-for-statement-head.js:13: strict mode: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all +test262/test/staging/sm/statements/try-completion.js:11: Test262Error: Expected SameValue(«"try"», «undefined») to be true +test262/test/staging/sm/statements/try-completion.js:11: strict mode: Test262Error: Expected SameValue(«"try"», «undefined») to be true diff --git a/src/couch_quickjs/quickjs/tests/test262.patch b/src/couch_quickjs/quickjs/tests/test262.patch index b6f4aa5ece..d7cba88cc0 100644 --- a/src/couch_quickjs/quickjs/tests/test262.patch +++ b/src/couch_quickjs/quickjs/tests/test262.patch @@ -1,5 +1,5 @@ diff --git a/harness/atomicsHelper.js b/harness/atomicsHelper.js -index 9828b15..4a5919d 100644 +index 9828b15..9e24d64 100644 --- a/harness/atomicsHelper.js +++ b/harness/atomicsHelper.js @@ -272,10 +272,14 @@ $262.agent.waitUntil = function(typedArray, index, expected) { @@ -14,9 +14,9 @@ index 9828b15..4a5919d 100644 +// small: 200, +// long: 1000, +// huge: 10000, -+ yield: 20, -+ small: 20, -+ long: 100, ++ yield: 40, ++ small: 40, ++ long: 200, + huge: 1000, }; @@ -70,36 +70,3 @@ index b397be0..c197ddc 100644 } return result; } -diff --git a/harness/sm/non262.js b/harness/sm/non262.js -index c1829e3..3a3ee27 100644 ---- a/harness/sm/non262.js -+++ b/harness/sm/non262.js -@@ -41,8 +41,6 @@ globalThis.createNewGlobal = function() { - return $262.createRealm().global - } - --function print(...args) { --} - function assertEq(...args) { - assert.sameValue(...args) - } -@@ -71,4 +69,4 @@ if (globalThis.createExternalArrayBuffer === undefined) { - if (globalThis.enableGeckoProfilingWithSlowAssertions === undefined) { - globalThis.enableGeckoProfilingWithSlowAssertions = globalThis.enableGeckoProfiling = - globalThis.disableGeckoProfiling = () => {} --} -\ No newline at end of file -+} -diff --git a/test/staging/sm/misc/new-with-non-constructor.js b/test/staging/sm/misc/new-with-non-constructor.js -index 18c2f0c..f9aa209 100644 ---- a/test/staging/sm/misc/new-with-non-constructor.js -+++ b/test/staging/sm/misc/new-with-non-constructor.js -@@ -16,7 +16,7 @@ function checkConstruct(thing) { - new thing(); - assert.sameValue(0, 1, "not reached " + thing); - } catch (e) { -- assert.sameValue(e.message.includes(" is not a constructor") || -+ assert.sameValue(e.message.includes("not a constructor") || - e.message === "Function.prototype.toString called on incompatible object", true); - } - } diff --git a/src/couch_quickjs/rebar.config.script b/src/couch_quickjs/rebar.config.script index 60ab64070b..f6546e9997 100644 --- a/src/couch_quickjs/rebar.config.script +++ b/src/couch_quickjs/rebar.config.script @@ -15,8 +15,13 @@ % Windows is "special" so we treat it "specially" Msys = "msys2_shell.cmd -defterm -no-start -ucrt64 -here -lc ". +% Disabled CONFIG_LTO=y as of June 2025. It interfered with -j8 and microbench +% on quickjs repo didn't show any measurable per benefit for it. It even looked +% a bit slower on ubuntu-noble/x86_64 (lto:9032ns vs nolto:8746ns) and only a +% tiny bit faster on MacOS x86_64 (lto:12646 vs nolto:12614) +% PreHooks = [ - {"(linux|darwin)", compile, "make CONFIG_LTO=y -C quickjs -j8 libquickjs.lto.a qjsc"}, + {"(linux|darwin)", compile, "make -C quickjs -j8 libquickjs.a qjsc"}, {"freebsd", compile, "gmake -C quickjs -j8 libquickjs.a qjsc"}, {"win32", compile, Msys ++ "'make -C quickjs -j8 libquickjs.a qjsc.exe'"}, {"(linux|darwin|freebsd|win32)", compile, "escript build_js.escript compile"} @@ -38,14 +43,9 @@ ResetFlags = [ {"EXE_LDFLAGS", ""} ]. -LinuxDarwinEnv = [ - {"CFLAGS", "$CFLAGS -flto -g -Wall -DCONFIG_LTO=y -O2 -Iquickjs"}, - {"LDFLAGS", "$LDFLAGS -flto -lm quickjs/libquickjs.lto.a"} -] ++ ResetFlags. - -FreeBSDEnv = [ +UnixEnv = [ {"CFLAGS", "$CFLAGS -g -Wall -O2 -Iquickjs"}, - {"LDFLAGS", "$LDFLAGS -lm -lpthread quickjs/libquickjs.a"} + {"LDFLAGS", "$LDFLAGS quickjs/libquickjs.a -lpthread -lm"} ] ++ ResetFlags. WindowsEnv = [ @@ -73,10 +73,8 @@ WindowsMainjsSrc = WindowsBaseSrc ++ UnixMainjsSrc. WindowsCoffeeSrc = WindowsBaseSrc ++ UnixCoffeeSrc. PortSpecs = [ - {"(linux|darwin)", "priv/couchjs_mainjs", UnixMainjsSrc, [{env, LinuxDarwinEnv}]}, - {"(linux|darwin)", "priv/couchjs_coffee", UnixCoffeeSrc, [{env, LinuxDarwinEnv}]}, - {"freebsd", "priv/couchjs_mainjs", UnixMainjsSrc, [{env, FreeBSDEnv}]}, - {"freebsd", "priv/couchjs_coffee", UnixCoffeeSrc, [{env, FreeBSDEnv}]}, + {"(linux|darwin|freebsd)", "priv/couchjs_mainjs", UnixMainjsSrc, [{env, UnixEnv}]}, + {"(linux|darwin|freebsd)", "priv/couchjs_coffee", UnixCoffeeSrc, [{env, UnixEnv}]}, {"win32", "priv/couchjs_mainjs.exe", WindowsMainjsSrc, [{env, WindowsEnv}]}, {"win32", "priv/couchjs_coffee.exe", WindowsCoffeeSrc, [{env, WindowsEnv}]} ]. diff --git a/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl b/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl index 0d7b233de4..91385010c3 100644 --- a/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl +++ b/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl @@ -23,6 +23,7 @@ shards/2, db_opened/2, doc_id/3, + doc_fdi/3, doc/3, db_closing/2 ]). @@ -149,7 +150,7 @@ db_opened(#st{} = St, Db) -> #st{max_docs = MaxDocs, max_step = MaxStep} = St, {ok, DocTotal} = couch_db:get_doc_count(Db), Step = min(MaxStep, max(1, DocTotal div MaxDocs)), - {ok, St#st{doc_cnt = 0, doc_step = Step, docs = []}}. + {0, [], St#st{doc_cnt = 0, doc_step = Step, docs = []}}. doc_id(#st{} = St, <>, _Db) -> {skip, St}; @@ -162,6 +163,12 @@ doc_id(#st{doc_cnt = C, doc_step = S} = St, _DocId, _Db) when C rem S /= 0 -> doc_id(#st{doc_cnt = C} = St, _DocId, _Db) -> {ok, St#st{doc_cnt = C + 1}}. +doc_fdi(#st{} = St, #full_doc_info{deleted = true}, _Db) -> + % Skip deleted; don't even open the doc body + {stop, St}; +doc_fdi(#st{} = St, #full_doc_info{}, _Db) -> + {ok, St}. + doc(#st{} = St, Db, #doc{id = DocId} = Doc) -> #st{sid = SId} = St, JsonDoc = couch_query_servers:json_doc(Doc), diff --git a/src/couch_quickjs/update_and_apply_patches.sh b/src/couch_quickjs/update.sh similarity index 88% rename from src/couch_quickjs/update_and_apply_patches.sh rename to src/couch_quickjs/update.sh index 92fbb63550..87454c9c3d 100755 --- a/src/couch_quickjs/update_and_apply_patches.sh +++ b/src/couch_quickjs/update.sh @@ -7,9 +7,6 @@ set -e # This is the main branch of the github mirror URL=https://github.com/bellard/quickjs/archive/refs/heads/master.zip # -# The other alternatives: -# https://github.com/quickjs-ng/quickjs/commits/master/ - echo echo " * backup quickjs to quickjs.bak" @@ -50,6 +47,9 @@ echo " * removing quickjs.bak" rm -rf quickjs.bak echo -# Example how to generate patches: +# Example how to update patches themselves: # +# Run +# ./update_patches.sh +# OR manually run after cloning and unzipping master.zip from quickjs: # diff -u quickjs-master/quickjs.c quickjs/quickjs.c > patches/01-spidermonkey-185-mode.patch diff --git a/src/couch_quickjs/update_patches.sh b/src/couch_quickjs/update_patches.sh new file mode 100755 index 0000000000..f56a0e0c06 --- /dev/null +++ b/src/couch_quickjs/update_patches.sh @@ -0,0 +1,28 @@ +# Update patches +# +# Call this script after using update_and_apply_patches.sh to adjust +# the patches themselves. Sometimes line offsets drift and so this takes +# a new diff from the master QuickJS vs current source tree and regenerates +# the patch. + +set -e + +URL=https://github.com/bellard/quickjs/archive/refs/heads/master.zip + +echo " * wget ${URL}" +rm -rf master.zip quickjs-master +wget -q ${URL} +echo " * unzip master.zip to quickjs-master" +unzip -q -o master.zip + +set +e + +echo " * updating 01-spidermonkey-185-mode.patch" +diff -u quickjs-master/quickjs.c quickjs/quickjs.c > patches/01-spidermonkey-185-mode.patch + +echo " * updating 02-test262-errors.patch" +diff -u quickjs-master/test262_errors.txt quickjs/test262_errors.txt> patches/02-test262-errors.patch +set -e + +echo " * cleaning up" +rm -rf master.zip quickjs-master diff --git a/src/couch_replicator/src/couch_replicator_api_wrap.erl b/src/couch_replicator/src/couch_replicator_api_wrap.erl index ce22ceb64b..9364757d6c 100644 --- a/src/couch_replicator/src/couch_replicator_api_wrap.erl +++ b/src/couch_replicator/src/couch_replicator_api_wrap.erl @@ -115,10 +115,10 @@ db_open(#httpdb{} = Db1, Create, CreateParams) -> throw(Error); error:Error -> db_close(Db), - erlang:error(Error); + error(Error); exit:Error -> db_close(Db), - erlang:exit(Error) + exit(Error) end. db_close(#httpdb{httpc_pool = Pool} = HttpDb) -> @@ -300,7 +300,7 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> % hammer approach to making sure it releases % that connection back to the pool. spawn(fun() -> - Ref = erlang:monitor(process, Self), + Ref = monitor(process, Self), receive {'DOWN', Ref, process, Self, normal} -> exit(Streamer, {streamer_parent_died, Self}); @@ -352,7 +352,7 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) -> NewRetries = Retries - 1, case NewRetries > 0 of true -> - Wait = 2 * erlang:min(Wait0 * 2, ?MAX_WAIT), + Wait = 2 * min(Wait0 * 2, ?MAX_WAIT), LogRetryMsg = "Retrying GET to ~s in ~p seconds due to error ~w", couch_log:notice(LogRetryMsg, [Url, Wait / 1000, error_reason(Else)]), ok = timer:sleep(Wait), @@ -532,7 +532,7 @@ changes_since( UserFun, Options ) -> - Timeout = erlang:max(1000, InactiveTimeout div 3), + Timeout = max(1000, InactiveTimeout div 3), EncodedSeq = couch_replicator_utils:seq_encode(StartSeq), BaseQArgs = case get_value(continuous, Options, false) of @@ -820,7 +820,7 @@ run_user_fun(UserFun, Arg, UserAcc, OldRef) -> end), receive {started_open_doc_revs, NewRef} -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), exit(Pid, kill), restart_remote_open_doc_revs(OldRef, NewRef); {'DOWN', Ref, process, Pid, {exit_ok, Ret}} -> @@ -828,9 +828,9 @@ run_user_fun(UserFun, Arg, UserAcc, OldRef) -> {'DOWN', Ref, process, Pid, {exit_throw, Reason}} -> throw(Reason); {'DOWN', Ref, process, Pid, {exit_error, Reason}} -> - erlang:error(Reason); + error(Reason); {'DOWN', Ref, process, Pid, {exit_exit, Reason}} -> - erlang:exit(Reason) + exit(Reason) end. restart_remote_open_doc_revs(Ref, NewRef) -> @@ -844,7 +844,7 @@ restart_remote_open_doc_revs(Ref, NewRef) -> {headers, Ref, _} -> restart_remote_open_doc_revs(Ref, NewRef) after 0 -> - erlang:error({restart_open_doc_revs, NewRef}) + error({restart_open_doc_revs, NewRef}) end. remote_open_doc_revs_streamer_start(Parent) -> diff --git a/src/couch_replicator/src/couch_replicator_auth_session.erl b/src/couch_replicator/src/couch_replicator_auth_session.erl index 182e3cc865..316ce015ee 100644 --- a/src/couch_replicator/src/couch_replicator_auth_session.erl +++ b/src/couch_replicator/src/couch_replicator_auth_session.erl @@ -62,7 +62,7 @@ handle_call/3, handle_cast/2, handle_info/2, - format_status/2 + format_status/1 ]). -include_lib("ibrowse/include/ibrowse.hrl"). @@ -154,13 +154,21 @@ handle_info(Msg, State) -> couch_log:error("~p : Received un-expected message ~p", [?MODULE, Msg]), {noreply, State}. -format_status(_Opt, [_PDict, State]) -> - [ - {epoch, State#state.epoch}, - {user, State#state.user}, - {session_url, State#state.session_url}, - {refresh_tstamp, State#state.refresh_tstamp} - ]. +format_status(Status) -> + maps:map( + fun + (state, State) -> + #{ + epoch => State#state.epoch, + user => State#state.user, + session_url => State#state.session_url, + refresh_tstamp => State#state.refresh_tstamp + }; + (_, Value) -> + Value + end, + Status + ). %% Private helper functions @@ -336,7 +344,7 @@ stop_worker_if_server_requested(ResultHeaders0, Worker) -> ResultHeaders = mochiweb_headers:make(ResultHeaders0), case mochiweb_headers:get_value("Connection", ResultHeaders) of "close" -> - Ref = erlang:monitor(process, Worker), + Ref = monitor(process, Worker), ibrowse_http_client:stop(Worker), receive {'DOWN', Ref, _, _, _} -> @@ -379,7 +387,7 @@ parse_cookie(Headers) -> {error, cookie_not_found}; [_ | _] = Cookies -> case get_auth_session_cookies_and_age(Cookies) of - [] -> {error, cookie_format_invalid}; + [] -> {error, cookie_not_found}; [{Cookie, MaxAge} | _] -> {ok, MaxAge, Cookie} end end. @@ -800,4 +808,14 @@ get_auth_session_cookies_and_age_test() -> ]) ). +parse_cookie_test() -> + NotFound = {error, cookie_not_found}, + ?assertEqual(NotFound, parse_cookie([])), + ?assertEqual(NotFound, parse_cookie([{"abc", "def"}])), + ?assertEqual(NotFound, parse_cookie([{"set-cookiee", "c=v"}])), + ?assertEqual(NotFound, parse_cookie([{"set-cookie", ""}])), + ?assertEqual(NotFound, parse_cookie([{"Set-cOokie", "c=v"}])), + ?assertEqual({ok, undefined, "x"}, parse_cookie([{"set-cookie", "authsession=x"}])), + ?assertEqual({ok, 4, "x"}, parse_cookie([{"set-cookie", "authsession=x; max-age=4"}])). + -endif. diff --git a/src/couch_replicator/src/couch_replicator_connection.erl b/src/couch_replicator/src/couch_replicator_connection.erl index 978962bd76..110c792bb8 100644 --- a/src/couch_replicator/src/couch_replicator_connection.erl +++ b/src/couch_replicator/src/couch_replicator_connection.erl @@ -77,6 +77,9 @@ init([]) -> {inactivity_timeout, Interval}, {worker_trap_exits, false} ]), + % Try loading all the OS CA certs to give users an early indication in the + % logs if there is an error. + couch_replicator_utils:cacert_get(), {ok, #state{close_interval = Interval, timer = Timer}}. acquire(Url) -> diff --git a/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl b/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl index b9468080c6..5a62b3e06d 100644 --- a/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl +++ b/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl @@ -66,7 +66,7 @@ worker_fun(Id, Rep, WaitSec, WRef) -> {'DOWN', Ref, _, Pid, Result} -> exit(#doc_worker_result{id = Id, wref = WRef, result = Result}) after ?WORKER_TIMEOUT_MSEC -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), exit(Pid, kill), {DbName, DocId} = Id, TimeoutSec = round(?WORKER_TIMEOUT_MSEC / 1000), diff --git a/src/couch_replicator/src/couch_replicator_docs.erl b/src/couch_replicator/src/couch_replicator_docs.erl index 6b324c97a4..d28ae908cd 100644 --- a/src/couch_replicator/src/couch_replicator_docs.erl +++ b/src/couch_replicator/src/couch_replicator_docs.erl @@ -159,7 +159,7 @@ update_rep_doc(RepDbName, RepDocId, KVs, Wait) when is_binary(RepDocId) -> throw:conflict -> Msg = "Conflict when updating replication doc `~s`. Retrying.", couch_log:error(Msg, [RepDocId]), - ok = timer:sleep(rand:uniform(erlang:min(128, Wait)) * 100), + ok = timer:sleep(rand:uniform(min(128, Wait)) * 100), update_rep_doc(RepDbName, RepDocId, KVs, Wait * 2) end; update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs, _Try) -> diff --git a/src/couch_replicator/src/couch_replicator_fabric.erl b/src/couch_replicator/src/couch_replicator_fabric.erl index cb441fea71..9f6d945be3 100644 --- a/src/couch_replicator/src/couch_replicator_fabric.erl +++ b/src/couch_replicator/src/couch_replicator_fabric.erl @@ -121,7 +121,7 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> offset = Offset }}; false -> - FinalOffset = erlang:min(Total, Offset + State#collector.skip), + FinalOffset = min(Total, Offset + State#collector.skip), Meta = [{total, Total}, {offset, FinalOffset}], {Go, Acc} = Callback({meta, Meta}, AccIn), {Go, State#collector{ diff --git a/src/couch_replicator/src/couch_replicator_fabric_rpc.erl b/src/couch_replicator/src/couch_replicator_fabric_rpc.erl index 1a3582928e..680a758b88 100644 --- a/src/couch_replicator/src/couch_replicator_fabric_rpc.erl +++ b/src/couch_replicator/src/couch_replicator_fabric_rpc.erl @@ -106,7 +106,7 @@ docs_test_() -> fun() -> ok end, fun(_) -> ok end, [ - ?TDEF_FE(t_docs) + ?TDEF_FE(t_docs, 15) ] }. @@ -140,10 +140,10 @@ docs_cb_test_() -> end, fun(_) -> meck:unload() end, [ - ?TDEF_FE(t_docs_cb_meta), - ?TDEF_FE(t_docs_cb_row_skip), - ?TDEF_FE(t_docs_cb_row), - ?TDEF_FE(t_docs_cb_complete) + ?TDEF_FE(t_docs_cb_meta, 15), + ?TDEF_FE(t_docs_cb_row_skip, 15), + ?TDEF_FE(t_docs_cb_row, 15), + ?TDEF_FE(t_docs_cb_complete, 15) ] }. diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl b/src/couch_replicator/src/couch_replicator_httpc.erl index cd5e4d75df..fe81e65ea0 100644 --- a/src/couch_replicator/src/couch_replicator_httpc.erl +++ b/src/couch_replicator/src/couch_replicator_httpc.erl @@ -38,7 +38,7 @@ % consuming the request. This threshold gives us confidence we'll % continue to properly close changes feeds while avoiding any case % where we may end up processing an unbounded number of messages. --define(MAX_DISCARDED_MESSAGES, 16). +-define(MAX_DISCARDED_MESSAGES, 100). setup(Db) -> #httpdb{ @@ -154,7 +154,7 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) -> %% {error, req_timedout} error. While in reality is not really a timeout, just %% a race condition. stop_and_release_worker(Pool, Worker) -> - Ref = erlang:monitor(process, Worker), + Ref = monitor(process, Worker), ibrowse_http_client:stop(Worker), receive {'DOWN', Ref, _, _, _} -> @@ -230,7 +230,7 @@ process_stream_response(ReqId, Worker, HttpDb, Params, Callback) -> Ok =:= 413 -> put(?STOP_HTTP_WORKER, stop); true -> ok end, - ibrowse:stream_next(ReqId), + ok = ibrowse:stream_next(ReqId), try Ret = Callback(Ok, Headers, StreamDataFun), Ret @@ -297,12 +297,16 @@ clean_mailbox({ibrowse_req_id, ReqId}, Count) when Count > 0 -> discard_message(ReqId, Worker, Count); false -> put(?STREAM_STATUS, ended), - ok + % Worker is not alive but we may still messages + % in the mailbox from it so recurse to clean them + clean_mailbox({ibrowse_req_id, ReqId}, Count) end; Status when Status == init; Status == ended -> receive {ibrowse_async_response, ReqId, _} -> clean_mailbox({ibrowse_req_id, ReqId}, Count - 1); + {ibrowse_async_response_timeout, ReqId} -> + clean_mailbox({ibrowse_req_id, ReqId}, Count - 1); {ibrowse_async_response_end, ReqId} -> put(?STREAM_STATUS, ended), ok @@ -314,16 +318,26 @@ clean_mailbox(_, Count) when Count > 0 -> ok. discard_message(ReqId, Worker, Count) -> - ibrowse:stream_next(ReqId), - receive - {ibrowse_async_response, ReqId, _} -> - clean_mailbox({ibrowse_req_id, ReqId}, Count - 1); - {ibrowse_async_response_end, ReqId} -> + case ibrowse:stream_next(ReqId) of + ok -> + receive + {ibrowse_async_response, ReqId, _} -> + clean_mailbox({ibrowse_req_id, ReqId}, Count - 1); + {ibrowse_async_response_timeout, ReqId} -> + clean_mailbox({ibrowse_req_id, ReqId}, Count - 1); + {ibrowse_async_response_end, ReqId} -> + put(?STREAM_STATUS, ended), + ok + after 30000 -> + exit(Worker, {timeout, ibrowse_stream_cleanup}), + exit({timeout, ibrowse_stream_cleanup}) + end; + {error, unknown_req_id} -> + % The stream is being torn down so expect to handle stream ids not + % being found. We don't want to sleep for 30 seconds and then exit. + % Just clean any left-over mailbox messages and move on. put(?STREAM_STATUS, ended), - ok - after 30000 -> - exit(Worker, {timeout, ibrowse_stream_cleanup}), - exit({timeout, ibrowse_stream_cleanup}) + clean_mailbox({ibrowse_req_id, ReqId}, Count) end. -spec maybe_retry(any(), pid(), #httpdb{}, list()) -> no_return(). @@ -341,7 +355,7 @@ maybe_retry( false -> ok = timer:sleep(Wait), log_retry_error(Params, HttpDb, Wait, Error), - Wait2 = erlang:min(Wait * 2, ?MAX_WAIT), + Wait2 = min(Wait * 2, ?MAX_WAIT), HttpDb1 = HttpDb#httpdb{retries = Retries - 1, wait = Wait2}, HttpDb2 = update_first_error_timestamp(HttpDb1), throw({retry, HttpDb2, Params}) @@ -403,7 +417,7 @@ error_cause(Cause) -> stream_data_self(#httpdb{timeout = T} = HttpDb, Params, Worker, ReqId, Cb) -> case accumulate_messages(ReqId, [], T + 500) of {Data, ibrowse_async_response} -> - ibrowse:stream_next(ReqId), + ok = ibrowse:stream_next(ReqId), {Data, fun() -> stream_data_self(HttpDb, Params, Worker, ReqId, Cb) end}; {Data, ibrowse_async_response_end} -> put(?STREAM_STATUS, ended), @@ -540,4 +554,125 @@ merge_headers_test() -> ?assertEqual([{"a", "y"}], merge_headers([{"A", "z"}, {"a", "y"}], [])), ?assertEqual([{"a", "y"}], merge_headers([], [{"A", "z"}, {"a", "y"}])). +clean_mailbox_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_clean_noop), + ?TDEF_FE(t_clean_skip_other_messages), + ?TDEF_FE(t_clean_when_init), + ?TDEF_FE(t_clean_when_ended), + ?TDEF_FE(t_clean_when_streaming), + ?TDEF_FE(t_clean_when_streaming_dead_pid), + ?TDEF_FE(t_other_req_id_is_ignored) + ] + }. + +setup() -> + meck:new(ibrowse), + meck:expect(ibrowse, stream_next, 1, ok), + ok. + +teardown(_) -> + meck:unload(). + +t_clean_noop(_) -> + ReqId = make_ref(), + ?assertEqual(ok, clean_mailbox(random_junk)), + meck:expect(ibrowse, stream_next, 1, {error, unknown_req_id}), + set_stream_status({streaming, self()}), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId})), + set_stream_status(init), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId})), + set_stream_status(ended), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId})). + +t_clean_skip_other_messages(_) -> + set_stream_status(init), + self() ! other_message, + ReqId = make_ref(), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId})), + ?assertEqual([other_message], flush()). + +t_clean_when_init(_) -> + set_stream_status(init), + ReqId = make_ref(), + add_all_message_types(ReqId), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId})), + ?assertEqual([], flush()), + ?assertEqual(ended, stream_status()). + +t_clean_when_ended(_) -> + set_stream_status(init), + ReqId = make_ref(), + add_all_message_types(ReqId), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId})), + ?assertEqual([], flush()), + ?assertEqual(ended, stream_status()). + +t_clean_when_streaming(_) -> + set_stream_status({streaming, self()}), + ReqId = make_ref(), + add_all_message_types(ReqId), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId})), + ?assertEqual([], flush()), + ?assertEqual(ended, stream_status()). + +t_clean_when_streaming_dead_pid(_) -> + {Pid, Ref} = spawn_monitor(fun() -> ok end), + receive + {'DOWN', Ref, _, _, _} -> ok + end, + set_stream_status({streaming, Pid}), + ReqId = make_ref(), + add_all_message_types(ReqId), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId})), + ?assertEqual([], flush()), + ?assertEqual(ended, stream_status()). + +t_other_req_id_is_ignored(_) -> + set_stream_status({streaming, self()}), + ReqId1 = make_ref(), + add_all_message_types(ReqId1), + ReqId2 = make_ref(), + add_all_message_types(ReqId2), + ?assertEqual(ok, clean_mailbox({ibrowse_req_id, ReqId1})), + ?assertEqual( + [ + {ibrowse_async_response, ReqId2, foo}, + {ibrowse_async_response_timeout, ReqId2}, + {ibrowse_async_response_end, ReqId2} + ], + flush() + ), + ?assertEqual(ended, stream_status()). + +stream_status() -> + get(?STREAM_STATUS). + +set_stream_status(Status) -> + put(?STREAM_STATUS, Status). + +add_all_message_types(ReqId) -> + Messages = [ + {ibrowse_async_response, ReqId, foo}, + {ibrowse_async_response_timeout, ReqId}, + {ibrowse_async_response_end, ReqId} + ], + [self() ! M || M <- Messages], + ok. + +flush() -> + flush([]). + +flush(Acc) -> + receive + Msg -> + flush([Msg | Acc]) + after 0 -> + lists:reverse(Acc) + end. + -endif. diff --git a/src/couch_replicator/src/couch_replicator_httpc_pool.erl b/src/couch_replicator/src/couch_replicator_httpc_pool.erl index fb15dcea16..3c2f6edfa2 100644 --- a/src/couch_replicator/src/couch_replicator_httpc_pool.erl +++ b/src/couch_replicator/src/couch_replicator_httpc_pool.erl @@ -19,7 +19,7 @@ % gen_server API -export([init/1, handle_call/3, handle_info/2, handle_cast/2]). --export([format_status/2]). +-export([format_status/1]). -include_lib("couch/include/couch_db.hrl"). @@ -135,27 +135,31 @@ handle_info({'DOWN', Ref, process, _, _}, #state{callers = Callers} = State) -> {noreply, State} end. -format_status(_Opt, [_PDict, State]) -> - #state{ - url = Url, - proxy_url = ProxyUrl - } = State, - [ - {data, [ - {"State", State#state{ - url = couch_util:url_strip_password(Url), - proxy_url = couch_util:url_strip_password(ProxyUrl) - }} - ]} - ]. +format_status(Status) -> + maps:map( + fun + (state, State) -> + #state{ + url = Url, + proxy_url = ProxyUrl + } = State, + State#state{ + url = couch_util:url_strip_password(Url), + proxy_url = couch_util:url_strip_password(ProxyUrl) + }; + (_, Value) -> + Value + end, + Status + ). monitor_client(Callers, Worker, {ClientPid, _}) -> - [{Worker, erlang:monitor(process, ClientPid)} | Callers]. + [{Worker, monitor(process, ClientPid)} | Callers]. demonitor_client(Callers, Worker) -> case lists:keysearch(Worker, 1, Callers) of {value, {Worker, MonRef}} -> - erlang:demonitor(MonRef, [flush]), + demonitor(MonRef, [flush]), lists:keydelete(Worker, 1, Callers); false -> Callers @@ -196,13 +200,16 @@ release_worker_internal(Worker, State) -> format_status_test_() -> ?_test(begin - State = #state{ - url = "https://username1:password1@$ACCOUNT2.cloudant.com/db", - proxy_url = "https://username2:password2@proxy.thing.com:8080/" + Status = #{ + state => + #state{ + url = "https://username1:password1@$ACCOUNT2.cloudant.com/db", + proxy_url = "https://username2:password2@proxy.thing.com:8080/" + } }, - [{data, [{"State", ScrubbedN}]}] = format_status(normal, [[], State]), - ?assertEqual("https://username1:*****@$ACCOUNT2.cloudant.com/db", ScrubbedN#state.url), - ?assertEqual("https://username2:*****@proxy.thing.com:8080/", ScrubbedN#state.proxy_url), + #{state := State} = format_status(Status), + ?assertEqual("https://username1:*****@$ACCOUNT2.cloudant.com/db", State#state.url), + ?assertEqual("https://username2:*****@proxy.thing.com:8080/", State#state.proxy_url), ok end). diff --git a/src/couch_replicator/src/couch_replicator_notifier.erl b/src/couch_replicator/src/couch_replicator_notifier.erl index 21c6d5a25c..4b33e0d06a 100644 --- a/src/couch_replicator/src/couch_replicator_notifier.erl +++ b/src/couch_replicator/src/couch_replicator_notifier.erl @@ -14,6 +14,8 @@ -behaviour(gen_event). +-define(NAME, couch_replication). + % public API -export([start_link/1, stop/1, notify/1]). @@ -21,17 +23,20 @@ -export([init/1]). -export([handle_event/2, handle_call/2, handle_info/2]). --include_lib("couch/include/couch_db.hrl"). - start_link(FunAcc) -> - couch_event_sup:start_link( - couch_replication, - {couch_replicator_notifier, make_ref()}, - FunAcc - ). + couch_event_sup:start_link(?NAME, {?MODULE, make_ref()}, FunAcc). notify(Event) -> - gen_event:notify(couch_replication, Event). + try + gen_event:notify(?NAME, Event) + catch + _:_ -> + % It's possible some jobs may remain around after the notification + % service had shut down or crashed. Avoid making a mess in the logs + % and just ignore that. At that point nobody will notice the + % notification anyway. + ok + end. stop(Pid) -> couch_event_sup:stop(Pid). @@ -51,3 +56,12 @@ handle_call(_Msg, State) -> handle_info(_Msg, State) -> {ok, State}. + +-ifdef(TEST). + +-include_lib("couch/include/couch_eunit.hrl"). + +couch_replicator_notify_when_stopped_test() -> + ?assertEqual(ok, notify({stopped, foo})). + +-endif. diff --git a/src/couch_replicator/src/couch_replicator_parse.erl b/src/couch_replicator/src/couch_replicator_parse.erl index b72e1f5765..5f8437992b 100644 --- a/src/couch_replicator/src/couch_replicator_parse.erl +++ b/src/couch_replicator/src/couch_replicator_parse.erl @@ -488,15 +488,28 @@ ssl_params(Url) -> -spec ssl_verify_options(true | false) -> [_]. ssl_verify_options(true) -> - CAFile = cfg("ssl_trusted_certificates_file"), - [ - {verify, verify_peer}, - {customize_hostname_check, [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]}, - {cacertfile, CAFile} - ]; + % https://security.erlef.org/secure_coding_and_deployment_hardening/ssl.html + ssl_ca_cert_opts() ++ + [ + {verify, verify_peer}, + {customize_hostname_check, [ + {match_fun, public_key:pkix_verify_hostname_match_fun(https)} + ]} + ]; ssl_verify_options(false) -> [{verify, verify_none}]. +ssl_ca_cert_opts() -> + % Try to use the CA cert file from config first, and if not specified, use + % the CA certificates from the OS. If those can't be loaded either, then + % crash: cacerts_get/0 raises an error in that case and we do not catch it. + case cfg("ssl_trusted_certificates_file") of + undefined -> + [{cacerts, public_key:cacerts_get()}]; + CAFile when is_list(CAFile) -> + [{cacertfile, CAFile}] + end. + get_value(Key, Props) -> couch_util:get_value(Key, Props). diff --git a/src/couch_replicator/src/couch_replicator_pg.erl b/src/couch_replicator/src/couch_replicator_pg.erl index 25937ec15f..5f4f3bba78 100644 --- a/src/couch_replicator/src/couch_replicator_pg.erl +++ b/src/couch_replicator/src/couch_replicator_pg.erl @@ -47,7 +47,14 @@ join({_, _} = RepId, Pid) when is_pid(Pid) -> % quicker. % leave({_, _} = RepId, Pid) when is_pid(Pid) -> - pg:leave(?MODULE, id(RepId), Pid). + try + pg:leave(?MODULE, id(RepId), Pid) + catch + _:_ -> + ok + % If this is called during shutdown the pg gen_server might be + % gone. So we avoid blocking on it or making a mess in the logs + end. % Determine if a replication job should start on a particular node. If it % should, return `yes`, otherwise return `{no, OtherPid}`. `OtherPid` is @@ -150,4 +157,9 @@ t_should_run(_) -> ok = join(RepId, InitPid), ?assertEqual({no, InitPid}, should_run(RepId, Pid)). +couch_replicator_pg_test_leave_when_stopped_test() -> + RepId = {"a", "+b"}, + Pid = self(), + ?assertEqual(ok, leave(RepId, Pid)). + -endif. diff --git a/src/couch_replicator/src/couch_replicator_rate_limiter.erl b/src/couch_replicator/src/couch_replicator_rate_limiter.erl index c98f98d614..ccffd90114 100644 --- a/src/couch_replicator/src/couch_replicator_rate_limiter.erl +++ b/src/couch_replicator/src/couch_replicator_rate_limiter.erl @@ -170,9 +170,9 @@ update_failure(_Key, Interval, Timestamp, Now) when % Ignore too frequent updates. Interval; update_failure(Key, Interval, _Timestamp, Now) -> - Interval1 = erlang:max(Interval, ?BASE_INTERVAL), + Interval1 = max(Interval, ?BASE_INTERVAL), Interval2 = round(Interval1 * ?BACKOFF_FACTOR), - Interval3 = erlang:min(Interval2, ?MAX_INTERVAL), + Interval3 = min(Interval2, ?MAX_INTERVAL), insert(Key, Interval3, Now). -spec insert(any(), interval(), msec()) -> interval(). @@ -195,7 +195,7 @@ interval_and_timestamp(Key) -> -spec time_decay(msec(), interval()) -> interval(). time_decay(Dt, Interval) when Dt > ?TIME_DECAY_THRESHOLD -> DecayedInterval = Interval - ?TIME_DECAY_FACTOR * Dt, - erlang:max(round(DecayedInterval), 0); + max(round(DecayedInterval), 0); time_decay(_Dt, Interval) -> Interval. diff --git a/src/couch_replicator/src/couch_replicator_scheduler.erl b/src/couch_replicator/src/couch_replicator_scheduler.erl index a3aa7601d2..aabd7febde 100644 --- a/src/couch_replicator/src/couch_replicator_scheduler.erl +++ b/src/couch_replicator/src/couch_replicator_scheduler.erl @@ -25,7 +25,7 @@ handle_call/3, handle_info/2, handle_cast/2, - format_status/2 + format_status/1 ]). -export([ @@ -71,6 +71,13 @@ -define(DEFAULT_MAX_HISTORY, 20). -define(DEFAULT_SCHEDULER_INTERVAL, 60000). +% Worker children get a default 5 second shutdown timeout, so pick a value just +% a bit less than that: 4.5 seconds. In couch_replicator_sup our scheduler +% worker doesn't specify the timeout, so it up picks ups the OTP default of 5 +% seconds https://www.erlang.org/doc/system/sup_princ.html#child-specification +% +-define(TERMINATE_SHUTDOWN_TIME, 4500). + -record(state, { interval = ?DEFAULT_SCHEDULER_INTERVAL, timer, @@ -346,15 +353,24 @@ handle_info(_, State) -> {noreply, State}. terminate(_Reason, _State) -> + stop_clear_all_jobs(?TERMINATE_SHUTDOWN_TIME), couch_replicator_share:clear(), ok. -format_status(_Opt, [_PDict, State]) -> - [ - {max_jobs, State#state.max_jobs}, - {running_jobs, running_job_count()}, - {pending_jobs, pending_job_count()} - ]. +format_status(Status) -> + maps:map( + fun + (state, State) -> + #{ + max_jobs => State#state.max_jobs, + running_jobs => running_job_count(), + pending_jobs => pending_job_count() + }; + (_, Value) -> + Value + end, + Status + ). %% config listener functions @@ -387,6 +403,33 @@ handle_config_terminate(_, _, _) -> %% Private functions +% Stop jobs in parallel +% +stop_clear_all_jobs(TimeLeftMSec) -> + ShutdownFun = fun(#job{pid = Pid}) -> + Pid ! shutdown, + monitor(process, Pid) + end, + Refs = lists:map(ShutdownFun, running_jobs()), + ets:delete_all_objects(?MODULE), + wait_jobs_stop(TimeLeftMSec, Refs). + +wait_jobs_stop(_, []) -> + ok; +wait_jobs_stop(TimeLeftMSec, _) when TimeLeftMSec =< 0 -> + % If some survive a bit longer we let them finish checkpointing. + timeout; +wait_jobs_stop(TimeLeftMsec, [Ref | Refs]) -> + T0 = erlang:monotonic_time(), + receive + {'DOWN', Ref, _, _, _} -> + Dt = erlang:monotonic_time() - T0, + DtMSec = erlang:convert_time_unit(Dt, native, millisecond), + wait_jobs_stop(TimeLeftMsec - DtMSec, Refs) + after TimeLeftMsec -> + ok + end. + % Handle crashed jobs. Handling differs between transient and permanent jobs. % Transient jobs are those posted to the _replicate endpoint. They don't have a % db associated with them. When those jobs crash, they are not restarted. That @@ -594,7 +637,7 @@ backoff_micros(CrashCount) -> % exponent in Base * 2 ^ CrashCount to achieve an exponential backoff % doubling every consecutive failure, starting with the base value of % ?BACKOFF_INTERVAL_MICROS. - BackoffExp = erlang:min(CrashCount - 1, ?MAX_BACKOFF_EXPONENT), + BackoffExp = min(CrashCount - 1, ?MAX_BACKOFF_EXPONENT), (1 bsl BackoffExp) * ?BACKOFF_INTERVAL_MICROS. -spec add_job_int(#job{}) -> boolean(). @@ -895,7 +938,7 @@ update_running_jobs_stats(StatsPid) when is_pid(StatsPid) -> ok. start_stats_updater() -> - erlang:spawn_link(?MODULE, stats_updater_loop, [undefined]). + spawn_link(?MODULE, stats_updater_loop, [undefined]). stats_updater_loop(Timer) -> receive @@ -908,7 +951,7 @@ stats_updater_loop(Timer) -> ok = stats_updater_refresh(), ?MODULE:stats_updater_loop(undefined); Else -> - erlang:exit({stats_updater_bad_msg, Else}) + exit({stats_updater_bad_msg, Else}) end. -spec stats_updater_refresh() -> ok. @@ -946,7 +989,7 @@ existing_replication(#rep{} = NewRep) -> -ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_eunit.hrl"). backoff_micros_test_() -> BaseInterval = ?BACKOFF_INTERVAL_MICROS, @@ -1045,464 +1088,412 @@ scheduler_test_() -> fun setup/0, fun teardown/1, [ - t_pending_jobs_simple(), - t_pending_jobs_skip_crashed(), - t_pending_jobs_skip_running(), - t_one_job_starts(), - t_no_jobs_start_if_max_is_0(), - t_one_job_starts_if_max_is_1(), - t_max_churn_does_not_throttle_initial_start(), - t_excess_oneshot_only_jobs(), - t_excess_continuous_only_jobs(), - t_excess_prefer_continuous_first(), - t_stop_oldest_first(), - t_start_oldest_first(), - t_jobs_churn_even_if_not_all_max_jobs_are_running(), - t_jobs_dont_churn_if_there_are_available_running_slots(), - t_start_only_pending_jobs_do_not_churn_existing_ones(), - t_dont_stop_if_nothing_pending(), - t_max_churn_limits_number_of_rotated_jobs(), - t_existing_jobs(), - t_if_pending_less_than_running_start_all_pending(), - t_running_less_than_pending_swap_all_running(), - t_oneshot_dont_get_rotated(), - t_rotate_continuous_only_if_mixed(), - t_oneshot_dont_get_starting_priority(), - t_oneshot_will_hog_the_scheduler(), - t_if_excess_is_trimmed_rotation_still_happens(), - t_if_transient_job_crashes_it_gets_removed(), - t_if_permanent_job_crashes_it_stays_in_ets(), - t_job_summary_running(), - t_job_summary_pending(), - t_job_summary_crashing_once(), - t_job_summary_crashing_many_times(), - t_job_summary_proxy_fields() + ?TDEF_FE(t_pending_jobs_simple), + ?TDEF_FE(t_pending_jobs_skip_crashed), + ?TDEF_FE(t_pending_jobs_skip_running), + ?TDEF_FE(t_one_job_starts), + ?TDEF_FE(t_no_jobs_start_if_max_is_0), + ?TDEF_FE(t_one_job_starts_if_max_is_1), + ?TDEF_FE(t_max_churn_does_not_throttle_initial_start), + ?TDEF_FE(t_excess_oneshot_only_jobs), + ?TDEF_FE(t_excess_continuous_only_jobs), + ?TDEF_FE(t_excess_prefer_continuous_first), + ?TDEF_FE(t_stop_oldest_first), + ?TDEF_FE(t_start_oldest_first), + ?TDEF_FE(t_jobs_churn_even_if_not_all_max_jobs_are_running), + ?TDEF_FE(t_jobs_dont_churn_if_there_are_available_running_slots), + ?TDEF_FE(t_start_only_pending_jobs_do_not_churn_existing_ones), + ?TDEF_FE(t_dont_stop_if_nothing_pending), + ?TDEF_FE(t_max_churn_limits_number_of_rotated_jobs), + ?TDEF_FE(t_existing_jobs), + ?TDEF_FE(t_if_pending_less_than_running_start_all_pending), + ?TDEF_FE(t_running_less_than_pending_swap_all_running), + ?TDEF_FE(t_oneshot_dont_get_rotated), + ?TDEF_FE(t_rotate_continuous_only_if_mixed), + ?TDEF_FE(t_oneshot_dont_get_starting_priority), + ?TDEF_FE(t_oneshot_will_hog_the_scheduler), + ?TDEF_FE(t_if_excess_is_trimmed_rotation_still_happens), + ?TDEF_FE(t_if_transient_job_crashes_it_gets_removed), + ?TDEF_FE(t_if_permanent_job_crashes_it_stays_in_ets), + ?TDEF_FE(t_stop_all_stops_jobs), + ?TDEF_FE(t_job_summary_running), + ?TDEF_FE(t_job_summary_pending), + ?TDEF_FE(t_job_summary_crashing_once), + ?TDEF_FE(t_job_summary_crashing_many_times), + ?TDEF_FE(t_job_summary_proxy_fields) ] } }. -t_pending_jobs_simple() -> - ?_test(begin - Job1 = oneshot(1), - Job2 = oneshot(2), - setup_jobs([Job2, Job1]), - ?assertEqual([], pending_jobs(0)), - ?assertEqual([Job1], pending_jobs(1)), - ?assertEqual([Job1, Job2], pending_jobs(2)), - ?assertEqual([Job1, Job2], pending_jobs(3)) - end). - -t_pending_jobs_skip_crashed() -> - ?_test(begin - Job = oneshot(1), - Ts = os:timestamp(), - History = [crashed(Ts), started(Ts) | Job#job.history], - Job1 = Job#job{history = History}, - Job2 = oneshot(2), - Job3 = oneshot(3), - setup_jobs([Job2, Job1, Job3]), - ?assertEqual([Job2], pending_jobs(1)), - ?assertEqual([Job2, Job3], pending_jobs(2)), - ?assertEqual([Job2, Job3], pending_jobs(3)) - end). - -t_pending_jobs_skip_running() -> - ?_test(begin - Job1 = continuous(1), - Job2 = continuous_running(2), - Job3 = oneshot(3), - Job4 = oneshot_running(4), - Jobs = [Job1, Job2, Job3, Job4], - setup_jobs(Jobs), - ?assertEqual([Job1, Job3], pending_jobs(4)) - end). - -t_one_job_starts() -> - ?_test(begin - setup_jobs([oneshot(1)]), - ?assertEqual({0, 1}, run_stop_count()), - reschedule(mock_state(?DEFAULT_MAX_JOBS)), - ?assertEqual({1, 0}, run_stop_count()) - end). - -t_no_jobs_start_if_max_is_0() -> - ?_test(begin - setup_jobs([oneshot(1)]), - reschedule(mock_state(0)), - ?assertEqual({0, 1}, run_stop_count()) - end). - -t_one_job_starts_if_max_is_1() -> - ?_test(begin - setup_jobs([oneshot(1), oneshot(2)]), - reschedule(mock_state(1)), - ?assertEqual({1, 1}, run_stop_count()) - end). - -t_max_churn_does_not_throttle_initial_start() -> - ?_test(begin - setup_jobs([oneshot(1), oneshot(2)]), - reschedule(mock_state(?DEFAULT_MAX_JOBS, 0)), - ?assertEqual({2, 0}, run_stop_count()) - end). - -t_excess_oneshot_only_jobs() -> - ?_test(begin - setup_jobs([oneshot_running(1), oneshot_running(2)]), - ?assertEqual({2, 0}, run_stop_count()), - reschedule(mock_state(1)), - ?assertEqual({1, 1}, run_stop_count()), - reschedule(mock_state(0)), - ?assertEqual({0, 2}, run_stop_count()) - end). - -t_excess_continuous_only_jobs() -> - ?_test(begin - setup_jobs([continuous_running(1), continuous_running(2)]), - ?assertEqual({2, 0}, run_stop_count()), - reschedule(mock_state(1)), - ?assertEqual({1, 1}, run_stop_count()), - reschedule(mock_state(0)), - ?assertEqual({0, 2}, run_stop_count()) - end). - -t_excess_prefer_continuous_first() -> - ?_test(begin - Jobs = [ - continuous_running(1), - oneshot_running(2), - continuous_running(3) - ], - setup_jobs(Jobs), - ?assertEqual({3, 0}, run_stop_count()), - ?assertEqual({1, 0}, oneshot_run_stop_count()), - reschedule(mock_state(2)), - ?assertEqual({2, 1}, run_stop_count()), - ?assertEqual({1, 0}, oneshot_run_stop_count()), - reschedule(mock_state(1)), - ?assertEqual({1, 0}, oneshot_run_stop_count()), - reschedule(mock_state(0)), - ?assertEqual({0, 1}, oneshot_run_stop_count()) - end). - -t_stop_oldest_first() -> - ?_test(begin - Jobs = [ - continuous_running(7), - continuous_running(4), - continuous_running(5) - ], - setup_jobs(Jobs), - reschedule(mock_state(2, 1)), - ?assertEqual({2, 1}, run_stop_count()), - ?assertEqual([4], jobs_stopped()), - reschedule(mock_state(1, 1)), - ?assertEqual([7], jobs_running()) - end). - -t_start_oldest_first() -> - ?_test(begin - setup_jobs([continuous(7), continuous(2), continuous(5)]), - reschedule(mock_state(1)), - ?assertEqual({1, 2}, run_stop_count()), - ?assertEqual([2], jobs_running()), - reschedule(mock_state(2)), - ?assertEqual({2, 1}, run_stop_count()), - % After rescheduling with max_jobs = 2, 2 was stopped and 5, 7 should - % be running. - ?assertEqual([2], jobs_stopped()) - end). - -t_jobs_churn_even_if_not_all_max_jobs_are_running() -> - ?_test(begin - setup_jobs([ - continuous_running(7), - continuous(2), - continuous(5) - ]), - reschedule(mock_state(2, 2)), - ?assertEqual({2, 1}, run_stop_count()), - ?assertEqual([7], jobs_stopped()) - end). - -t_jobs_dont_churn_if_there_are_available_running_slots() -> - ?_test(begin - setup_jobs([ - continuous_running(1), - continuous_running(2) - ]), - reschedule(mock_state(2, 2)), - ?assertEqual({2, 0}, run_stop_count()), - ?assertEqual([], jobs_stopped()), - ?assertEqual(0, meck:num_calls(couch_replicator_scheduler_job, start_link, 1)) - end). - -t_start_only_pending_jobs_do_not_churn_existing_ones() -> - ?_test(begin - setup_jobs([ - continuous(1), - continuous_running(2) - ]), - reschedule(mock_state(2, 2)), - ?assertEqual(1, meck:num_calls(couch_replicator_scheduler_job, start_link, 1)), - ?assertEqual([], jobs_stopped()), - ?assertEqual({2, 0}, run_stop_count()) - end). - -t_dont_stop_if_nothing_pending() -> - ?_test(begin - setup_jobs([continuous_running(1), continuous_running(2)]), - reschedule(mock_state(2)), - ?assertEqual({2, 0}, run_stop_count()) - end). - -t_max_churn_limits_number_of_rotated_jobs() -> - ?_test(begin - Jobs = [ - continuous(1), - continuous_running(2), - continuous(3), - continuous_running(4) - ], - setup_jobs(Jobs), - reschedule(mock_state(2, 1)), - ?assertEqual([2, 3], jobs_stopped()) - end). - -t_if_pending_less_than_running_start_all_pending() -> - ?_test(begin - Jobs = [ - continuous(1), - continuous_running(2), - continuous(3), - continuous_running(4), - continuous_running(5) - ], - setup_jobs(Jobs), - reschedule(mock_state(3)), - ?assertEqual([1, 2, 5], jobs_running()) - end). - -t_running_less_than_pending_swap_all_running() -> - ?_test(begin - Jobs = [ - continuous(1), - continuous(2), - continuous(3), - continuous_running(4), - continuous_running(5) - ], - setup_jobs(Jobs), - reschedule(mock_state(2)), - ?assertEqual([3, 4, 5], jobs_stopped()) - end). - -t_oneshot_dont_get_rotated() -> - ?_test(begin - setup_jobs([oneshot_running(1), continuous(2)]), - reschedule(mock_state(1)), - ?assertEqual([1], jobs_running()) - end). - -t_rotate_continuous_only_if_mixed() -> - ?_test(begin - setup_jobs([continuous(1), oneshot_running(2), continuous_running(3)]), - reschedule(mock_state(2)), - ?assertEqual([1, 2], jobs_running()) - end). - -t_oneshot_dont_get_starting_priority() -> - ?_test(begin - setup_jobs([continuous(1), oneshot(2), continuous_running(3)]), - reschedule(mock_state(1)), - ?assertEqual([1], jobs_running()) - end). +t_pending_jobs_simple(_) -> + Job1 = oneshot(1), + Job2 = oneshot(2), + setup_jobs([Job2, Job1]), + ?assertEqual([], pending_jobs(0)), + ?assertEqual([Job1], pending_jobs(1)), + ?assertEqual([Job1, Job2], pending_jobs(2)), + ?assertEqual([Job1, Job2], pending_jobs(3)). + +t_pending_jobs_skip_crashed(_) -> + Job = oneshot(1), + Ts = os:timestamp(), + History = [crashed(Ts), started(Ts) | Job#job.history], + Job1 = Job#job{history = History}, + Job2 = oneshot(2), + Job3 = oneshot(3), + setup_jobs([Job2, Job1, Job3]), + ?assertEqual([Job2], pending_jobs(1)), + ?assertEqual([Job2, Job3], pending_jobs(2)), + ?assertEqual([Job2, Job3], pending_jobs(3)). + +t_pending_jobs_skip_running(_) -> + Job1 = continuous(1), + Job2 = continuous_running(2), + Job3 = oneshot(3), + Job4 = oneshot_running(4), + Jobs = [Job1, Job2, Job3, Job4], + setup_jobs(Jobs), + ?assertEqual([Job1, Job3], pending_jobs(4)). + +t_one_job_starts(_) -> + setup_jobs([oneshot(1)]), + ?assertEqual({0, 1}, run_stop_count()), + reschedule(mock_state(?DEFAULT_MAX_JOBS)), + ?assertEqual({1, 0}, run_stop_count()). + +t_no_jobs_start_if_max_is_0(_) -> + setup_jobs([oneshot(1)]), + reschedule(mock_state(0)), + ?assertEqual({0, 1}, run_stop_count()). + +t_one_job_starts_if_max_is_1(_) -> + setup_jobs([oneshot(1), oneshot(2)]), + reschedule(mock_state(1)), + ?assertEqual({1, 1}, run_stop_count()). + +t_max_churn_does_not_throttle_initial_start(_) -> + setup_jobs([oneshot(1), oneshot(2)]), + reschedule(mock_state(?DEFAULT_MAX_JOBS, 0)), + ?assertEqual({2, 0}, run_stop_count()). + +t_excess_oneshot_only_jobs(_) -> + setup_jobs([oneshot_running(1), oneshot_running(2)]), + ?assertEqual({2, 0}, run_stop_count()), + reschedule(mock_state(1)), + ?assertEqual({1, 1}, run_stop_count()), + reschedule(mock_state(0)), + ?assertEqual({0, 2}, run_stop_count()). + +t_excess_continuous_only_jobs(_) -> + setup_jobs([continuous_running(1), continuous_running(2)]), + ?assertEqual({2, 0}, run_stop_count()), + reschedule(mock_state(1)), + ?assertEqual({1, 1}, run_stop_count()), + reschedule(mock_state(0)), + ?assertEqual({0, 2}, run_stop_count()). + +t_excess_prefer_continuous_first(_) -> + Jobs = [ + continuous_running(1), + oneshot_running(2), + continuous_running(3) + ], + setup_jobs(Jobs), + ?assertEqual({3, 0}, run_stop_count()), + ?assertEqual({1, 0}, oneshot_run_stop_count()), + reschedule(mock_state(2)), + ?assertEqual({2, 1}, run_stop_count()), + ?assertEqual({1, 0}, oneshot_run_stop_count()), + reschedule(mock_state(1)), + ?assertEqual({1, 0}, oneshot_run_stop_count()), + reschedule(mock_state(0)), + ?assertEqual({0, 1}, oneshot_run_stop_count()). + +t_stop_oldest_first(_) -> + Jobs = [ + continuous_running(7), + continuous_running(4), + continuous_running(5) + ], + setup_jobs(Jobs), + reschedule(mock_state(2, 1)), + ?assertEqual({2, 1}, run_stop_count()), + ?assertEqual([4], jobs_stopped()), + reschedule(mock_state(1, 1)), + ?assertEqual([7], jobs_running()). + +t_start_oldest_first(_) -> + setup_jobs([continuous(7), continuous(2), continuous(5)]), + reschedule(mock_state(1)), + ?assertEqual({1, 2}, run_stop_count()), + ?assertEqual([2], jobs_running()), + reschedule(mock_state(2)), + ?assertEqual({2, 1}, run_stop_count()), + % After rescheduling with max_jobs = 2, 2 was stopped and 5, 7 should + % be running. + ?assertEqual([2], jobs_stopped()). + +t_jobs_churn_even_if_not_all_max_jobs_are_running(_) -> + setup_jobs([ + continuous_running(7), + continuous(2), + continuous(5) + ]), + reschedule(mock_state(2, 2)), + ?assertEqual({2, 1}, run_stop_count()), + ?assertEqual([7], jobs_stopped()). + +t_jobs_dont_churn_if_there_are_available_running_slots(_) -> + setup_jobs([ + continuous_running(1), + continuous_running(2) + ]), + reschedule(mock_state(2, 2)), + ?assertEqual({2, 0}, run_stop_count()), + ?assertEqual([], jobs_stopped()), + ?assertEqual(0, meck:num_calls(couch_replicator_scheduler_job, start_link, 1)). + +t_start_only_pending_jobs_do_not_churn_existing_ones(_) -> + setup_jobs([ + continuous(1), + continuous_running(2) + ]), + reschedule(mock_state(2, 2)), + ?assertEqual(1, meck:num_calls(couch_replicator_scheduler_job, start_link, 1)), + ?assertEqual([], jobs_stopped()), + ?assertEqual({2, 0}, run_stop_count()). + +t_dont_stop_if_nothing_pending(_) -> + setup_jobs([continuous_running(1), continuous_running(2)]), + reschedule(mock_state(2)), + ?assertEqual({2, 0}, run_stop_count()). + +t_max_churn_limits_number_of_rotated_jobs(_) -> + Jobs = [ + continuous(1), + continuous_running(2), + continuous(3), + continuous_running(4) + ], + setup_jobs(Jobs), + reschedule(mock_state(2, 1)), + ?assertEqual([2, 3], jobs_stopped()). + +t_if_pending_less_than_running_start_all_pending(_) -> + Jobs = [ + continuous(1), + continuous_running(2), + continuous(3), + continuous_running(4), + continuous_running(5) + ], + setup_jobs(Jobs), + reschedule(mock_state(3)), + ?assertEqual([1, 2, 5], jobs_running()). + +t_running_less_than_pending_swap_all_running(_) -> + Jobs = [ + continuous(1), + continuous(2), + continuous(3), + continuous_running(4), + continuous_running(5) + ], + setup_jobs(Jobs), + reschedule(mock_state(2)), + ?assertEqual([3, 4, 5], jobs_stopped()). + +t_oneshot_dont_get_rotated(_) -> + setup_jobs([oneshot_running(1), continuous(2)]), + reschedule(mock_state(1)), + ?assertEqual([1], jobs_running()). + +t_rotate_continuous_only_if_mixed(_) -> + setup_jobs([continuous(1), oneshot_running(2), continuous_running(3)]), + reschedule(mock_state(2)), + ?assertEqual([1, 2], jobs_running()). + +t_oneshot_dont_get_starting_priority(_) -> + setup_jobs([continuous(1), oneshot(2), continuous_running(3)]), + reschedule(mock_state(1)), + ?assertEqual([1], jobs_running()). % This tested in other test cases, it is here to mainly make explicit a property % of one-shot replications -- they can starve other jobs if they "take control" % of all the available scheduler slots. -t_oneshot_will_hog_the_scheduler() -> - ?_test(begin - Jobs = [ - oneshot_running(1), - oneshot_running(2), - oneshot(3), - continuous(4) - ], - setup_jobs(Jobs), - reschedule(mock_state(2)), - ?assertEqual([1, 2], jobs_running()) - end). - -t_if_excess_is_trimmed_rotation_still_happens() -> - ?_test(begin - Jobs = [ - continuous(1), - continuous_running(2), - continuous_running(3) - ], - setup_jobs(Jobs), - reschedule(mock_state(1)), - ?assertEqual([1], jobs_running()) - end). - -t_if_transient_job_crashes_it_gets_removed() -> - ?_test(begin - Pid = mock_pid(), - Rep = continuous_rep(), - Job = #job{ - id = job1, - pid = Pid, - history = [added()], - rep = Rep#rep{db_name = null} - }, - setup_jobs([Job]), - ?assertEqual(1, ets:info(?MODULE, size)), - State = #state{max_history = 3, stats_pid = self()}, - {noreply, State} = handle_info( - {'EXIT', Pid, failed}, - State - ), - ?assertEqual(0, ets:info(?MODULE, size)) - end). - -t_if_permanent_job_crashes_it_stays_in_ets() -> - ?_test(begin - Pid = mock_pid(), - Rep = continuous_rep(), - Job = #job{ - id = job1, - pid = Pid, - history = [added()], - rep = Rep#rep{db_name = <<"db1">>} - }, - setup_jobs([Job]), - ?assertEqual(1, ets:info(?MODULE, size)), - State = #state{ - max_jobs = 1, - max_history = 3, - stats_pid = self() - }, - {noreply, State} = handle_info( - {'EXIT', Pid, failed}, - State - ), - ?assertEqual(1, ets:info(?MODULE, size)), - [Job1] = ets:lookup(?MODULE, job1), - [Latest | _] = Job1#job.history, - ?assertMatch({{crashed, failed}, _}, Latest) - end). - -t_existing_jobs() -> - ?_test(begin - Rep0 = continuous_rep(<<"s">>, <<"t">>), - Rep = Rep0#rep{id = job1, db_name = <<"db">>}, - setup_jobs([#job{id = Rep#rep.id, rep = Rep}]), - NewRep0 = continuous_rep(<<"s">>, <<"t">>), - NewRep = NewRep0#rep{id = Rep#rep.id, db_name = <<"db">>}, - ?assert(existing_replication(NewRep)), - ?assertNot(existing_replication(NewRep#rep{source = <<"s1">>})), - ?assertNot(existing_replication(NewRep#rep{target = <<"t1">>})), - ?assertNot(existing_replication(NewRep#rep{options = []})) - end). - -t_job_summary_running() -> - ?_test(begin - Rep = rep(<<"s">>, <<"t">>), - Job = #job{ - id = job1, - pid = mock_pid(), - history = [added()], - rep = Rep#rep{db_name = <<"db1">>} - }, - setup_jobs([Job]), - Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), - ?assertEqual(running, proplists:get_value(state, Summary)), - ?assertEqual(null, proplists:get_value(info, Summary)), - ?assertEqual(0, proplists:get_value(error_count, Summary)), - - Stats = [{source_seq, <<"1-abc">>}], - handle_cast({update_job_stats, job1, Stats}, mock_state(1)), - Summary1 = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), - ?assertEqual({Stats}, proplists:get_value(info, Summary1)) - end). - -t_job_summary_pending() -> - ?_test(begin - Job = #job{ - id = job1, - pid = undefined, - history = [stopped(20), started(10), added()], - rep = rep(<<"s">>, <<"t">>) - }, - setup_jobs([Job]), - Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), - ?assertEqual(pending, proplists:get_value(state, Summary)), - ?assertEqual(null, proplists:get_value(info, Summary)), - ?assertEqual(0, proplists:get_value(error_count, Summary)), - - Stats = [{doc_write_failures, 1}], - handle_cast({update_job_stats, job1, Stats}, mock_state(1)), - Summary1 = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), - ?assertEqual({Stats}, proplists:get_value(info, Summary1)) - end). - -t_job_summary_crashing_once() -> - ?_test(begin - Job = #job{ - id = job1, - history = [crashed(?DEFAULT_HEALTH_THRESHOLD_SEC + 1), started(0)], - rep = rep(<<"s">>, <<"t">>) - }, - setup_jobs([Job]), - Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), - ?assertEqual(crashing, proplists:get_value(state, Summary)), - Info = proplists:get_value(info, Summary), - ?assertEqual({[{<<"error">>, <<"some_reason">>}]}, Info), - ?assertEqual(0, proplists:get_value(error_count, Summary)) - end). - -t_job_summary_crashing_many_times() -> - ?_test(begin - Job = #job{ - id = job1, - history = [crashed(4), started(3), crashed(2), started(1)], - rep = rep(<<"s">>, <<"t">>) - }, - setup_jobs([Job]), - Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), - ?assertEqual(crashing, proplists:get_value(state, Summary)), - Info = proplists:get_value(info, Summary), - ?assertEqual({[{<<"error">>, <<"some_reason">>}]}, Info), - ?assertEqual(2, proplists:get_value(error_count, Summary)) - end). - -t_job_summary_proxy_fields() -> - ?_test(begin - Src = #httpdb{ - url = "https://s", - proxy_url = "http://u:p@sproxy:12" - }, - Tgt = #httpdb{ - url = "http://t", - proxy_url = "socks5://u:p@tproxy:34" - }, - Job = #job{ - id = job1, - history = [started(10), added()], - rep = rep(Src, Tgt) - }, - setup_jobs([Job]), - Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), - ?assertEqual( - <<"http://u:*****@sproxy:12">>, - proplists:get_value(source_proxy, Summary) - ), - ?assertEqual( - <<"socks5://u:*****@tproxy:34">>, - proplists:get_value(target_proxy, Summary) - ) - end). +t_oneshot_will_hog_the_scheduler(_) -> + Jobs = [ + oneshot_running(1), + oneshot_running(2), + oneshot(3), + continuous(4) + ], + setup_jobs(Jobs), + reschedule(mock_state(2)), + ?assertEqual([1, 2], jobs_running()). + +t_if_excess_is_trimmed_rotation_still_happens(_) -> + Jobs = [ + continuous(1), + continuous_running(2), + continuous_running(3) + ], + setup_jobs(Jobs), + reschedule(mock_state(1)). + +t_if_transient_job_crashes_it_gets_removed(_) -> + Pid = mock_pid(), + Rep = continuous_rep(), + Job = #job{ + id = job1, + pid = Pid, + history = [added()], + rep = Rep#rep{db_name = null} + }, + setup_jobs([Job]), + ?assertEqual(1, ets:info(?MODULE, size)), + State = #state{max_history = 3, stats_pid = self()}, + {noreply, State} = handle_info( + {'EXIT', Pid, failed}, + State + ), + ?assertEqual(0, ets:info(?MODULE, size)). + +t_if_permanent_job_crashes_it_stays_in_ets(_) -> + Pid = mock_pid(), + Rep = continuous_rep(), + Job = #job{ + id = job1, + pid = Pid, + history = [added()], + rep = Rep#rep{db_name = <<"db1">>} + }, + setup_jobs([Job]), + ?assertEqual(1, ets:info(?MODULE, size)), + State = #state{ + max_jobs = 1, + max_history = 3, + stats_pid = self() + }, + {noreply, State} = handle_info( + {'EXIT', Pid, failed}, + State + ), + ?assertEqual(1, ets:info(?MODULE, size)), + [Job1] = ets:lookup(?MODULE, job1), + [Latest | _] = Job1#job.history, + ?assertMatch({{crashed, failed}, _}, Latest). + +t_stop_all_stops_jobs(_) -> + Jobs = [ + oneshot_running(1), + oneshot_running(2), + oneshot(3), + continuous(4) + ], + setup_jobs(Jobs), + ?assertEqual(ok, stop_clear_all_jobs(?TERMINATE_SHUTDOWN_TIME)), + ?assertEqual([], jobs_running()), + ?assertEqual(0, ets:info(?MODULE, size)). + +t_existing_jobs(_) -> + Rep0 = continuous_rep(<<"s">>, <<"t">>), + Rep = Rep0#rep{id = job1, db_name = <<"db">>}, + setup_jobs([#job{id = Rep#rep.id, rep = Rep}]), + NewRep0 = continuous_rep(<<"s">>, <<"t">>), + NewRep = NewRep0#rep{id = Rep#rep.id, db_name = <<"db">>}, + ?assert(existing_replication(NewRep)), + ?assertNot(existing_replication(NewRep#rep{source = <<"s1">>})), + ?assertNot(existing_replication(NewRep#rep{target = <<"t1">>})), + ?assertNot(existing_replication(NewRep#rep{options = []})). + +t_job_summary_running(_) -> + Rep = rep(<<"s">>, <<"t">>), + Job = #job{ + id = job1, + pid = mock_pid(), + history = [added()], + rep = Rep#rep{db_name = <<"db1">>} + }, + setup_jobs([Job]), + Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), + ?assertEqual(running, proplists:get_value(state, Summary)), + ?assertEqual(null, proplists:get_value(info, Summary)), + ?assertEqual(0, proplists:get_value(error_count, Summary)), + + Stats = [{source_seq, <<"1-abc">>}], + handle_cast({update_job_stats, job1, Stats}, mock_state(1)), + Summary1 = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), + ?assertEqual({Stats}, proplists:get_value(info, Summary1)). + +t_job_summary_pending(_) -> + Job = #job{ + id = job1, + pid = undefined, + history = [stopped(20), started(10), added()], + rep = rep(<<"s">>, <<"t">>) + }, + setup_jobs([Job]), + Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), + ?assertEqual(pending, proplists:get_value(state, Summary)), + ?assertEqual(null, proplists:get_value(info, Summary)), + ?assertEqual(0, proplists:get_value(error_count, Summary)), + + Stats = [{doc_write_failures, 1}], + handle_cast({update_job_stats, job1, Stats}, mock_state(1)), + Summary1 = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), + ?assertEqual({Stats}, proplists:get_value(info, Summary1)). + +t_job_summary_crashing_once(_) -> + Job = #job{ + id = job1, + history = [crashed(?DEFAULT_HEALTH_THRESHOLD_SEC + 1), started(0)], + rep = rep(<<"s">>, <<"t">>) + }, + setup_jobs([Job]), + Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), + ?assertEqual(crashing, proplists:get_value(state, Summary)), + Info = proplists:get_value(info, Summary), + ?assertEqual({[{<<"error">>, <<"some_reason">>}]}, Info), + ?assertEqual(0, proplists:get_value(error_count, Summary)). + +t_job_summary_crashing_many_times(_) -> + Job = #job{ + id = job1, + history = [crashed(4), started(3), crashed(2), started(1)], + rep = rep(<<"s">>, <<"t">>) + }, + setup_jobs([Job]), + Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), + ?assertEqual(crashing, proplists:get_value(state, Summary)), + Info = proplists:get_value(info, Summary), + ?assertEqual({[{<<"error">>, <<"some_reason">>}]}, Info), + ?assertEqual(2, proplists:get_value(error_count, Summary)). + +t_job_summary_proxy_fields(_) -> + Src = #httpdb{ + url = "https://s", + proxy_url = "http://u:p@sproxy:12" + }, + Tgt = #httpdb{ + url = "http://t", + proxy_url = "socks5://u:p@tproxy:34" + }, + Job = #job{ + id = job1, + history = [started(10), added()], + rep = rep(Src, Tgt) + }, + setup_jobs([Job]), + Summary = job_summary(job1, ?DEFAULT_HEALTH_THRESHOLD_SEC), + ?assertEqual( + <<"http://u:*****@sproxy:12">>, + proplists:get_value(source_proxy, Summary) + ), + ?assertEqual( + <<"socks5://u:*****@tproxy:34">>, + proplists:get_value(target_proxy, Summary) + ). % Test helper functions diff --git a/src/couch_replicator/src/couch_replicator_scheduler_job.erl b/src/couch_replicator/src/couch_replicator_scheduler_job.erl index 544c5602a3..f82e362693 100644 --- a/src/couch_replicator/src/couch_replicator_scheduler_job.erl +++ b/src/couch_replicator/src/couch_replicator_scheduler_job.erl @@ -25,7 +25,7 @@ handle_call/3, handle_info/2, handle_cast/2, - format_status/2, + format_status/1, sum_stats/2, report_seq_done/3 ]). @@ -47,6 +47,7 @@ -define(LOWEST_SEQ, 0). -define(DEFAULT_CHECKPOINT_INTERVAL, 30000). -define(STARTUP_JITTER_DEFAULT, 5000). +-define(STOP_TIMEOUT_MSEC, 5000). -record(rep_state, { rep_details, @@ -104,13 +105,14 @@ start_link(#rep{id = Id = {BaseId, Ext}, source = Src, target = Tgt} = Rep) -> end. stop(Pid) when is_pid(Pid) -> - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), unlink(Pid), % In the rare case the job is already stopping as we try to stop it, it % won't return ok but exit the calling process, usually the scheduler, so % we guard against that. See: % www.erlang.org/doc/apps/stdlib/gen_server.html#stop/3 - catch gen_server:stop(Pid, shutdown, infinity), + catch gen_server:stop(Pid, shutdown, ?STOP_TIMEOUT_MSEC), + exit(Pid, kill), receive {'DOWN', Ref, _, _, Reason} -> Reason end, @@ -475,38 +477,46 @@ terminate_cleanup(#rep_state{rep_details = #rep{id = RepId}} = State) -> couch_replicator_api_wrap:db_close(State#rep_state.source), couch_replicator_api_wrap:db_close(State#rep_state.target). -format_status(_Opt, [_PDict, State]) -> - #rep_state{ - source = Source, - target = Target, - rep_details = RepDetails, - start_seq = StartSeq, - source_seq = SourceSeq, - committed_seq = CommitedSeq, - current_through_seq = ThroughSeq, - highest_seq_done = HighestSeqDone, - session_id = SessionId - } = state_strip_creds(State), - #rep{ - id = RepId, - options = Options, - doc_id = DocId, - db_name = DbName - } = RepDetails, - [ - {rep_id, RepId}, - {source, couch_replicator_api_wrap:db_uri(Source)}, - {target, couch_replicator_api_wrap:db_uri(Target)}, - {db_name, DbName}, - {doc_id, DocId}, - {options, Options}, - {session_id, SessionId}, - {start_seq, StartSeq}, - {source_seq, SourceSeq}, - {committed_seq, CommitedSeq}, - {current_through_seq, ThroughSeq}, - {highest_seq_done, HighestSeqDone} - ]. +format_status(Status) -> + maps:map( + fun + (state, State) -> + #rep_state{ + source = Source, + target = Target, + rep_details = RepDetails, + start_seq = StartSeq, + source_seq = SourceSeq, + committed_seq = CommitedSeq, + current_through_seq = ThroughSeq, + highest_seq_done = HighestSeqDone, + session_id = SessionId + } = state_strip_creds(State), + #rep{ + id = RepId, + options = Options, + doc_id = DocId, + db_name = DbName + } = RepDetails, + #{ + rep_id => RepId, + source => couch_replicator_api_wrap:db_uri(Source), + target => couch_replicator_api_wrap:db_uri(Target), + db_name => DbName, + doc_id => DocId, + options => Options, + session_id => SessionId, + start_seq => StartSeq, + source_seq => SourceSeq, + committed_seq => CommitedSeq, + current_through_seq => ThroughSeq, + highest_seq_done => HighestSeqDone + }; + (_, Value) -> + Value + end, + Status + ). sum_stats(Pid, Stats) when is_pid(Pid) -> gen_server:cast(Pid, {sum_stats, Stats}). @@ -520,7 +530,7 @@ startup_jitter() -> "startup_jitter", ?STARTUP_JITTER_DEFAULT ), - rand:uniform(erlang:max(1, Jitter)). + rand:uniform(max(1, Jitter)). headers_strip_creds([], Acc) -> lists:reverse(Acc); @@ -1228,29 +1238,31 @@ t_scheduler_job_format_status(_) -> doc_id = <<"mydoc">>, db_name = <<"mydb">> }, - State = #rep_state{ - rep_details = Rep, - source = Rep#rep.source, - target = Rep#rep.target, - session_id = <<"a">>, - start_seq = <<"1">>, - source_seq = <<"2">>, - committed_seq = <<"3">>, - current_through_seq = <<"4">>, - highest_seq_done = <<"5">> + Status = #{ + state => #rep_state{ + rep_details = Rep, + source = Rep#rep.source, + target = Rep#rep.target, + session_id = <<"a">>, + start_seq = <<"1">>, + source_seq = <<"2">>, + committed_seq = <<"3">>, + current_through_seq = <<"4">>, + highest_seq_done = <<"5">> + } }, - Format = format_status(opts_ignored, [pdict, State]), - ?assertEqual("http://h1/d1/", proplists:get_value(source, Format)), - ?assertEqual("http://h2/d2/", proplists:get_value(target, Format)), - ?assertEqual({"base", "+ext"}, proplists:get_value(rep_id, Format)), - ?assertEqual([{create_target, true}], proplists:get_value(options, Format)), - ?assertEqual(<<"mydoc">>, proplists:get_value(doc_id, Format)), - ?assertEqual(<<"mydb">>, proplists:get_value(db_name, Format)), - ?assertEqual(<<"a">>, proplists:get_value(session_id, Format)), - ?assertEqual(<<"1">>, proplists:get_value(start_seq, Format)), - ?assertEqual(<<"2">>, proplists:get_value(source_seq, Format)), - ?assertEqual(<<"3">>, proplists:get_value(committed_seq, Format)), - ?assertEqual(<<"4">>, proplists:get_value(current_through_seq, Format)), - ?assertEqual(<<"5">>, proplists:get_value(highest_seq_done, Format)). + #{state := State} = format_status(Status), + ?assertEqual("http://h1/d1/", maps:get(source, State)), + ?assertEqual("http://h2/d2/", maps:get(target, State)), + ?assertEqual({"base", "+ext"}, maps:get(rep_id, State)), + ?assertEqual([{create_target, true}], maps:get(options, State)), + ?assertEqual(<<"mydoc">>, maps:get(doc_id, State)), + ?assertEqual(<<"mydb">>, maps:get(db_name, State)), + ?assertEqual(<<"a">>, maps:get(session_id, State)), + ?assertEqual(<<"1">>, maps:get(start_seq, State)), + ?assertEqual(<<"2">>, maps:get(source_seq, State)), + ?assertEqual(<<"3">>, maps:get(committed_seq, State)), + ?assertEqual(<<"4">>, maps:get(current_through_seq, State)), + ?assertEqual(<<"5">>, maps:get(highest_seq_done, State)). -endif. diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl index 9ce7866460..5e8187200b 100644 --- a/src/couch_replicator/src/couch_replicator_utils.erl +++ b/src/couch_replicator/src/couch_replicator_utils.erl @@ -30,7 +30,8 @@ normalize_basic_auth/1, seq_encode/1, valid_endpoint_protocols_log/1, - verify_ssl_certificates_log/1 + verify_ssl_certificates_log/1, + cacert_get/0 ]). -include_lib("ibrowse/include/ibrowse.hrl"). @@ -40,6 +41,10 @@ -include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl"). -include_lib("public_key/include/public_key.hrl"). +-define(CACERT_KEY, {?MODULE, cacert_timestamp_key}). +-define(CACERT_DEFAULT_TIMESTAMP, -(1 bsl 59)). +-define(CACERT_DEFAULT_INTERVAL_HOURS, 24). + -import(couch_util, [ get_value/2, get_value/3 @@ -76,7 +81,7 @@ get_json_value(Key, Props, Default) when is_atom(Key) -> Ref = make_ref(), case get_value(Key, Props, Ref) of Ref -> - get_value(?l2b(atom_to_list(Key)), Props, Default); + get_value(atom_to_binary(Key), Props, Default); Else -> Else end; @@ -84,7 +89,7 @@ get_json_value(Key, Props, Default) when is_binary(Key) -> Ref = make_ref(), case get_value(Key, Props, Ref) of Ref -> - get_value(list_to_atom(?b2l(Key)), Props, Default); + get_value(binary_to_atom(Key), Props, Default); Else -> Else end. @@ -402,6 +407,34 @@ rep_principal(#rep{user_ctx = #user_ctx{name = Name}}) when is_binary(Name) -> rep_principal(#rep{}) -> "by unknown principal". +cacert_get() -> + Now = erlang:monotonic_time(second), + Max = cacert_reload_interval_sec(), + TStamp = persistent_term:get(?CACERT_KEY, ?CACERT_DEFAULT_TIMESTAMP), + cacert_load(TStamp, Now, Max), + public_key:cacerts_get(). + +cacert_load(TStamp, Now, Max) when (Now - TStamp) > Max -> + public_key:cacerts_clear(), + case public_key:cacerts_load() of + ok -> + Count = length(public_key:cacerts_get()), + InfoMsg = "~p : loaded ~p os ca certificates", + couch_log:info(InfoMsg, [?MODULE, Count]); + {error, Reason} -> + ErrMsg = "~p : error loading os ca certificates: ~p", + couch_log:error(ErrMsg, [?MODULE, Reason]) + end, + persistent_term:put(?CACERT_KEY, Now), + loaded; +cacert_load(_TStamp, _Now, _Max) -> + not_loaded. + +cacert_reload_interval_sec() -> + Default = ?CACERT_DEFAULT_INTERVAL_HOURS, + Hrs = config:get_integer("replicator", "cacert_reload_interval_hours", Default), + Hrs * 3600. + -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). @@ -778,4 +811,19 @@ t_allow_canceling_transient_jobs(_) -> ?assertEqual(ok, valid_endpoint_protocols_log(#rep{})), ?assertEqual(0, meck:num_calls(couch_log, warning, 2)). +cacert_test() -> + Old = ?CACERT_DEFAULT_TIMESTAMP, + Now = erlang:monotonic_time(second), + Max = 0, + ?assertEqual(loaded, cacert_load(Old, Now, Max)), + ?assertEqual(not_loaded, cacert_load(Now, Now, Max)), + try cacert_get() of + CACerts -> + ?assert(is_list(CACerts)) + catch + error:_Err -> + % This is ok, some environments may not have OS certs + ?assert(true) + end. + -endif. diff --git a/src/couch_replicator/src/couch_replicator_worker.erl b/src/couch_replicator/src/couch_replicator_worker.erl index 078e6e7e0f..3a6edc0b85 100644 --- a/src/couch_replicator/src/couch_replicator_worker.erl +++ b/src/couch_replicator/src/couch_replicator_worker.erl @@ -19,7 +19,7 @@ % gen_server callbacks -export([init/1]). -export([handle_call/3, handle_cast/2, handle_info/2]). --export([format_status/2]). +-export([format_status/1]). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl"). @@ -242,25 +242,33 @@ handle_info({'EXIT', _Pid, {doc_write_failed, _} = Err}, State) -> handle_info({'EXIT', Pid, Reason}, State) -> {stop, {process_died, Pid, Reason}, State}. -format_status(_Opt, [_PDict, State]) -> - #state{ - cp = MainJobPid, - loop = LoopPid, - source = Source, - target = Target, - readers = Readers, - pending_fetch = PendingFetch, - batch = #batch{size = BatchSize} - } = State, - [ - {main_pid, MainJobPid}, - {loop, LoopPid}, - {source, couch_replicator_api_wrap:db_uri(Source)}, - {target, couch_replicator_api_wrap:db_uri(Target)}, - {num_readers, length(Readers)}, - {pending_fetch, PendingFetch}, - {batch_size, BatchSize} - ]. +format_status(Status) -> + maps:map( + fun + (state, State) -> + #state{ + cp = MainJobPid, + loop = LoopPid, + source = Source, + target = Target, + readers = Readers, + pending_fetch = PendingFetch, + batch = #batch{size = BatchSize} + } = State, + #{ + main_pid => MainJobPid, + loop => LoopPid, + source => couch_replicator_api_wrap:db_uri(Source), + target => couch_replicator_api_wrap:db_uri(Target), + num_readers => length(Readers), + pending_fetch => PendingFetch, + batch_size => BatchSize + }; + (_, Value) -> + Value + end, + Status + ). sum_stats(Pid, Stats) when is_pid(Pid) -> ok = gen_server:cast(Pid, {sum_stats, Stats}). @@ -733,23 +741,25 @@ maybe_report_stats(#state{} = State) -> -include_lib("eunit/include/eunit.hrl"). replication_worker_format_status_test() -> - State = #state{ - cp = self(), - loop = self(), - source = #httpdb{url = "http://u:p@h/d1"}, - target = #httpdb{url = "http://u:p@h/d2"}, - readers = [r1, r2, r3], - pending_fetch = nil, - batch = #batch{size = 5} + Status = #{ + state => #state{ + cp = self(), + loop = self(), + source = #httpdb{url = "http://u:p@h/d1"}, + target = #httpdb{url = "http://u:p@h/d2"}, + readers = [r1, r2, r3], + pending_fetch = nil, + batch = #batch{size = 5} + } }, - Format = format_status(opts_ignored, [pdict, State]), - ?assertEqual(self(), proplists:get_value(main_pid, Format)), - ?assertEqual(self(), proplists:get_value(loop, Format)), - ?assertEqual("http://u:*****@h/d1", proplists:get_value(source, Format)), - ?assertEqual("http://u:*****@h/d2", proplists:get_value(target, Format)), - ?assertEqual(3, proplists:get_value(num_readers, Format)), - ?assertEqual(nil, proplists:get_value(pending_fetch, Format)), - ?assertEqual(5, proplists:get_value(batch_size, Format)). + #{state := State} = format_status(Status), + ?assertEqual(self(), maps:get(main_pid, State)), + ?assertEqual(self(), maps:get(loop, State)), + ?assertEqual("http://u:*****@h/d1", maps:get(source, State)), + ?assertEqual("http://u:*****@h/d2", maps:get(target, State)), + ?assertEqual(3, maps:get(num_readers, State)), + ?assertEqual(nil, maps:get(pending_fetch, State)), + ?assertEqual(5, maps:get(batch_size, State)). bulk_get_attempt_test() -> Now = erlang:monotonic_time(second), diff --git a/src/couch_replicator/src/json_stream_parse.erl b/src/couch_replicator/src/json_stream_parse.erl index a76c1dffff..8b0c34fa22 100644 --- a/src/couch_replicator/src/json_stream_parse.erl +++ b/src/couch_replicator/src/json_stream_parse.erl @@ -300,7 +300,7 @@ toke_string(DF, <<$\\, $t, Rest/binary>>, Acc) -> toke_string(DF, Rest, [$\t | Acc]); toke_string(DF, <<$\\, $u, Rest/binary>>, Acc) -> {<>, DF2} = must_df(DF, 4, Rest, missing_hex), - UTFChar = erlang:list_to_integer([A, B, C, D], 16), + UTFChar = list_to_integer([A, B, C, D], 16), if UTFChar == 16#FFFF orelse UTFChar == 16#FFFE -> err(invalid_utf_char); diff --git a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl index df8074f1fd..d3e883f846 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl @@ -170,7 +170,7 @@ wait_target_in_sync(Source, Target) -> wait_target_in_sync_loop(SourceDocCount, Target, 300). wait_target_in_sync_loop(_DocCount, _TargetName, 0) -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -202,7 +202,7 @@ compare_databases(Source, Target) -> {ok, DocT} -> DocT; Error -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -235,14 +235,14 @@ compact_db(Type, Db0) -> Name = couch_db:name(Db0), {ok, Db} = couch_db:open_int(Name, []), {ok, CompactPid} = couch_db:start_compact(Db), - MonRef = erlang:monitor(process, CompactPid), + MonRef = monitor(process, CompactPid), receive {'DOWN', MonRef, process, CompactPid, normal} -> ok; {'DOWN', MonRef, process, CompactPid, noproc} -> ok; {'DOWN', MonRef, process, CompactPid, Reason} -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -258,7 +258,7 @@ compact_db(Type, Db0) -> ]} ) after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -282,7 +282,7 @@ wait_for_compaction(Type, Db) -> {error, noproc} -> ok; {error, Reason} -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -330,7 +330,7 @@ pause_writer(Pid) -> {paused, Ref} -> ok after ?TIMEOUT_WRITER -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -346,7 +346,7 @@ resume_writer(Pid) -> {ok, Ref} -> ok after ?TIMEOUT_WRITER -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -362,7 +362,7 @@ get_writer_num_docs_written(Pid) -> {count, Ref, Count} -> Count after ?TIMEOUT_WRITER -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -378,12 +378,12 @@ stop_writer(Pid) -> Pid ! {stop, Ref}, receive {stopped, Ref, DocsWritten} -> - MonRef = erlang:monitor(process, Pid), + MonRef = monitor(process, Pid), receive {'DOWN', MonRef, process, Pid, _Reason} -> DocsWritten after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -392,7 +392,7 @@ stop_writer(Pid) -> ) end after ?TIMEOUT_WRITER -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -409,7 +409,7 @@ writer_loop(Db0, Parent, Counter) -> fun(I) -> couch_doc:from_json_obj( {[ - {<<"_id">>, ?l2b(integer_to_list(Counter + I))}, + {<<"_id">>, integer_to_binary(Counter + I)}, {<<"value">>, Counter + I}, {<<"_attachments">>, {[ diff --git a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl index aa75bd746f..9e39122548 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl @@ -187,7 +187,7 @@ user_pass() -> {User, Pass, B64Auth}. worker_internals(Pid) -> - Dict = io_lib:format("~p", [erlang:process_info(Pid, dictionary)]), + Dict = io_lib:format("~p", [process_info(Pid, dictionary)]), State = io_lib:format("~p", [sys:get_state(Pid)]), lists:flatten([Dict, State]). diff --git a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl index 2d6079a28e..4988bb41df 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl @@ -454,7 +454,7 @@ wait_target_in_sync(Source, Target) -> wait_target_in_sync_loop(SourceDocCount, Target, 300). wait_target_in_sync_loop(_DocCount, _TargetName, 0) -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl index 512d020370..c8e6d0af74 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl @@ -120,7 +120,7 @@ get_client_worker({Pid, Ref}, ClientName) -> {worker, Ref, Worker} -> Worker after ?TIMEOUT -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl index acb931b9bf..03c1c1a5f7 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl @@ -57,7 +57,7 @@ setup() -> Pid. teardown(Pid) -> - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), unlink(Pid), exit(Pid, kill), receive diff --git a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl index f413e5cf4e..47fbaa7c6f 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl @@ -80,13 +80,13 @@ t_stats_retained_on_job_removal({_Ctx, {Source, Target}}) -> couch_replicator_scheduler:remove_job(RepId). stop_job(RepPid) -> - Ref = erlang:monitor(process, RepPid), + Ref = monitor(process, RepPid), gen_server:cast(couch_replicator_scheduler, {set_max_jobs, 0}), couch_replicator_scheduler:reschedule(), receive {'DOWN', Ref, _, _, _} -> ok after ?TIMEOUT -> - erlang:error(timeout) + error(timeout) end. start_job() -> @@ -187,7 +187,7 @@ wait_target_in_sync(DocCount, Target) when is_integer(DocCount) -> wait_target_in_sync_loop(DocCount, Target, 300). wait_target_in_sync_loop(_DocCount, _TargetName, 0) -> - erlang:error( + error( {assertion_failed, [ {module, ?MODULE}, {line, ?LINE}, diff --git a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl index c7e89e8be6..88e9e8801c 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl @@ -63,7 +63,7 @@ binary_chunk(Size) when is_integer(Size), Size > 0 -> add_docs(DbName, DocCount, DocSize, AttSize) -> [ begin - DocId = iolist_to_binary(["doc", integer_to_list(Id)]), + DocId = <<"doc", (integer_to_binary(Id))/binary>>, add_doc(DbName, DocId, DocSize, AttSize) end || Id <- lists:seq(1, DocCount) diff --git a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl index 843af79192..5f2cfa25f2 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl @@ -181,7 +181,7 @@ replicate({[_ | _]} = RepObject) -> ok = couch_replicator_scheduler:add_job(Rep), couch_replicator_scheduler:reschedule(), Pid = get_pid(Rep#rep.id), - MonRef = erlang:monitor(process, Pid), + MonRef = monitor(process, Pid), receive {'DOWN', MonRef, process, Pid, _} -> ok diff --git a/src/couch_scanner/src/couch_scanner_plugin.erl b/src/couch_scanner/src/couch_scanner_plugin.erl index 04f394c128..84bbf64964 100644 --- a/src/couch_scanner/src/couch_scanner_plugin.erl +++ b/src/couch_scanner/src/couch_scanner_plugin.erl @@ -115,15 +115,25 @@ -callback shards(St :: term(), [#shard{}]) -> {[#shard{}], St1 :: term()}. -% Optional +% Optional. Called right after a shard file is opened so it gets a Db handle. +% Should return the change feed start sequence and a list of options along with any changes +% in a private context. The change feed start sequence should normally be 0 and the list +% of option can be []. The list of options will be passed directly to couch_db:fold_changes, +% so any {dir, Dir}, {end_key, EndSeq} could work there. +% -callback db_opened(St :: term(), Db :: term()) -> - {ok, St :: term()}. + {ChangesSeq :: non_neg_integer(), ChangesOpts :: [term()], St1 :: term()}. -% Optional. If doc is not defined, then ddoc_id default action is {skip, St}. -% If it is defined, the default action is {ok, St}. +% Optional. If doc and doc_fdi are not defined, then doc_id default +% action is {skip, St}. If it is defined, the default action is {ok, St}. -callback doc_id(St :: term(), DocId :: binary(), Db :: term()) -> {ok | skip | stop, St1 :: term()}. +% Optional. If doc is not defined, then doc_fdi default action is {stop, St}. +% If it is defined, the default action is {ok, St}. +-callback doc_fdi(St :: term(), FDI :: #full_doc_info{}, Db :: term()) -> + {ok | stop, St1 :: term()}. + % Optional. -callback doc(St :: term(), Db :: term(), #doc{}) -> {ok | stop, St1 :: term()}. @@ -139,6 +149,7 @@ shards/2, db_opened/2, doc_id/3, + doc_fdi/3, doc/3, db_closing/2 ]). @@ -153,12 +164,14 @@ {shards, 2, fun default_shards/3}, {db_opened, 2, fun default_db_opened/3}, {doc_id, 3, fun default_doc_id/3}, + {doc_fdi, 3, fun default_doc_fdi/3}, {doc, 3, fun default_doc/3}, {db_closing, 2, fun default_db_closing/3} ]). -define(CHECKPOINT_INTERVAL_SEC, 10). -define(STOP_TIMEOUT_SEC, 5). +-define(DDOC_BATCH_SIZE, 100). -record(st, { id, @@ -171,6 +184,8 @@ cursor, shards_db, db, + changes_seq = 0, + changes_opts = [], checkpoint_sec = 0, start_sec = 0, skip_dbs, @@ -183,7 +198,7 @@ spawn_link(Id) -> stop(Pid) when is_pid(Pid) -> unlink(Pid), - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), Pid ! stop, receive {'DOWN', Ref, _, _, _} -> ok @@ -231,7 +246,7 @@ init_from_checkpoint(#st{} = St) -> <<"start_sec">> := StartSec } -> Now = tsec(), - PSt = resume_callback(Cbks, SId, EJsonPSt), + PSt = resume_callback(Mod, Cbks, SId, EJsonPSt), St#st{ pst = PSt, cursor = Cur, @@ -285,6 +300,9 @@ scan_dbs(#st{cursor = Cursor} = St) -> couch_db:close(Db) end. +scan_dbs_fold(#full_doc_info{id = <>}, Acc) -> + % In case user added a design doc in the dbs database + {ok, Acc}; scan_dbs_fold(#full_doc_info{} = FDI, #st{shards_db = Db} = Acc) -> Acc1 = Acc#st{cursor = FDI#full_doc_info.id}, Acc2 = maybe_checkpoint(Acc1), @@ -300,9 +318,7 @@ scan_dbs_fold(#full_doc_info{} = FDI, #st{shards_db = Db} = Acc) -> {ok, Acc2} end. -scan_db([], #st{} = St) -> - {ok, St}; -scan_db([_ | _] = Shards, #st{} = St) -> +scan_db(Shards, #st{} = St) -> #st{dbname = DbName, callbacks = Cbks, pst = PSt, skip_dbs = Skip} = St, #{db := DbCbk} = Cbks, case match_skip_pat(DbName, Skip) of @@ -312,7 +328,7 @@ scan_db([_ | _] = Shards, #st{} = St) -> case Go of ok -> St2 = rate_limit(St1, db), - St3 = fold_ddocs(fun scan_ddocs_fold/2, St2), + St3 = scan_ddocs(St2), {Shards1, St4} = shards_callback(St3, Shards), St5 = scan_shards(Shards1, St4), {ok, St5}; @@ -325,16 +341,6 @@ scan_db([_ | _] = Shards, #st{} = St) -> {ok, St} end. -scan_ddocs_fold({meta, _}, #st{} = Acc) -> - {ok, Acc}; -scan_ddocs_fold({row, RowProps}, #st{} = Acc) -> - DDoc = couch_util:get_value(doc, RowProps), - scan_ddoc(ejson_to_doc(DDoc), Acc); -scan_ddocs_fold(complete, #st{} = Acc) -> - {ok, Acc}; -scan_ddocs_fold({error, Error}, _Acc) -> - exit({shutdown, {scan_ddocs_fold, Error}}). - scan_shards([], #st{} = St) -> St; scan_shards([#shard{} = Shard | Rest], #st{} = St) -> @@ -363,9 +369,10 @@ scan_docs(#st{} = St, #shard{name = ShardDbName}) -> try St2 = St1#st{db = Db}, St3 = db_opened_callback(St2), - {ok, St4} = couch_db:fold_docs(Db, fun scan_docs_fold/2, St3, []), + #st{changes_seq = Seq, changes_opts = Opts} = St3, + {ok, St4} = couch_db:fold_changes(Db, Seq, fun scan_docs_fold/2, St3, Opts), St5 = db_closing_callback(St4), - erlang:garbage_collect(), + garbage_collect(), St5#st{db = undefined} after couch_db:close(Db) @@ -382,7 +389,7 @@ scan_docs_fold(#full_doc_info{id = Id} = FDI, #st{} = St) -> {Go, PSt1} = DocIdCbk(PSt, Id, Db), St1 = St#st{pst = PSt1}, case Go of - ok -> scan_doc(FDI, St1); + ok -> scan_fdi(FDI, St1); skip -> {ok, St1}; stop -> {stop, St1} end; @@ -390,6 +397,16 @@ scan_docs_fold(#full_doc_info{id = Id} = FDI, #st{} = St) -> {ok, St} end. +scan_fdi(#full_doc_info{} = FDI, #st{} = St) -> + #st{db = Db, callbacks = Cbks, pst = PSt} = St, + #{doc_fdi := FDICbk} = Cbks, + {Go, PSt1} = FDICbk(PSt, FDI, Db), + St1 = St#st{pst = PSt1}, + case Go of + ok -> scan_doc(FDI, St1); + stop -> {stop, St1} + end. + scan_doc(#full_doc_info{} = FDI, #st{} = St) -> #st{db = Db, callbacks = Cbks, pst = PSt} = St, St1 = rate_limit(St, doc), @@ -410,7 +427,7 @@ maybe_checkpoint(#st{checkpoint_sec = LastCheckpointTSec} = St) -> stop -> exit({shutdown, stop}) after 0 -> ok end, - erlang:garbage_collect(), + garbage_collect(), case tsec() - LastCheckpointTSec > ?CHECKPOINT_INTERVAL_SEC of true -> checkpoint(St); false -> St @@ -486,31 +503,44 @@ start_callback(Mod, Cbks, Now, ScanId, LastStartSec, #{} = EJson) when TSec when is_integer(TSec), TSec =< Now -> #{start := StartCbk} = Cbks, case StartCbk(ScanId, EJson) of - {ok, PSt} -> PSt; - skip -> exit_resched(infinity); - reset -> exit_resched(reset) + {ok, PSt} -> + PSt; + skip -> + % If plugin skipped start, count this as an attempt and + % reschedule to possibly retry in the future. + SkipReschedTSec = schedule_time(Mod, Now, Now), + exit_resched(SkipReschedTSec); + reset -> + exit_resched(reset) end; TSec when is_integer(TSec), TSec > Now -> exit_resched(TSec) end. -resume_callback(#{} = Cbks, SId, #{} = EJsonPSt) when is_binary(SId) -> +resume_callback(Mod, #{} = Cbks, SId, #{} = EJsonPSt) when is_binary(SId) -> #{resume := ResumeCbk} = Cbks, case ResumeCbk(SId, EJsonPSt) of - {ok, PSt} -> PSt; - skip -> exit_resched(infinity); - reset -> exit_resched(reset) + {ok, PSt} -> + PSt; + skip -> + % If plugin skipped resume, count this as an attempt and + % reschedule to possibly retry in the future + Now = tsec(), + SkipReschedTSec = schedule_time(Mod, Now, Now), + exit_resched(SkipReschedTSec); + reset -> + exit_resched(reset) end. db_opened_callback(#st{pst = PSt, callbacks = Cbks, db = Db} = St) -> #{db_opened := DbOpenedCbk} = Cbks, - {ok, PSt1} = DbOpenedCbk(PSt, Db), - St#st{pst = PSt1}. + {Seq, Opts, PSt1} = DbOpenedCbk(PSt, Db), + St#st{pst = PSt1, changes_seq = Seq, changes_opts = Opts}. db_closing_callback(#st{pst = PSt, callbacks = Cbks, db = Db} = St) -> #{db_closing := DbClosingCbk} = Cbks, {ok, PSt1} = DbClosingCbk(PSt, Db), - St#st{pst = PSt1}. + St#st{pst = PSt1, changes_seq = 0, changes_opts = []}. shards_callback(#st{pst = PSt, callbacks = Cbks} = St, Shards) -> #{shards := ShardsCbk} = Cbks, @@ -575,6 +605,7 @@ default_shards(Mod, _F, _A) when is_atom(Mod) -> case is_exported(Mod, db_opened, 2) orelse is_exported(Mod, doc_id, 3) orelse + is_exported(Mod, doc_fdi, 3) orelse is_exported(Mod, doc, 3) orelse is_exported(Mod, db_closing, 2) of @@ -583,14 +614,20 @@ default_shards(Mod, _F, _A) when is_atom(Mod) -> end. default_db_opened(Mod, _F, _A) when is_atom(Mod) -> - fun(St, _Db) -> {ok, St} end. + fun(St, _Db) -> {0, [], St} end. default_doc_id(Mod, _F, _A) when is_atom(Mod) -> - case is_exported(Mod, doc, 3) of + case is_exported(Mod, doc, 3) orelse is_exported(Mod, doc_fdi, 3) of true -> fun(St, _DocId, _Db) -> {ok, St} end; false -> fun(St, _DocId, _Db) -> {skip, St} end end. +default_doc_fdi(Mod, _F, _A) when is_atom(Mod) -> + case is_exported(Mod, doc, 3) of + true -> fun(St, _FDI, _Db) -> {ok, St} end; + false -> fun(St, _FDI, _Db) -> {stop, St} end + end. + default_doc(Mod, _F, _A) when is_atom(Mod) -> fun(St, _Db, _Doc) -> {ok, St} end. @@ -622,27 +659,86 @@ shards_by_range(Shards) -> Dict = lists:foldl(Fun, orddict:new(), Shards), orddict:to_list(Dict). -% Design doc fetching helper +scan_ddocs(#st{mod = Mod} = St) -> + case is_exported(Mod, ddoc, 3) of + true -> + try + fold_ddocs_batched(St, <>) + catch + error:database_does_not_exist -> + St + end; + false -> + % If the plugin doesn't export the ddoc callback, don't bother calling + % fabric:all_docs, as it's expensive + St + end. -fold_ddocs(Fun, #st{dbname = DbName} = Acc) -> +fold_ddocs_batched(#st{dbname = DbName} = St, <<_/binary>> = StartKey) -> QArgs = #mrargs{ include_docs = true, - extra = [{namespace, <<"_design">>}] + start_key = StartKey, + extra = [{namespace, <>}], + % Need limit > 1 for the algorithm below to work + limit = max(2, cfg_ddoc_batch_size()) }, - try - {ok, Acc1} = fabric:all_docs(DbName, [?ADMIN_CTX], Fun, Acc, QArgs), - Acc1 - catch - error:database_does_not_exist -> - Acc + Cbk = + fun + ({meta, _}, {Cnt, Id, DDocs}) -> + {ok, {Cnt, Id, DDocs}}; + ({row, Props}, {Cnt, _Id, DDocs}) -> + EJson = couch_util:get_value(doc, Props), + DDoc = #doc{id = Id} = ejson_to_doc(EJson), + case Id =:= StartKey of + true -> + % We get there if we're continuing batched iteration so + % we skip this ddoc as we already processed it. In the + % first batch StartKey will be <<"_design/">> and + % that's an invalid document ID so will never match. + {ok, {Cnt + 1, Id, DDocs}}; + false -> + {ok, {Cnt + 1, Id, [DDoc | DDocs]}} + end; + (complete, {Cnt, Id, DDocs}) -> + {ok, {Cnt, Id, lists:reverse(DDocs)}}; + ({error, Error}, {_Cnt, _Id, _DDocs}) -> + exit({shutdown, {scan_ddocs_fold, Error}}) + end, + Acc0 = {0, StartKey, []}, + {ok, {Cnt, LastId, DDocs}} = fabric:all_docs(DbName, [?ADMIN_CTX], Cbk, Acc0, QArgs), + case scan_ddoc_batch(DDocs, {ok, St}) of + {ok, #st{} = St1} -> + if + is_integer(Cnt), Cnt < QArgs#mrargs.limit -> + % We got less than we asked for so we're done + St1; + Cnt == QArgs#mrargs.limit -> + % We got all the docs we asked for, there are probably more docs + % so we recurse and fetch the next batch. + fold_ddocs_batched(St1, LastId) + end; + {stop, #st{} = St1} -> + % Plugin wanted to stop scanning ddocs, so we stop + St1 end. +% Call plugin ddocs callback. These may take an arbitrarily long time to +% process. +scan_ddoc_batch(_, {stop, #st{} = St}) -> + {stop, St}; +scan_ddoc_batch([], {ok, #st{} = St}) -> + {ok, St}; +scan_ddoc_batch([#doc{} = DDoc | Rest], {ok, #st{} = St}) -> + scan_ddoc_batch(Rest, scan_ddoc(DDoc, St)). + % Simple ejson to #doc{} function to avoid all the extra validation in from_json_obj/1. % We just got these docs from the cluster, they are already saved on disk. ejson_to_doc({[_ | _] = Props}) -> {value, {_, DocId}, Props1} = lists:keytake(<<"_id">>, 1, Props), - Props2 = [{K, V} || {K, V} <- Props1, K =:= <<>> orelse binary:first(K) =/= $_], - #doc{id = DocId, body = {Props2}}. + {value, {_, Rev}, Props2} = lists:keytake(<<"_rev">>, 1, Props1), + {Pos, RevId} = couch_doc:parse_rev(Rev), + Props3 = [{K, V} || {K, V} <- Props2, K =:= <<>> orelse binary:first(K) =/= $_], + #doc{id = DocId, revs = {Pos, [RevId]}, body = {Props3}}. % Skip patterns @@ -667,6 +763,9 @@ cfg(Mod, Key, Default) when is_list(Key) -> Section = atom_to_list(Mod), config:get(Section, Key, Default). +cfg_ddoc_batch_size() -> + config:get_integer("couch_scanner", "ddoc_batch_size", ?DDOC_BATCH_SIZE). + schedule_time(Mod, LastSec, NowSec) -> After = cfg(Mod, "after", "restart"), Repeat = cfg(Mod, "repeat", "restart"), diff --git a/src/couch_scanner/src/couch_scanner_plugin_conflict_finder.erl b/src/couch_scanner/src/couch_scanner_plugin_conflict_finder.erl index 5f8f175271..4d57cbd91d 100644 --- a/src/couch_scanner/src/couch_scanner_plugin_conflict_finder.erl +++ b/src/couch_scanner/src/couch_scanner_plugin_conflict_finder.erl @@ -19,7 +19,7 @@ complete/1, checkpoint/1, db/2, - doc_id/3 + doc_fdi/3 ]). -include_lib("couch_scanner/include/couch_scanner_plugin.hrl"). @@ -76,12 +76,10 @@ checkpoint(#st{sid = SId, opts = Opts}) -> db(#st{} = St, _DbName) -> {ok, St}. -doc_id(#st{} = St, <>, _Db) -> - {skip, St}; -doc_id(#st{} = St, DocId, Db) -> - {ok, #doc_info{revs = Revs}} = couch_db:get_doc_info(Db, DocId), +doc_fdi(#st{} = St, #full_doc_info{} = FDI, Db) -> + #doc_info{revs = Revs} = couch_doc:to_doc_info(FDI), DbName = mem3:dbname(couch_db:name(Db)), - {ok, check(St, DbName, DocId, Revs)}. + {ok, check(St, DbName, FDI#full_doc_info.id, Revs)}. % Private diff --git a/src/couch_scanner/src/couch_scanner_plugin_find.erl b/src/couch_scanner/src/couch_scanner_plugin_find.erl index af5c8a5503..12b1e22b5a 100644 --- a/src/couch_scanner/src/couch_scanner_plugin_find.erl +++ b/src/couch_scanner/src/couch_scanner_plugin_find.erl @@ -23,7 +23,6 @@ complete/1, checkpoint/1, db/2, - ddoc/3, shards/2, db_opened/2, doc_id/3, @@ -77,11 +76,6 @@ db(#st{} = St, DbName) -> report_match(DbName, Pats, Meta), {ok, St}. -ddoc(#st{} = St, _DbName, #doc{} = _DDoc) -> - % We'll check doc bodies during the shard scan - % so no need to keep inspecting ddocs - {stop, St}. - shards(#st{sid = SId} = St, Shards) -> case debug() of true -> ?DEBUG(" ~p shards", [length(Shards)], #{sid => SId}); @@ -94,7 +88,9 @@ db_opened(#st{sid = SId} = St, Db) -> true -> ?DEBUG("", [], #{sid => SId, db => Db}); false -> ok end, - {ok, St}. + % Search backwards with the idea that we may be looking for some recent + % changes we just made to the database. + {couch_db:get_update_seq(Db), [{dir, rev}], St}. doc_id(#st{} = St, DocId, Db) -> #st{sid = SId, compiled_regexes = Pats} = St, diff --git a/src/couch_scanner/src/couch_scanner_rate_limiter.erl b/src/couch_scanner/src/couch_scanner_rate_limiter.erl index 2717be69e4..605445c19a 100644 --- a/src/couch_scanner/src/couch_scanner_rate_limiter.erl +++ b/src/couch_scanner/src/couch_scanner_rate_limiter.erl @@ -21,6 +21,24 @@ % % [1] https://en.wikipedia.org/wiki/Additive_increase/multiplicative_decrease % +% Example of usage: +% +% initialize: +% Limiter = couch_scanner_rate_limiter:get(), +% +% use: +% bulk_docs(#{docs => [doc1, doc2, doc3]}), +% {Wait, Limiter1} = couch_scanner_rate_limiter:update(Limiter, doc_write, 3), +% timer:sleep(Wait) +% or +% receive .... after Wait -> ... end +% +% The Type can be: +% * db : rate of clustered db opens +% * shard: rate of shard files opened +% * doc : rate of document reads +% * doc_write : rate of document writes (or other per document updates, could be purges, too) +% -module(couch_scanner_rate_limiter). @@ -29,7 +47,8 @@ -export([ start_link/0, get/0, - update/2 + update/2, + update/3 ]). % gen_server callbacks @@ -62,16 +81,17 @@ -define(DB_RATE_DEFAULT, 25). -define(SHARD_RATE_DEFAULT, 50). -define(DOC_RATE_DEFAULT, 1000). +-define(DOC_WRITE_RATE_DEFAULT, 500). % Atomic ref indices. They start at 1. --define(INDICES, #{db => 1, shard => 2, doc => 3}). +-define(INDICES, #{db => 1, shard => 2, doc => 3, doc_write => 4}). % Record maintained by the clients. Each client will have one of these handles. % With each update/2 call they will update their own backoff values. % -record(client_st, { ref, - % db|shard|doc => {Backoff, UpdateTStamp} + % db|shard|doc|doc_write => {Backoff, UpdateTStamp} backoffs = #{} }). @@ -83,13 +103,17 @@ get() -> Ref = gen_server:call(?MODULE, get, infinity), NowMSec = erlang:monotonic_time(millisecond), - Backoffs = maps:from_keys([db, shard, doc], {?INIT_BACKOFF, NowMSec}), + Backoffs = maps:from_keys([db, shard, doc, doc_write], {?INIT_BACKOFF, NowMSec}), #client_st{ref = Ref, backoffs = Backoffs}. -update(#client_st{ref = Ref, backoffs = Backoffs} = St, Type) when - Type =:= db orelse Type =:= shard orelse Type =:= doc +update(St, Type) -> + update(St, Type, 1). + +update(#client_st{ref = Ref, backoffs = Backoffs} = St, Type, Count) when + (is_integer(Count) andalso Count >= 0) andalso + (Type =:= db orelse Type =:= shard orelse Type =:= doc orelse Type =:= doc_write) -> - AtLimit = atomics:sub_get(Ref, map_get(Type, ?INDICES), 1) =< 0, + AtLimit = atomics:sub_get(Ref, map_get(Type, ?INDICES), Count) =< 0, {Backoff, TStamp} = map_get(Type, Backoffs), NowMSec = erlang:monotonic_time(millisecond), case NowMSec - TStamp > ?SENSITIVITY_MSEC of @@ -142,6 +166,7 @@ refill(#st{ref = Ref} = St) -> ok = atomics:put(Ref, map_get(db, ?INDICES), db_limit()), ok = atomics:put(Ref, map_get(shard, ?INDICES), shard_limit()), ok = atomics:put(Ref, map_get(doc, ?INDICES), doc_limit()), + ok = atomics:put(Ref, map_get(doc_write, ?INDICES), doc_write_limit()), schedule_refill(St). update_backoff(true, 0) -> @@ -160,6 +185,9 @@ shard_limit() -> doc_limit() -> cfg_int("doc_rate_limit", ?DOC_RATE_DEFAULT). +doc_write_limit() -> + cfg_int("doc_write_rate_limit", ?DOC_WRITE_RATE_DEFAULT). + cfg_int(Key, Default) when is_list(Key), is_integer(Default) -> config:get_integer("couch_scanner", Key, Default). @@ -175,6 +203,7 @@ couch_scanner_rate_limiter_test_() -> [ ?TDEF_FE(t_init), ?TDEF_FE(t_update), + ?TDEF_FE(t_update_multiple), ?TDEF_FE(t_refill) ] }. @@ -184,7 +213,8 @@ t_init(_) -> ?assertEqual(ok, refill()), ?assertMatch({Val, #client_st{}} when is_number(Val), update(ClientSt, db)), ?assertMatch({Val, #client_st{}} when is_number(Val), update(ClientSt, shard)), - ?assertMatch({Val, #client_st{}} when is_number(Val), update(ClientSt, doc)). + ?assertMatch({Val, #client_st{}} when is_number(Val), update(ClientSt, doc)), + ?assertMatch({Val, #client_st{}} when is_number(Val), update(ClientSt, doc_write)). t_update(_) -> ClientSt = ?MODULE:get(), @@ -196,6 +226,16 @@ t_update(_) -> {Backoff, _} = update(ClientSt1, db), ?assertEqual(?MAX_BACKOFF, Backoff). +t_update_multiple(_) -> + ClientSt = ?MODULE:get(), + Fun = fun(_, Acc) -> + {_, Acc1} = update(Acc, doc_write, 100), + reset_time(Acc1, doc_write) + end, + ClientSt1 = lists:foldl(Fun, ClientSt, lists:seq(1, 50)), + {Backoff, _} = update(ClientSt1, doc_write, 100), + ?assertEqual(?MAX_BACKOFF, Backoff). + t_refill(_) -> ClientSt = ?MODULE:get(), Fun = fun(_, Acc) -> diff --git a/src/couch_scanner/src/couch_scanner_server.erl b/src/couch_scanner/src/couch_scanner_server.erl index 7176173d21..2ec1ac5408 100644 --- a/src/couch_scanner/src/couch_scanner_server.erl +++ b/src/couch_scanner/src/couch_scanner_server.erl @@ -252,6 +252,8 @@ sched_exit_update(Id, #sched{} = Sched, {shutdown, reset}) -> couch_log:warning("~p : resetting plugin ~s", [?MODULE, Id]), couch_scanner_checkpoint:reset(Id), Sched#sched{start_time = 0, error_count = 0, reschedule = tsec()}; +sched_exit_update(_Id, #sched{} = Sched, {shutdown, stop}) -> + Sched#sched{start_time = 0, error_count = 0}; sched_exit_update(Id, #sched{} = Sched, Norm) when Norm == shutdown; Norm == normal -> diff --git a/src/couch_scanner/src/couch_scanner_util.erl b/src/couch_scanner/src/couch_scanner_util.erl index ff4edafd35..5dd44248c8 100644 --- a/src/couch_scanner/src/couch_scanner_util.erl +++ b/src/couch_scanner/src/couch_scanner_util.erl @@ -31,6 +31,7 @@ -define(DAY, 24 * ?HOUR). -define(WEEK, 7 * ?DAY). -define(MONTH, 30 * ?DAY). +-define(YEAR, 365 * ?DAY). new_scan_id() -> TSec = integer_to_binary(erlang:system_time(second)), @@ -240,15 +241,17 @@ parse_non_weekday_period(Period) -> undefined end. -parse_period_unit(Period) when is_list(Period) -> - case Period of - "sec" ++ _ -> ?SECOND; - "min" ++ _ -> ?MINUTE; - "hour" ++ _ -> ?HOUR; - "day" ++ _ -> ?DAY; - "week" ++ _ -> ?WEEK; - "month" ++ _ -> ?MONTH; - _ -> undefined +%% erlfmt-ignore +parse_period_unit(P) when is_list(P) -> + if + P == "s"; P == "sec"; P == "second"; P == "seconds" -> ?SECOND; + P == "min"; P == "minute"; P == "minutes" -> ?MINUTE; + P == "h"; P == "hrs"; P == "hour"; P == "hours" -> ?HOUR; + P == "d"; P == "day"; P == "days" -> ?DAY; + P == "w"; P == "week"; P == "weeks" -> ?WEEK; + P == "mon"; P == "month"; P == "months" -> ?MONTH; + P == "y"; P == "year"; P == "years" -> ?YEAR; + true -> undefined end. % Logging bits @@ -297,6 +300,9 @@ parse_after_test() -> parse_repeat_test() -> ?assertEqual(undefined, parse_repeat("foo")), ?assertEqual(undefined, parse_repeat("ReStarT")), + ?assertEqual(undefined, parse_repeat("1_ms")), + ?assertEqual(undefined, parse_repeat("1_x")), + ?assertEqual(undefined, parse_repeat("1_m")), ?assertEqual({weekday, 1}, parse_repeat("mon")), ?assertEqual({weekday, 1}, parse_repeat("Monday")), ?assertEqual({weekday, 2}, parse_repeat("tuesday")), @@ -306,16 +312,25 @@ parse_repeat_test() -> ?assertEqual({weekday, 6}, parse_repeat("sAt")), ?assertEqual({weekday, 7}, parse_repeat("sundays")), ?assertEqual(1, parse_repeat("1_sec")), + ?assertEqual(1, parse_repeat("1_s")), ?assertEqual(1, parse_repeat("1_second")), ?assertEqual(1, parse_repeat("1_sec")), ?assertEqual(2, parse_repeat("2_sec")), ?assertEqual(3, parse_repeat("3_seconds")), ?assertEqual(60, parse_repeat("1_min")), + ?assertEqual(60, parse_repeat("1_minute")), ?assertEqual(2 * 60, parse_repeat("2_minutes")), ?assertEqual(60 * 60, parse_repeat("1_hour")), + ?assertEqual(3 * 60 * 60, parse_repeat("3_hours")), + ?assertEqual(2 * 60 * 60, parse_repeat("2_h")), ?assertEqual(24 * 60 * 60, parse_repeat("1_day")), ?assertEqual(7 * 24 * 60 * 60, parse_repeat("1_week")), - ?assertEqual(30 * 24 * 60 * 60, parse_repeat("1_month")). + ?assertEqual(2 * 7 * 24 * 60 * 60, parse_repeat("2_weeks")), + ?assertEqual(30 * 24 * 60 * 60, parse_repeat("1_month")), + ?assertEqual(30 * 24 * 60 * 60, parse_repeat("1_mon")), + ?assertEqual(2 * 30 * 24 * 60 * 60, parse_repeat("2_months")), + ?assertEqual(365 * 24 * 60 * 60, parse_repeat("1_year")), + ?assertEqual(2 * 365 * 24 * 60 * 60, parse_repeat("2_year")). repeat_period_test() -> %Fri, May 31, 2024 16:08:37 diff --git a/src/couch_scanner/test/eunit/couch_scanner_test.erl b/src/couch_scanner/test/eunit/couch_scanner_test.erl index 6f7f3d8e2e..5d6a22f387 100644 --- a/src/couch_scanner/test/eunit/couch_scanner_test.erl +++ b/src/couch_scanner/test/eunit/couch_scanner_test.erl @@ -29,6 +29,7 @@ couch_scanner_test_() -> ?TDEF_FE(t_conflict_finder_works, 30), ?TDEF_FE(t_config_skips, 10), ?TDEF_FE(t_resume_after_error, 10), + ?TDEF_FE(t_resume_after_skip, 10), ?TDEF_FE(t_reset, 10), ?TDEF_FE(t_schedule_repeat, 10), ?TDEF_FE(t_schedule_after, 15) @@ -49,15 +50,23 @@ couch_scanner_test_() -> setup() -> {module, _} = code:ensure_loaded(?FIND_PLUGIN), meck:new(?FIND_PLUGIN, [passthrough]), + meck:new(fabric, [passthrough]), meck:new(couch_scanner_server, [passthrough]), meck:new(couch_scanner_util, [passthrough]), Ctx = test_util:start_couch([fabric, couch_scanner]), + % Run with the smallest batch size to exercise the batched + % ddoc iteration + config:set("couch_scanner", "ddoc_batch_size", "2", false), DbName1 = <<"dbname1", (?tempdb())/binary>>, DbName2 = <<"dbname2", (?tempdb())/binary>>, DbName3 = <<"dbname3", (?tempdb())/binary>>, ok = fabric:create_db(DbName1, [{q, "2"}, {n, "1"}]), ok = fabric:create_db(DbName2, [{q, "2"}, {n, "1"}]), ok = fabric:create_db(DbName3, [{q, "2"}, {n, "1"}]), + % Add a design doc the shards db. Scanner should ignore it. + {ok, _} = couch_util:with_db(mem3_sync:shards_db(), fun(Db) -> + couch_db:update_doc(Db, #doc{id = <<"_design/foo">>}, []) + end), ok = add_doc(DbName1, ?DOC1, #{foo1 => bar}), ok = add_doc(DbName1, ?DOC2, #{ foo2 => baz, @@ -84,8 +93,8 @@ setup() -> ok = add_doc(DbName2, ?DOC3, #{foo3 => bax}), ok = add_doc(DbName2, ?DOC4, #{foo4 => baw, <<>> => this_is_ok_apparently}), add_docs(DbName3, [ - {doc, ?DOC5, {2, [<<"x">>, <<"z">>]}, {[]}, [], false, []}, - {doc, ?DOC5, {2, [<<"y">>, <<"z">>]}, {[]}, [], false, []} + #doc{id = ?DOC5, revs = {2, [<<"x">>, <<"z">>]}, deleted = false}, + #doc{id = ?DOC5, revs = {2, [<<"y">>, <<"z">>]}, deleted = false} ]), couch_scanner:reset_checkpoints(), {Ctx, {DbName1, DbName2, DbName3}}. @@ -108,6 +117,7 @@ teardown({Ctx, {DbName1, DbName2, DbName3}}) -> fabric:delete_db(DbName1), fabric:delete_db(DbName2), fabric:delete_db(DbName3), + couch_server:delete(mem3_sync:shards_db(), [?ADMIN_CTX]), test_util:stop_couch(Ctx), meck:unload(). @@ -141,8 +151,6 @@ t_run_through_all_callbacks_basic({_, {DbName1, DbName2, _}}) -> ?assertEqual(2, num_calls(checkpoint, 1)), ?assertEqual(1, num_calls(db, ['_', DbName1])), ?assertEqual(1, num_calls(db, ['_', DbName2])), - ?assertEqual(1, num_calls(ddoc, ['_', DbName1, '_'])), - ?assertEqual(1, num_calls(ddoc, ['_', DbName2, '_'])), ?assert(num_calls(shards, 2) >= 2), DbOpenedCount = num_calls(db_opened, 2), ?assert(DbOpenedCount >= 4), @@ -161,10 +169,14 @@ t_find_reporting_works(_) -> config:set(Plugin ++ ".regexes", "foo14", "foo(1|4)", false), config:set(Plugin ++ ".regexes", "baz", "baz", false), meck:reset(couch_scanner_server), + meck:reset(fabric), config:set("couch_scanner_plugins", Plugin, "true", false), wait_exit(10000), % doc2 should have a baz and doc1 and doc4 matches foo14 - ?assertEqual(3, log_calls(warning)). + ?assertEqual(3, log_calls(warning)), + % check that we didn't call fabric:all_docs fetching design docs + % as we don't need to for this plugin + ?assertEqual(0, meck:num_calls(fabric, all_docs, 5)). t_ddoc_features_works({_, {_, DbName2, _}}) -> % Run the "ddoc_features" plugin @@ -201,11 +213,20 @@ t_conflict_finder_works({_, {_, _, DbName3}}) -> % Add a deleted conflicting doc to the third database. % 3 reports are expected: 2 doc reports and 1 db report. add_docs(DbName3, [ - {doc, ?DOC6, {2, [<<"x">>, <<"z">>]}, {[]}, [], false, []}, - {doc, ?DOC6, {2, [<<"d">>, <<"z">>]}, {[]}, [], true, []} + #doc{id = ?DOC6, revs = {2, [<<"x">>, <<"z">>]}, deleted = false}, + #doc{id = ?DOC6, revs = {2, [<<"d">>, <<"z">>]}, deleted = true} ]), resume_couch_scanner(Plugin), ?assertEqual(3, meck:num_calls(couch_scanner_util, log, LogArgs)), + % Should work even if all revs are deleted (the whole FDI is deleted) + add_docs(DbName3, [ + #doc{id = ?DOC6, revs = {3, [<<"a">>, <<"x">>, <<"z">>]}, deleted = true} + ]), + % Confirm it's deleted (we did the revs paths manipulations correctly) + ?assertEqual({not_found, deleted}, fabric:open_doc(DbName3, ?DOC6, [])), + % But we can still find the conflicts + resume_couch_scanner(Plugin), + ?assertEqual(3, meck:num_calls(couch_scanner_util, log, LogArgs)), % Set doc_report to false to only have 1 db report. config:set(Plugin, "doc_report", "false", false), resume_couch_scanner(Plugin), @@ -257,6 +278,25 @@ t_resume_after_error(_) -> config:set("couch_scanner_plugins", Plugin, "true", false), meck:wait(?FIND_PLUGIN, resume, 2, 10000). +t_resume_after_skip(_) -> + meck:reset(?FIND_PLUGIN), + meck:expect( + ?FIND_PLUGIN, + start, + 2, + meck:seq([ + skip, + meck:passthrough() + ]) + ), + Plugin = atom_to_list(?FIND_PLUGIN), + config:set("couch_scanner", "min_penalty_sec", "1", false), + config:set("couch_scanner", "interval_sec", "1", false), + config:set(Plugin, "repeat", "2_sec", false), + couch_scanner:resume(), + config:set("couch_scanner_plugins", Plugin, "true", false), + meck:wait(?FIND_PLUGIN, complete, 1, 10000). + t_reset(_) -> meck:reset(?FIND_PLUGIN), meck:expect( diff --git a/src/couch_stats/src/couch_stats_httpd.erl b/src/couch_stats/src/couch_stats_httpd.erl index 88ea169d06..5d69384e2f 100644 --- a/src/couch_stats/src/couch_stats_httpd.erl +++ b/src/couch_stats/src/couch_stats_httpd.erl @@ -99,8 +99,8 @@ extract_path([_ | _], _NotAnObject) -> maybe_format_key(Key) when is_list(Key) -> list_to_binary(Key); maybe_format_key(Key) when is_atom(Key) -> - list_to_binary(atom_to_list(Key)); + atom_to_binary(Key); maybe_format_key(Key) when is_integer(Key) -> - list_to_binary(integer_to_list(Key)); + integer_to_binary(Key); maybe_format_key(Key) when is_binary(Key) -> Key. diff --git a/src/couch_stats/src/couch_stats_process_tracker.erl b/src/couch_stats/src/couch_stats_process_tracker.erl index 33cc137da3..c9f9f12914 100644 --- a/src/couch_stats/src/couch_stats_process_tracker.erl +++ b/src/couch_stats/src/couch_stats_process_tracker.erl @@ -49,7 +49,7 @@ handle_call(Msg, _From, State) -> handle_cast({track, Pid, Name}, State) -> couch_stats:increment_counter(Name), - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), ets:insert(?MODULE, {Ref, Name}), {noreply, State}; handle_cast(Msg, State) -> diff --git a/src/custodian/src/custodian_util.erl b/src/custodian/src/custodian_util.erl index 1a29ee7ad1..baf52dff7f 100644 --- a/src/custodian/src/custodian_util.erl +++ b/src/custodian/src/custodian_util.erl @@ -187,7 +187,7 @@ load_shards(Db, #full_doc_info{id = Id} = FDI) -> {ok, #doc{body = {Props}}} -> mem3_util:build_shards(Id, Props); {not_found, _} -> - erlang:error(database_does_not_exist, [Id]) + error(database_does_not_exist, [Id]) end. maybe_redirect(Nodes) -> diff --git a/src/ddoc_cache/src/ddoc_cache_entry.erl b/src/ddoc_cache/src/ddoc_cache_entry.erl index de9cb55cc0..694cafe8f9 100644 --- a/src/ddoc_cache/src/ddoc_cache_entry.erl +++ b/src/ddoc_cache/src/ddoc_cache_entry.erl @@ -75,16 +75,16 @@ start_link(Key, Default) -> {ok, Pid}. shutdown(Pid) -> - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), ok = gen_server:cast(Pid, shutdown), receive {'DOWN', Ref, process, Pid, normal} -> ok; {'DOWN', Ref, process, Pid, Reason} -> - erlang:exit(Reason) + exit(Reason) after ?ENTRY_SHUTDOWN_TIMEOUT -> - erlang:demonitor(Ref, [flush]), - erlang:exit({timeout, {entry_shutdown, Pid}}) + demonitor(Ref, [flush]), + exit({timeout, {entry_shutdown, Pid}}) end. open(Pid, Key) -> @@ -98,7 +98,7 @@ open(Pid, Key) -> end catch error:database_does_not_exist -> - erlang:error(database_does_not_exist); + error(database_does_not_exist); exit:_ -> % Its possible that this process was evicted just % before we tried talking to it. Just fallback @@ -257,7 +257,7 @@ handle_info(Msg, St) -> {stop, {bad_info, Msg}, St}. spawn_opener(Key) -> - {Pid, _} = erlang:spawn_monitor(?MODULE, do_open, [Key]), + {Pid, _} = spawn_monitor(?MODULE, do_open, [Key]), Pid. start_timer() -> @@ -269,10 +269,10 @@ start_timer() -> do_open(Key) -> try recover(Key) of Resp -> - erlang:exit({open_ok, Key, Resp}) + exit({open_ok, Key, Resp}) catch T:R:S -> - erlang:exit({open_error, Key, {T, R, S}}) + exit({open_error, Key, {T, R, S}}) end. update_lru(#st{key = Key, ts = Ts} = St) -> diff --git a/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl index 54f5c673f5..5d2e50ef2f 100644 --- a/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl +++ b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl @@ -26,7 +26,18 @@ ddocid(_) -> no_ddocid. recover(DbName) -> - {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)), + %% The VDU function is used to validate documents update before + %% storing them in the database. + %% Raise an error when invalid instead of returning an empty list. + DDocs = + case fabric:design_docs(mem3:dbname(DbName)) of + {ok, Resp} when is_list(Resp) -> + Resp; + {ok, Error} -> + error(Error); + {error, Error} -> + error(Error) + end, Funs = lists:flatmap( fun(DDoc) -> case couch_doc:get_validate_doc_fun(DbName, DDoc) of diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl b/src/ddoc_cache/src/ddoc_cache_lru.erl index 915f3e45b5..b3ed6b83f5 100644 --- a/src/ddoc_cache/src/ddoc_cache_lru.erl +++ b/src/ddoc_cache/src/ddoc_cache_lru.erl @@ -273,7 +273,7 @@ remove_key(#{} = Dbs, Key) -> end. unlink_and_flush(Pid) -> - erlang:unlink(Pid), + unlink(Pid), % Its possible that the entry process has already exited before % we unlink it so we have to flush out a possible 'EXIT' % message sitting in our message queue. Notice that we're diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_coverage_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_coverage_test.erl index a30f3b2ce2..f168c6c5cd 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_coverage_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_coverage_test.erl @@ -37,7 +37,7 @@ stop_on_evictor_death() -> Lru = whereis(ddoc_cache_lru), State = sys:get_state(Lru), Evictor = element(4, State), - Ref = erlang:monitor(process, Lru), + Ref = monitor(process, Lru), exit(Evictor, shutdown), receive {'DOWN', Ref, _, _, Reason} -> @@ -61,7 +61,7 @@ send_bad_messages(Name) -> end). wait_for_restart(Server, Fun) -> - Ref = erlang:monitor(process, whereis(Server)), + Ref = monitor(process, whereis(Server)), Fun(), receive {'DOWN', Ref, _, _, _} -> diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_entry_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_entry_test.erl index 19ae24094b..8cc14d1c0d 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_entry_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_entry_test.erl @@ -53,7 +53,7 @@ cancel_and_replace_opener(_) -> true = ets:insert_new(?CACHE, #entry{key = Key}), {ok, Entry} = ddoc_cache_entry:start_link(Key, undefined), Opener1 = element(4, sys:get_state(Entry)), - Ref1 = erlang:monitor(process, Opener1), + Ref1 = monitor(process, Opener1), gen_server:cast(Entry, force_refresh), receive {'DOWN', Ref1, _, _, _} -> ok @@ -102,7 +102,7 @@ evict_when_not_accessed(_) -> Key = {ddoc_cache_entry_custom, {<<"bar">>, ?MODULE}}, true = ets:insert_new(?CACHE, #entry{key = Key}), {ok, Entry} = ddoc_cache_entry:start_link(Key, undefined), - Ref = erlang:monitor(process, Entry), + Ref = monitor(process, Entry), AccessCount1 = element(7, sys:get_state(Entry)), ?assertEqual(1, AccessCount1), ok = gen_server:cast(Entry, refresh), diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_lru_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_lru_test.erl index 81285e8551..e89d24d4ec 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_lru_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_lru_test.erl @@ -91,7 +91,7 @@ check_multi_start(_) -> ), [#entry{pid = Pid}] = ets:tab2list(?CACHE), Opener = element(4, sys:get_state(Pid)), - OpenerRef = erlang:monitor(process, Opener), + OpenerRef = monitor(process, Opener), ?assert(is_process_alive(Opener)), Opener ! go, receive @@ -135,7 +135,7 @@ check_multi_open(_) -> ), [#entry{pid = Pid}] = ets:tab2list(?CACHE), Opener = element(4, sys:get_state(Pid)), - OpenerRef = erlang:monitor(process, Opener), + OpenerRef = monitor(process, Opener), ?assert(is_process_alive(Opener)), Opener ! go, receive @@ -162,7 +162,7 @@ check_capped_size(_) -> meck:reset(ddoc_cache_ev), lists:foreach( fun(I) -> - DbName = list_to_binary("big_" ++ integer_to_list(I)), + DbName = <<"bin_", (integer_to_binary(I))/binary>>, ddoc_cache:open_custom(DbName, ?MODULE), meck:wait(I, ddoc_cache_ev, event, [started, '_'], ?EVENT_TIMEOUT), ?assert(cache_size() < MaxSize * 2) @@ -171,7 +171,7 @@ check_capped_size(_) -> ), lists:foreach( fun(I) -> - DbName = list_to_binary("big_" ++ integer_to_list(I)), + DbName = <<"bin_", (integer_to_binary(I))/binary>>, ddoc_cache:open_custom(DbName, ?MODULE), meck:wait(I, ddoc_cache_ev, event, [started, '_'], ?EVENT_TIMEOUT), ?assert(cache_size() < MaxSize * 2) @@ -184,7 +184,7 @@ check_cache_refill({DbName, _}) -> meck:reset(ddoc_cache_ev), InitDDoc = fun(I) -> - NumBin = list_to_binary(integer_to_list(I)), + NumBin = integer_to_binary(I), DDocId = <<"_design/", NumBin/binary>>, Doc = #doc{id = DDocId, body = {[]}}, {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]), @@ -242,7 +242,7 @@ check_evict_and_exit(_) -> ?assertEqual({ok, <<"dbname">>}, ddoc_cache_lru:open(Key)), [#entry{key = Key, pid = Pid}] = ets:tab2list(?CACHE), - erlang:monitor(process, whereis(ddoc_cache_lru)), + monitor(process, whereis(ddoc_cache_lru)), % Pause the LRU so we can queue multiple messages erlang:suspend_process(whereis(ddoc_cache_lru)), diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_open_error_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_open_error_test.erl index a92f898be8..15f7461afe 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_open_error_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_open_error_test.erl @@ -18,7 +18,7 @@ start_couch() -> Ctx = ddoc_cache_tutil:start_couch(), meck:expect(fabric, open_doc, fun(_, ?FOOBAR, _) -> - erlang:error(test_kaboom) + error(test_kaboom) end), Ctx. diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_open_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_open_test.erl index 778ef6cbb6..59e7d6311f 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_open_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_open_test.erl @@ -29,7 +29,7 @@ ddocid(_) -> no_ddocid. recover({deleted, _DbName}) -> - erlang:error(database_does_not_exist); + error(database_does_not_exist); recover(DbName) -> ddoc_cache_entry_validation_funs:recover(DbName). diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_remove_test.erl b/src/ddoc_cache/test/eunit/ddoc_cache_remove_test.erl index 3186bbd631..dd5638dbbb 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_remove_test.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_remove_test.erl @@ -29,7 +29,7 @@ recover(DbName) -> <<"not_ok">> -> {ruh, roh}; <<"error">> -> - erlang:error(thpppt) + error(thpppt) end. start_couch() -> @@ -193,7 +193,7 @@ do_compact(ShardName) -> {ok, Db} = couch_db:open_int(ShardName, []), try {ok, Pid} = couch_db:start_compact(Db), - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), receive {'DOWN', Ref, _, _, _} -> ok diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl b/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl index 156472265e..5e2efbffac 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl @@ -50,7 +50,7 @@ clear() -> application:start(ddoc_cache). get_rev(DbName, DDocId) -> - {_, Ref} = erlang:spawn_monitor(fun() -> + {_, Ref} = spawn_monitor(fun() -> {ok, #doc{revs = Revs}} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]), {Depth, [RevId | _]} = Revs, exit({Depth, RevId}) diff --git a/src/docs/src/api/database/changes.rst b/src/docs/src/api/database/changes.rst index b52ee161a1..7a5334e2fd 100644 --- a/src/docs/src/api/database/changes.rst +++ b/src/docs/src/api/database/changes.rst @@ -108,8 +108,9 @@ the filtering criteria. :query number timeout: Maximum period in *milliseconds* to wait for a change before the response is sent, even if there are no results. - Only applicable for :ref:`longpoll ` or - :ref:`continuous ` feeds. + Only applicable for :ref:`longpoll `, + :ref:`continuous ` or + :ref:`eventsource ` feeds. Default value is specified by :config:option:`chttpd/changes_timeout` configuration option. Note that ``60000`` value is also the default maximum timeout to prevent undetected dead connections. diff --git a/src/docs/src/api/server/authn.rst b/src/docs/src/api/server/authn.rst index 67e59722dd..7b89a87bde 100644 --- a/src/docs/src/api/server/authn.rst +++ b/src/docs/src/api/server/authn.rst @@ -516,3 +516,94 @@ https://auth0.com/blog/critical-vulnerabilities-in-json-web-token-libraries/). Note that you don't need to request :ref:`session ` to be authenticated by this method if the required HTTP header is provided. + +Two-Factor Authentication (2FA) +=============================== + +CouchDB supports built-in time-based one-time password (TOTP) authentication, so that 2FA +can be enabled for any user without extra plugins or tools. Here’s how it can be set up. + +Setting up 2FA in CouchDB +------------------------- + +1. Generate random token key + +A random `base32`_ string is generated and used as the user’s TOTP secret. For `example`_, +the following command produces a secure, random key: + +.. _base32: https://en.wikipedia.org/wiki/Base32 +.. _example: https://support.yubico.com/hc/en-us/articles/360015668699-Generating-Base32-string-examples + +.. code-block:: bash + + LC_ALL=C tr -dc 'A-Z2-7' ` in replication documents. ``doc_ids``, ``filter``, and ``selector`` are mutually exclusive. :` for details. Confirms that the server is up, running, and ready to respond to requests. If :config:option:`maintenance_mode ` is - ``true`` or ``nolb``, the endpoint will return a 404 response. + ``true`` or ``nolb``, the endpoint will return a 404 response. The status field + in the response body also changes to reflect the current ``maintenance_mode`` + defaulting to ``ok``. + + If :config:option:`maintenance_mode ` is ``true`` the status + field is set to ``maintenance_mode``. + + If :config:option:`maintenance_mode ` is set to ``nolb`` the + status field is set to ``nolb``. :>header Content-Type: :mimetype:`application/json` :code 200: Request completed successfully diff --git a/src/docs/src/cluster/databases.rst b/src/docs/src/cluster/databases.rst index 1dcbc2ba81..8441d8026a 100644 --- a/src/docs/src/cluster/databases.rst +++ b/src/docs/src/cluster/databases.rst @@ -66,6 +66,9 @@ Add a key value pair of the form: "zone": "metro-dc-a" +Alternatively, you can set the ``COUCHDB_ZONE`` environment variable +on each node and CouchDB will configure this document for you on startup. + Do this for all of the nodes in your cluster. In your config file (``local.ini`` or ``default.ini``) on each node, define a diff --git a/src/docs/src/cluster/purging.rst b/src/docs/src/cluster/purging.rst index cc9087d897..af5d7556f4 100644 --- a/src/docs/src/cluster/purging.rst +++ b/src/docs/src/cluster/purging.rst @@ -141,10 +141,8 @@ These settings can be updated in the default.ini or local.ini: +-----------------------+--------------------------------------------+----------+ | Field | Description | Default | +=======================+============================================+==========+ -| max_document_id_number| Allowed maximum number of documents in one | 100 | -| | purge request | | +-----------------------+--------------------------------------------+----------+ -| max_revisions_number | Allowed maximum number of accumulated | 1000 | +| max_revisions_number | Allowed maximum number of accumulated | infinity | | | revisions in one purge request | | +-----------------------+--------------------------------------------+----------+ | allowed_purge_seq_lag | Beside purged_infos_limit, allowed | 100 | diff --git a/src/docs/src/cluster/sharding.rst b/src/docs/src/cluster/sharding.rst index 7b8ca52ff0..e98df513db 100644 --- a/src/docs/src/cluster/sharding.rst +++ b/src/docs/src/cluster/sharding.rst @@ -650,6 +650,9 @@ Do this for all of the nodes in your cluster. For example: "zone": "{zone-name}" }' +Alternatively, you can set the ``COUCHDB_ZONE`` environment variable +on each node and CouchDB will configure this document for you on startup. + In the local config file (``local.ini``) of each node, define a consistent cluster-wide setting like: @@ -669,7 +672,7 @@ when the database is created, using the same syntax as the ini file: .. code-block:: bash - curl -X PUT $COUCH_URL:5984/{db}?zone={zone} + curl -X PUT $COUCH_URL:5984/{db}?placement={zone} The ``placement`` argument may also be specified. Note that this *will* override the logic that determines the number of created replicas! diff --git a/src/docs/src/cluster/troubleshooting.rst b/src/docs/src/cluster/troubleshooting.rst index 798902251e..16803cc805 100644 --- a/src/docs/src/cluster/troubleshooting.rst +++ b/src/docs/src/cluster/troubleshooting.rst @@ -97,7 +97,7 @@ the ``--level`` option: %MEM RSS 0.3 25116 - [debug] Local RPC: erlang:nodes([]) [5000] + [debug] Local RPC: nodes([]) [5000] [debug] Local RPC: mem3:nodes([]) [5000] [warning] Cluster member node3@127.0.0.1 is not connected to this node. Please check whether it is down. [info] Process is using 0.3% of available RAM, totalling 25116 KB of real memory. diff --git a/src/docs/src/config/misc.rst b/src/docs/src/config/misc.rst index ff46197535..c5843680ba 100644 --- a/src/docs/src/config/misc.rst +++ b/src/docs/src/config/misc.rst @@ -66,6 +66,7 @@ UUIDs Configuration .. config:option:: algorithm :: Generation Algorithm .. versionchanged:: 1.3 Added ``utc_id`` algorithm. + .. versionchanged:: 3.5.1 Added ``uuid_v7`` algorithm. CouchDB provides various algorithms to generate the UUID values that are used for document `_id`'s by default:: @@ -158,6 +159,25 @@ UUIDs Configuration ] } + - ``uuid_v7``: UUID v7 string in hex. + + .. code-block:: javascript + + { + "uuids": [ + "0199d2456e7f7b0a9b7130f9a9db8bee", + "0199d2456e7f72dda9f758fcc259c5fc", + "0199d2456e7f751c80b461180f7c7717", + "0199d2456e7f7c569b317d53367ca45a", + "0199d2456e7f77bfbffe92682c9c8c69", + "0199d2456e7f703ea97286f3d976343e", + "0199d2456e7f7f729142ed3b2da9101f", + "0199d2456e7f7723905c1f91f40d54f5", + "0199d2456e7f7e40979c7e2e22ffeb6a", + "0199d2456e7f7a42b43acfcc1e18eb84" + ] + } + .. note:: **Impact of UUID choices:** the choice of UUID has a significant impact on the layout of the B-tree, prior to compaction. @@ -293,26 +313,17 @@ Configuration of Database Purge .. config:section:: purge :: Configuration of Database Purge - .. config:option:: max_document_id_number :: Allowed number of documents \ - per Delete-Request - - .. versionadded:: 3.0 - - Sets the maximum number of documents allowed in a single purge request:: - - [purge] - max_document_id_number = 100 - .. config:option:: max_revisions_number :: Allowed number of accumulated \ revisions per Purge-Request .. versionadded:: 3.0 + .. versionchanged:: 3.5.1 Sets the maximum number of accumulated revisions allowed in a single purge request:: [purge] - max_revisions_number = 1000 + max_revisions_number = infinity .. config:option:: index_lag_warn_seconds :: Allowed duration for purge \ checkpoint document diff --git a/src/docs/src/config/query-servers.rst b/src/docs/src/config/query-servers.rst index 285bf12807..9932c49ab3 100644 --- a/src/docs/src/config/query-servers.rst +++ b/src/docs/src/config/query-servers.rst @@ -139,7 +139,9 @@ Query Servers Configuration .. config:option:: reduce_limit :: Reduce limit control Controls `Reduce overflow` error that raises when output of - :ref:`reduce functions ` is too big:: + :ref:`reduce functions ` is too big. The possible values are + ``true``, ``false`` or ``log``. The ``log`` value will log a warning + instead of crashing the view :: [query_server_config] reduce_limit = true @@ -148,6 +150,16 @@ Query Servers Configuration option since main propose of `reduce` functions is to *reduce* the input. + .. config:option:: reduce_limit_threshold :: Reduce limit threshold + + The number of bytes a reduce result must exceed to trigger the ``reduce_limit`` + control. Defaults to 5000. + + .. config:option:: reduce_limit_ratio :: Reduce limit ratio + + The ratio of input/output that must be exceeded to trigger the ``reduce_limit`` + control. Defaults to 2.0. + .. _config/native_query_servers: Native Erlang Query Server diff --git a/src/docs/src/config/replicator.rst b/src/docs/src/config/replicator.rst index 2fb3b2ca6f..bca107ae3b 100644 --- a/src/docs/src/config/replicator.rst +++ b/src/docs/src/config/replicator.rst @@ -303,7 +303,9 @@ Replicator Database Configuration .. config:option:: verify_ssl_certificates :: Check peer certificates - Set to true to validate peer certificates:: + Set to true to validate peer certificates. If + ``ssl_trusted_certificates_file`` is set it will be used, otherwise the + operating system CA files will be used:: [replicator] verify_ssl_certificates = false @@ -325,6 +327,18 @@ Replicator Database Configuration [replicator] ssl_certificate_max_depth = 3 + .. config:option:: cacert_reload_interval_hours :: CA certificates reload interval + + How often to reload operating system CA certificates (in hours). + Erlang VM caches OS CA certificates in memory after they are loaded + the first time. This setting specifies how often to clear the cache + and force reload certificate from disk. This can be useful if the VM + node is up for a long time, and the the CA certificate files are + updated using operating system packaging system during that time:: + + [replicator] + cacert_reload_interval_hours = 24 + .. config:option:: auth_plugins :: List of replicator client authentication plugins .. versionadded:: 2.2 diff --git a/src/docs/src/config/resharding.rst b/src/docs/src/config/resharding.rst index 91531aa7d3..968b24b498 100644 --- a/src/docs/src/config/resharding.rst +++ b/src/docs/src/config/resharding.rst @@ -21,7 +21,7 @@ Resharding Resharding Configuration ======================== -.. config:section:: resharding :: Resharding Configuration +.. config:section:: reshard :: Resharding Configuration .. config:option:: max_jobs :: Maximum resharding jobs per node diff --git a/src/docs/src/config/scanner.rst b/src/docs/src/config/scanner.rst index 4be3d9c06c..f36619f4ac 100644 --- a/src/docs/src/config/scanner.rst +++ b/src/docs/src/config/scanner.rst @@ -85,6 +85,25 @@ Scanner Options [couch_scanner] doc_rate_limit = 1000 + .. config:option:: doc_write_rate_limit + + Limit the rate at which plugins update documents. This rate limit + applies to plugins which explicitly use the + ``couch_scanner_rate_limiter`` module for rate limiting :: + + [couch_scanner] + doc_write_rate_limit = 500 + + .. config:option:: ddoc_batch_size + + Batch size to use when fetching design documents. For lots of small + design documents this value could be increased to 500 or 1000. If + design documents are large (100KB+) it could make sense to decrease it + a bit to 25 or 10. :: + + [couch_scanner] + ddoc_batch_size = 100 + .. config:section:: couch_scanner_plugins :: Enable Scanner Plugins .. config:option:: {plugin} diff --git a/src/docs/src/ddocs/ddocs.rst b/src/docs/src/ddocs/ddocs.rst index 4d3c06633f..fcb741e5fa 100644 --- a/src/docs/src/ddocs/ddocs.rst +++ b/src/docs/src/ddocs/ddocs.rst @@ -152,6 +152,9 @@ that the main task of reduce functions is to *reduce* the mapped result, not to make it bigger. Generally, your reduce function should converge rapidly to a single value - which could be an array or similar object. +Set ``reduce_limit`` to ``log`` so views which would crash if the setting were +``true`` would instead return the result and log an ``info`` level warning. + .. _reducefun/builtin: Built-in Reduce Functions diff --git a/src/docs/src/install/nouveau.rst b/src/docs/src/install/nouveau.rst index bfa40d160c..0dc031914f 100644 --- a/src/docs/src/install/nouveau.rst +++ b/src/docs/src/install/nouveau.rst @@ -107,21 +107,22 @@ if the Nouveau server's certificate is signed by a private CA. Configuring Nouveau to authenticate clients ------------------------------------------- -Nouveau is built on the dropwizard framework, which directly supports the `HTTPS -` transports. +Nouveau is built on the dropwizard framework, which directly supports the +`HTTPS `_ +transport. Acquiring or generating client and server certificates are out of scope of this documentation and we assume they have been created from here onward. We further assume the user can construct a Java keystore. -in ``nouveau.yaml`` you should remove all connectors of type ``http`` and add new -ones using ``https``; +in ``nouveau.yaml`` you should remove all connectors of type ``h2c`` and add new +ones using ``h2``; .. code-block:: yaml server: applicationConnectors: - - type: https + - type: h2 port: 5987 keyStorePath: keyStorePassword: diff --git a/src/docs/src/install/troubleshooting.rst b/src/docs/src/install/troubleshooting.rst index fc3b0986df..f8d13eefd2 100644 --- a/src/docs/src/install/troubleshooting.rst +++ b/src/docs/src/install/troubleshooting.rst @@ -228,6 +228,16 @@ unlimited. A detailed discussion can be found on the erlang-questions list, but the short answer is that you should decrease ``ulimit -n`` or lower the ``vm.args`` parameter ``+Q`` to something reasonable like 1024. +The same issue can be manifested in some docker configurations which default +ulimit max files to ``unlimited``. In those cases it's also recommended to set +a lower ``nofile`` ulimit, something like 65536. In docker compose files that +can be done as: + +.. code-block:: yaml + + ulimits: + nofiles: 65535 + Function raised exception (Cannot encode 'undefined' value as JSON) ------------------------------------------------------------------- If you see this in the CouchDB error logs, the JavaScript code you are using diff --git a/src/docs/src/install/unix.rst b/src/docs/src/install/unix.rst index 0d60ec52d9..616ef46e6c 100644 --- a/src/docs/src/install/unix.rst +++ b/src/docs/src/install/unix.rst @@ -34,7 +34,6 @@ to install CouchDB is to use the convenience binary packages: * CentOS/RHEL 9 (with caveats: depends on EPEL repository) * Debian 11 (bullseye) * Debian 12 (bookworm) -* Ubuntu 20.04 (focal) * Ubuntu 22.04 (jammy) * Ubuntu 24.04 (noble) @@ -157,7 +156,7 @@ Dependencies You should have the following installed: -* `Erlang OTP (25, 26, 27) `_ +* `Erlang OTP (26, 27, 28) `_ * `ICU `_ * `OpenSSL `_ * `Mozilla SpiderMonkey (1.8.5, 60, 68, 78, 91, 102, 115, 128) `_ diff --git a/src/docs/src/replication/protocol.rst b/src/docs/src/replication/protocol.rst index d08d7eae21..9f967f6ee7 100644 --- a/src/docs/src/replication/protocol.rst +++ b/src/docs/src/replication/protocol.rst @@ -280,7 +280,7 @@ and well handled: .. code-block:: http - HTTP/1.1 500 Internal Server Error + HTTP/1.1 401 Unauthorized Cache-Control: must-revalidate Content-Length: 108 Content-Type: application/json diff --git a/src/docs/src/setup/cluster.rst b/src/docs/src/setup/cluster.rst index a1aa087bcd..4e6a3fef98 100644 --- a/src/docs/src/setup/cluster.rst +++ b/src/docs/src/setup/cluster.rst @@ -303,7 +303,7 @@ After that we can join all the nodes together. Choose one node as the "setup coordination node" to run all these commands on. This "setup coordination node" only manages the setup and requires all other nodes to be able to see it and vice versa. *It has no special purpose beyond the setup process; CouchDB -does not have the concept of a "master" node in a cluster.* +does not have the concept of a primary node in a cluster.* Setup will not work with unavailable nodes. All nodes must be online and properly preconfigured before the cluster setup process can begin. diff --git a/src/docs/src/whatsnew/3.5.rst b/src/docs/src/whatsnew/3.5.rst index 02124d6110..b4c560f964 100644 --- a/src/docs/src/whatsnew/3.5.rst +++ b/src/docs/src/whatsnew/3.5.rst @@ -20,6 +20,151 @@ :depth: 1 :local: +.. _release/3.5.1: + +Version 3.5.1 +============= + +Features +-------- + +* :ghissue:`5626`, :ghissue:`5665`: Debian Trixie support +* :ghissue:`5709`: Automatic Nouveau and Clouseau index cleanup +* :ghissue:`5697`: Add UUID v7 as a ``uuid`` algorithm option. The default is + still the default ``sequential`` algorithm. +* :ghissue:`5713`, :ghissue:`5697`, :ghissue:`5701`, :ghissue:`5704`: Purge + improvements and fixes. Optimize it up to ~30% faster for large batches. + ``max_document_id_number`` setting was removed and ``max_revisions_number`` + set to ``unlimited`` by default to match ``_bulk_docs`` and ``_bulk_get`` + endpoints. +* :ghissue:`5611`: Implement the ability to downgrade CouchDB versions +* :ghissue:`5588`: Populate zone from ``COUCHDB_ZONE`` env variable in Docker +* :ghissue:`5563`: Set Erlang/OTP 26 as minimum supported version +* :ghissue:`5546`, :ghissue:`5641`: Improve Clouseau service checks in + ``clouseau_rpc`` module. +* :ghissue:`5639`: Use OS certificates for replication +* :ghissue:`5728`: Configurable reduce limit threshold and ratio + +Performance +----------- + +* :ghissue:`5625`: BTree engine term cache +* :ghissue:`5617`: Optimize Nouveau searches when index is fresh +* :ghissue:`5598`: Use HTTP/2 for Nouveau +* :ghissue:`5701`: Optimize revid parsing: 50-90% faster. Should help purge + requests as well as ``_bulk_docs`` and ``_bulk_get`` endpoints. +* :ghissue:`5564`: Use the built-in binary hex encode +* :ghissue:`5613`: Improve scanner performance +* :ghissue:`5545`: Bump process limit to 1M + +Bugfixes +-------- + +* :ghissue:`5722`, :ghissue:`5683`, :ghissue:`5678`, :ghissue:`5646`, + :ghissue:`5630`, :ghissue:`5615`, :ghissue:`5696`: Scanner fixes. Add write + limiting and switch to traversing documents by sequence IDs instead of by + document IDs. +* :ghissue:`5707`, :ghissue:`5706`, :ghissue:`5706`, :ghissue:`5694`, + :ghissue:`5691`, :ghissue:`5669`, :ghissue:`5629`, :ghissue:`5574`, + :ghissue:`5573`, :ghissue:`5566`, :ghissue:`5553`, :ghissue:`5550`, + :ghissue:`5534`, :ghissue:`5730`: QuickJS Updates. Optimized string operations, + faster context creation, a lot of bug fixes. +* :ghissue:`5719`: Use "all" ring options for purged_infos +* :ghissue:`5649`: Retry call to dreyfus index on noproc errors +* :ghissue:`5663`: More informative error if epochs out of order +* :ghissue:`5649`: Dreyfus retries on error +* :ghissue:`5643`: Fix reduce_limit = log feature +* :ghissue:`5620`: Use copy_props in the compactor instead of set_props +* :ghissue:`5632`, :ghissue:`5627`, :ghissue:`5607`: Nouveau fixes. Enhance + ``_nouveau_cleanup``. Improve security on http/2. +* :ghissue:`5614`: Stop replication jobs to nodes which are not part of the cluster +* :ghissue:`5596`: Fix query args parsing during cluster upgrades +* :ghissue:`5595`: Make replicator shutdown a bit more orderly +* :ghissue:`5595`: Avoid making a mess in the logs when stopping replicator app +* :ghissue:`5588`: Fix ``couch_util:set_value/3`` +* :ghissue:`5587`: Improve ``mem3_rep:find_source_seq/4`` logging +* :ghissue:`5586`: Don't wait indefinitely for replication jobs to stop +* :ghissue:`5578`: Use ``[sync]`` option in ``couch_bt_engine:commit_data/1`` +* :ghissue:`5556`: Add guards to ``fabric:design_docs/1`` to prevent + ``function_clause`` error +* :ghissue:`5555`: Improve replicator client mailbox flush +* :ghissue:`5551`: Handle ``bad_generator`` and ``case_clause`` in ``ken_server`` +* :ghissue:`5552`: Improve cluster startup logging +* :ghissue:`5552`: Improve mem3 supervisor +* :ghissue:`5552`: Handle shard opener tables not being initializes better +* :ghissue:`5549`: Don't spawn more than one ``init_delete_dir`` instance +* :ghissue:`5535`: Disk monitor always allows ``mem3_rep`` checkpoints +* :ghissue:`5536`: Fix ``mem3_util`` overlapping shards +* :ghissue:`5533`: No cfile support for 32bit systems +* :ghissue:`5688`: Handle timeout in ``dreyfus_fabric_search`` +* :ghissue:`5548`: Fix config key typo in mem3_reshard_dbdoc +* :ghissue:`5540`: Ignore extraneous cookie in replicator session plugin + +Cleanups +-------- + +* :ghissue:`5717`: Do not check for Dreyfus. It's part of the tree now. +* :ghissue:`5715`: Remove Hastings references +* :ghissue:`5714`: Cleanup fabric r/w parameter handling +* :ghissue:`5693`: Remove explicit erlang module prefix for auto-imported functions +* :ghissue:`5686`: Remove ``erlang:`` prefix from ``erlang:error()`` +* :ghissue:`5686`: Fix ``case_clause`` when got ``missing_target`` error +* :ghissue:`5690`: Fix props caching in mem3 +* :ghissue:`5680`: Implement db doc updating +* :ghissue:`5666`: Replace ``gen_server:format_status/2`` with ``format_status/1`` +* :ghissue:`5672`: Cache and store mem3 shard properties in one place only +* :ghissue:`5644`: Remove redundant ``*_to_list`` / ``list_to_*`` conversion +* :ghissue:`5633`: Use ``config:get_integer/3`` in couch_btree +* :ghissue:`5618`: DRY out ``couch_bt_engine`` header pointer term access +* :ghissue:`5614`: Stop replication jobs to nodes which are not part of the cluster +* :ghissue:`5610`: Add a ``range_to_hex/1`` utility function +* :ghissue:`5565`: Use maps comprehensions and generators in a few places +* :ghissue:`5649`: Remove pointless message +* :ghissue:`5649`: Remove obsolete clauses from dreyfus +* :ghissue:`5621`: Minor couch_btree refactoring + +Docs +---- + +* :ghissue:`5705`: Docs: Update the /_up endpoint docs to include status responses +* :ghissue:`5653`: Document that _all_dbs endpoint supports inclusive_end query param +* :ghissue:`5575`: Document how to mitigate high memory usage in docker +* :ghissue:`5600`: Avoid "master" wording at setup cluster +* :ghissue:`5381`: Change unauthorized example to 401 for replication +* :ghissue:`5682`: Update install instructions +* :ghissue:`5674`: Add setup documentation for two factor authentication +* :ghissue:`5562`: Add AI policy +* :ghissue:`5548`: Fix reshard doc section name +* :ghissue:`5543`: Add ``https`` to allowed replication proxy protocols + +Tests/CI/Builds +--------------- + +* :ghissue:`5720`: Update deps: Fauxton, meck and PropEr +* :ghissue:`5708`: Improve search test +* :ghissue:`5702`: Increase timeout for `process_response/3` to fix flaky tests +* :ghissue:`5703`: Use deterministic doc IDs in Mango key test +* :ghissue:`5692`: Implement 'assert_on_status' macro +* :ghissue:`5684`: Sequester docker ARM builds and fail early +* :ghissue:`5679`: Add ``--disable-spidermonkey`` to ``--dev[-with-nouveau]`` +* :ghissue:`5671`: Print request/response body on errors from mango test suite +* :ghissue:`5670`: Fix ``make clean`` after ``dev/run --enable-tls`` +* :ghissue:`5668`: Update xxHash +* :ghissue:`5667`: Update mochiweb to v3.3.0 +* :ghissue:`5664`: Disable ppc64le and s390x builds +* :ghissue:`5604`: Use ASF fork of ``gun`` for ``cowlib`` dependency +* :ghissue:`5636`: Reduce btree prop test count a bit +* :ghissue:`5633`: Fix and improve couch_btree testing +* :ghissue:`5572`: Remove a few more instances of Ubuntu Focal +* :ghissue:`5571`: Upgrade Erlang for CI +* :ghissue:`5570`: Skip macos CI for now and remove Ubuntu Focal +* :ghissue:`5488`: Bump Clouseau to 2.25.0 +* :ghissue:`5541`: Enable Clouseau for the Windows CI +* :ghissue:`5537`: Add retries to native full CI stage +* :ghissue:`5531`: Fix Erlang cookie configuration in ``dev/run`` +* :ghissue:`5662`: Remove old Jenkinsfiles +* :ghissue:`5661`: Unify CI jobs + .. _release/3.5.0: Version 3.5.0 diff --git a/src/dreyfus/src/clouseau_rpc.erl b/src/dreyfus/src/clouseau_rpc.erl index 12520da9a9..c036a5a9af 100644 --- a/src/dreyfus/src/clouseau_rpc.erl +++ b/src/dreyfus/src/clouseau_rpc.erl @@ -23,6 +23,7 @@ -export([analyze/2, version/0, disk_size/1]). -export([set_purge_seq/2, get_purge_seq/1, get_root_dir/0]). -export([connected/0]). +-export([check_service/1, check_service/2, check_services/0, check_services/1]). %% string represented as binary -type string_as_binary(_Value) :: nonempty_binary(). @@ -31,6 +32,7 @@ -type path() :: string_as_binary(_). -type error() :: any(). +-type throw(_Reason) :: no_return(). -type analyzer_name() :: string_as_binary(_). @@ -64,6 +66,8 @@ | {string_as_binary(stopwords), [field_name()]} ]. +-define(SEARCH_SERVICE_TIMEOUT, 2000). + -spec open_index(Peer :: pid(), Path :: shard(), Analyzer :: analyzer()) -> {ok, indexer_pid()} | error(). open_index(Peer, Path, Analyzer) -> @@ -258,10 +262,17 @@ rename(DbName) -> %% and an analyzer represented in a Javascript function in a design document. %% `Sig` is used to check if an index description is changed, %% and the index needs to be reconstructed. --spec cleanup(DbName :: string_as_binary(_), ActiveSigs :: [sig()]) -> +-spec cleanup(DbName :: string_as_binary(_), SigList :: list() | SigMap :: #{sig() => true}) -> ok. -cleanup(DbName, ActiveSigs) -> +% Compatibility clause to help when running search index cleanup during +% a mixed cluster state. Remove after version 3.6 +% +cleanup(DbName, SigList) when is_list(SigList) -> + SigMap = #{Sig => true || Sig <- SigList}, + cleanup(DbName, SigMap); +cleanup(DbName, #{} = SigMap) -> + ActiveSigs = maps:keys(SigMap), gen_server:cast({cleanup, clouseau()}, {cleanup, DbName, ActiveSigs}). %% a binary with value <<"tokens">> @@ -285,10 +296,25 @@ analyze(Analyzer, Text) -> version() -> rpc({main, clouseau()}, version). +-spec clouseau_major_vsn() -> binary() | throw({atom(), binary()}). +clouseau_major_vsn() -> + case version() of + {ok, <>} -> + <>; + {'EXIT', noconnection} -> + throw({noconnection, <<"Clouseau node is not connected.">>}); + {ok, null} -> + %% Backward compatibility: + %% If we run Clouseau from source code, remsh will return + %% `{ok, null}` for Clouseau <= 2.25.0. + %% See PR: https://github.com/cloudant-labs/clouseau/pull/106 + <<$2>> + end. + -spec connected() -> boolean(). connected() -> - HiddenNodes = erlang:nodes(hidden), + HiddenNodes = nodes(hidden), case lists:member(clouseau(), HiddenNodes) of true -> true; @@ -316,3 +342,61 @@ rpc(Ref, Msg) -> clouseau() -> list_to_atom(config:get("dreyfus", "name", "clouseau@127.0.0.1")). + +-type service() :: sup | main | analyzer | cleanup. +-type liveness_status() :: alive | timeout. + +-spec clouseau_services(MajorVsn) -> [service()] when + MajorVsn :: binary(). +clouseau_services(_MajorVsn) -> + [sup, main, analyzer, cleanup]. + +-spec is_valid(Service) -> boolean() when + Service :: atom(). +is_valid(Service) when is_atom(Service) -> + lists:member(Service, clouseau_services(clouseau_major_vsn())). + +-spec check_service(Service) -> Result when + Service :: service(), + Result :: {service(), liveness_status()} | throw({atom(), binary()}). +check_service(Service) -> + check_service(Service, ?SEARCH_SERVICE_TIMEOUT). + +-spec check_service(Service, Timeout) -> Result when + Service :: service(), + Timeout :: timeout(), + Result :: {service(), liveness_status()} | throw({atom(), binary()}). +check_service(Service, Timeout) when is_list(Service) -> + check_service(list_to_atom(Service), Timeout); +check_service(Service, Timeout) when is_atom(Service) -> + case is_valid(Service) of + true -> + Ref = make_ref(), + {Service, clouseau()} ! {ping, self(), Ref}, + receive + {pong, Ref} -> + {Service, alive} + after Timeout -> + {Service, timeout} + end; + false -> + NoService = atom_to_binary(Service), + throw({not_found, <<"no such service: ", NoService/binary>>}) + end. + +-spec check_services() -> Result when + Result :: [{service(), liveness_status()}] | throw({atom(), binary()}). +check_services() -> + check_services(?SEARCH_SERVICE_TIMEOUT). + +-spec check_services(Timeout) -> Result when + Timeout :: timeout(), + Result :: [{service(), liveness_status()}] | throw({atom(), binary()}). +check_services(Timeout) -> + case connected() of + true -> + Services = clouseau_services(clouseau_major_vsn()), + [check_service(S, Timeout) || S <- Services]; + false -> + throw({noconnection, <<"Clouseau node is not connected.">>}) + end. diff --git a/src/dreyfus/src/dreyfus_fabric_cleanup.erl b/src/dreyfus/src/dreyfus_fabric_cleanup.erl index e2710744d9..0488211be9 100644 --- a/src/dreyfus/src/dreyfus_fabric_cleanup.erl +++ b/src/dreyfus/src/dreyfus_fabric_cleanup.erl @@ -14,92 +14,50 @@ -module(dreyfus_fabric_cleanup). --include("dreyfus.hrl"). --include_lib("mem3/include/mem3.hrl"). --include_lib("couch/include/couch_db.hrl"). - --export([go/1]). +-export([go/1, go_local/3]). go(DbName) -> - {ok, DesignDocs} = fabric:design_docs(DbName), - ActiveSigs = lists:usort( - lists:flatmap( - fun active_sigs/1, - [couch_doc:from_json_obj(DD) || DD <- DesignDocs] - ) - ), - cleanup_local_purge_doc(DbName, ActiveSigs), - clouseau_rpc:cleanup(DbName, ActiveSigs), - ok. + case fabric_util:get_design_doc_records(DbName) of + {ok, DDocs} when is_list(DDocs) -> + Sigs = dreyfus_util:get_signatures_from_ddocs(DbName, DDocs), + Shards = mem3:shards(DbName), + ByNode = maps:groups_from_list(fun mem3:node/1, fun mem3:name/1, Shards), + Fun = fun(Node, Dbs, Acc) -> + erpc:send_request(Node, ?MODULE, go_local, [DbName, Dbs, Sigs], Node, Acc) + end, + Reqs = maps:fold(Fun, erpc:reqids_new(), ByNode), + recv(DbName, Reqs, fabric_util:abs_request_timeout()); + Error -> + couch_log:error("~p : error fetching ddocs db:~p ~p", [?MODULE, DbName, Error]), + Error + end. -active_sigs(#doc{body = {Fields}} = Doc) -> +% erpc endpoint for go/1 and fabric_index_cleanup:cleanup_indexes/2 +% +go_local(DbName, Dbs, #{} = Sigs) -> try - {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), - {IndexNames, _} = lists:unzip(RawIndexes), - [ - begin - {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName), - Index#index.sig - end - || IndexName <- IndexNames - ] + lists:foreach( + fun(Db) -> + Checkpoints = dreyfus_util:get_purge_checkpoints(Db), + ok = couch_index_util:cleanup_purges(Db, Sigs, Checkpoints) + end, + Dbs + ), + clouseau_rpc:cleanup(DbName, Sigs), + ok catch - error:{badmatch, _Error} -> - [] + error:database_does_not_exist -> + ok end. -cleanup_local_purge_doc(DbName, ActiveSigs) -> - {ok, BaseDir} = clouseau_rpc:get_root_dir(), - DbNamePattern = <>, - Pattern0 = filename:join([BaseDir, "shards", "*", DbNamePattern, "*"]), - Pattern = binary_to_list(iolist_to_binary(Pattern0)), - DirListStrs = filelib:wildcard(Pattern), - DirList = [iolist_to_binary(DL) || DL <- DirListStrs], - LocalShards = mem3:local_shards(DbName), - ActiveDirs = lists:foldl( - fun(LS, AccOuter) -> - lists:foldl( - fun(Sig, AccInner) -> - DirName = filename:join([BaseDir, LS#shard.name, Sig]), - [DirName | AccInner] - end, - AccOuter, - ActiveSigs - ) - end, - [], - LocalShards - ), - - DeadDirs = DirList -- ActiveDirs, - lists:foreach( - fun(IdxDir) -> - Sig = dreyfus_util:get_signature_from_idxdir(IdxDir), - case Sig of - undefined -> - ok; - _ -> - DocId = dreyfus_util:get_local_purge_doc_id(Sig), - LocalShards = mem3:local_shards(DbName), - lists:foreach( - fun(LS) -> - ShardDbName = LS#shard.name, - {ok, ShardDb} = couch_db:open_int(ShardDbName, []), - case couch_db:open_doc(ShardDb, DocId, []) of - {ok, LocalPurgeDoc} -> - couch_db:update_doc( - ShardDb, - LocalPurgeDoc#doc{deleted = true}, - [?ADMIN_CTX] - ); - {not_found, _} -> - ok - end, - couch_db:close(ShardDb) - end, - LocalShards - ) - end - end, - DeadDirs - ). +recv(DbName, Reqs, Timeout) -> + case erpc:receive_response(Reqs, Timeout, true) of + {ok, _Lable, Reqs1} -> + recv(DbName, Reqs1, Timeout); + {Error, Label, Reqs1} -> + ErrMsg = "~p : error cleaning dreyfus indexes db:~p req:~p error:~p", + couch_log:error(ErrMsg, [?MODULE, DbName, Label, Error]), + recv(DbName, Reqs1, Timeout); + no_request -> + ok + end. diff --git a/src/dreyfus/src/dreyfus_fabric_group1.erl b/src/dreyfus/src/dreyfus_fabric_group1.erl index 9b08a94ebe..990d6d24e8 100644 --- a/src/dreyfus/src/dreyfus_fabric_group1.erl +++ b/src/dreyfus/src/dreyfus_fabric_group1.erl @@ -63,8 +63,8 @@ go(DbName, DDoc, IndexName, #index_query_args{} = QueryArgs) -> #shard.ref, fun handle_message/3, State, - infinity, - 1000 * 60 * 60 + fabric_util:timeout("search", "infinity"), + fabric_util:timeout("search_permsg", "3600000") ) after rexi_monitor:stop(RexiMon), diff --git a/src/dreyfus/src/dreyfus_fabric_group2.erl b/src/dreyfus/src/dreyfus_fabric_group2.erl index 3059aa30ee..613ac6555d 100644 --- a/src/dreyfus/src/dreyfus_fabric_group2.erl +++ b/src/dreyfus/src/dreyfus_fabric_group2.erl @@ -68,8 +68,8 @@ go(DbName, DDoc, IndexName, #index_query_args{} = QueryArgs) -> #shard.ref, fun handle_message/3, State, - infinity, - 1000 * 60 * 60 + fabric_util:timeout("search", "infinity"), + fabric_util:timeout("search_permsg", "3600000") ) after rexi_monitor:stop(RexiMon), diff --git a/src/dreyfus/src/dreyfus_fabric_search.erl b/src/dreyfus/src/dreyfus_fabric_search.erl index 75a2a5a3bf..0d07db6ef9 100644 --- a/src/dreyfus/src/dreyfus_fabric_search.erl +++ b/src/dreyfus/src/dreyfus_fabric_search.erl @@ -142,7 +142,9 @@ go(DbName, DDoc, IndexName, QueryArgs, Counters, Bookmark, RingOpts) -> {ok, Bookmark1, TotalHits, Hits1, Counts, Ranges} end; {error, Reason} -> - {error, Reason} + {error, Reason}; + {timeout, _State} -> + {error, timeout} after rexi_monitor:stop(RexiMon), fabric_streams:cleanup(Workers) diff --git a/src/dreyfus/src/dreyfus_index.erl b/src/dreyfus/src/dreyfus_index.erl index c97a837d51..5295a0065f 100644 --- a/src/dreyfus/src/dreyfus_index.erl +++ b/src/dreyfus/src/dreyfus_index.erl @@ -22,13 +22,13 @@ % public api. -export([ start_link/2, - design_doc_to_index/2, + design_doc_to_index/3, await/2, search/2, info/1, group1/2, group2/2, - design_doc_to_indexes/1 + design_doc_to_indexes/2 ]). % gen_server api. @@ -87,14 +87,14 @@ to_index_pid(Pid) -> false -> Pid end. -design_doc_to_indexes(#doc{body = {Fields}} = Doc) -> +design_doc_to_indexes(DbName, #doc{body = {Fields}} = Doc) -> RawIndexes = couch_util:get_value(<<"indexes">>, Fields, {[]}), case RawIndexes of {IndexList} when is_list(IndexList) -> {IndexNames, _} = lists:unzip(IndexList), lists:flatmap( fun(IndexName) -> - case (catch design_doc_to_index(Doc, IndexName)) of + case (catch design_doc_to_index(DbName, Doc, IndexName)) of {ok, #index{} = Index} -> [Index]; _ -> [] end @@ -301,7 +301,7 @@ open_index(DbName, #index{analyzer = Analyzer, sig = Sig}) -> Error end. -design_doc_to_index(#doc{id = Id, body = {Fields}}, IndexName) -> +design_doc_to_index(DbName, #doc{id = Id, body = {Fields}}, IndexName) -> Language = couch_util:get_value(<<"language">>, Fields, <<"javascript">>), {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), InvalidDDocError = @@ -323,6 +323,7 @@ design_doc_to_index(#doc{id = Id, body = {Fields}}, IndexName) -> ) ), {ok, #index{ + dbname = DbName, analyzer = Analyzer, ddoc_id = Id, def = Def, diff --git a/src/dreyfus/src/dreyfus_rpc.erl b/src/dreyfus/src/dreyfus_rpc.erl index 2ebc5ffe58..3aed82d76c 100644 --- a/src/dreyfus/src/dreyfus_rpc.erl +++ b/src/dreyfus/src/dreyfus_rpc.erl @@ -17,6 +17,8 @@ -include("dreyfus.hrl"). -import(couch_query_servers, [get_os_process/1, ret_os_process/1, proc_prompt/2]). +-define(RETRY_DELAY, 2100). + % public api. -export([search/4, group1/4, group2/4, info/3, disk_size/3]). @@ -44,31 +46,34 @@ call(Fun, DbName, DDoc, IndexName, QueryArgs0) -> stale = Stale } = QueryArgs, {_LastSeq, MinSeq} = calculate_seqs(Db, Stale), - case dreyfus_index:design_doc_to_index(DDoc, IndexName) of + case dreyfus_index:design_doc_to_index(DbName, DDoc, IndexName) of {ok, Index} -> - case dreyfus_index_manager:get_index(DbName, Index) of - {ok, Pid} -> - case dreyfus_index:await(Pid, MinSeq) of - {ok, IndexPid, _Seq} -> - Result = dreyfus_index:Fun(IndexPid, QueryArgs), - rexi:reply(Result); - % obsolete clauses, remove after upgrade - ok -> - Result = dreyfus_index:Fun(Pid, QueryArgs), - rexi:reply(Result); - {ok, _Seq} -> - Result = dreyfus_index:Fun(Pid, QueryArgs), - rexi:reply(Result); - Error -> - rexi:reply(Error) - end; - Error -> - rexi:reply(Error) + try + rexi:reply(index_call(Fun, DbName, Index, QueryArgs, MinSeq)) + catch + exit:{noproc, _} -> + timer:sleep(?RETRY_DELAY), + %% try one more time to handle the case when Clouseau's LRU + %% closed the index in the middle of our call + rexi:reply(index_call(Fun, DbName, Index, QueryArgs, MinSeq)) end; Error -> rexi:reply(Error) end. +index_call(Fun, DbName, Index, QueryArgs, MinSeq) -> + case dreyfus_index_manager:get_index(DbName, Index) of + {ok, Pid} -> + case dreyfus_index:await(Pid, MinSeq) of + {ok, IndexPid, _Seq} -> + dreyfus_index:Fun(IndexPid, QueryArgs); + Error -> + Error + end; + Error -> + Error + end. + info(DbName, DDoc, IndexName) -> MFA = {?MODULE, info_int, [DbName, DDoc, IndexName]}, dreyfus_util:time([rpc, info], MFA). @@ -76,7 +81,7 @@ info(DbName, DDoc, IndexName) -> info_int(DbName, DDoc, IndexName) -> erlang:put(io_priority, {search, DbName}), check_interactive_mode(), - case dreyfus_index:design_doc_to_index(DDoc, IndexName) of + case dreyfus_index:design_doc_to_index(DbName, DDoc, IndexName) of {ok, Index} -> case dreyfus_index_manager:get_index(DbName, Index) of {ok, Pid} -> @@ -97,7 +102,7 @@ info_int(DbName, DDoc, IndexName) -> disk_size(DbName, DDoc, IndexName) -> erlang:put(io_priority, {search, DbName}), check_interactive_mode(), - case dreyfus_index:design_doc_to_index(DDoc, IndexName) of + case dreyfus_index:design_doc_to_index(DbName, DDoc, IndexName) of {ok, Index} -> Result = dreyfus_index_manager:get_disk_size(DbName, Index), rexi:reply(Result); diff --git a/src/dreyfus/src/dreyfus_util.erl b/src/dreyfus/src/dreyfus_util.erl index 301d3887ac..b8806c0893 100644 --- a/src/dreyfus/src/dreyfus_util.erl +++ b/src/dreyfus/src/dreyfus_util.erl @@ -25,9 +25,11 @@ ensure_local_purge_docs/2, get_value_from_options/2, get_local_purge_doc_id/1, + get_purge_checkpoints/1, get_local_purge_doc_body/4, maybe_create_local_purge_doc/2, maybe_create_local_purge_doc/3, + get_signatures_from_ddocs/2, get_signature_from_idxdir/1, verify_index_exists/2 ]). @@ -241,7 +243,7 @@ export(QueryArgs) -> time(Metric, {M, F, A}) when is_list(Metric) -> Start = os:timestamp(), try - erlang:apply(M, F, A) + apply(M, F, A) after Length = timer:now_diff(os:timestamp(), Start) / 1000, couch_stats:update_histogram([dreyfus | Metric], Length) @@ -305,7 +307,7 @@ ensure_local_purge_docs(DbName, DDocs) -> undefined -> false; _ -> - try dreyfus_index:design_doc_to_indexes(DDoc) of + try dreyfus_index:design_doc_to_indexes(DbName, DDoc) of SIndexes -> ensure_local_purge_doc(Db, SIndexes) catch _:_ -> @@ -360,6 +362,32 @@ maybe_create_local_purge_doc(Db, IndexPid, Index) -> get_local_purge_doc_id(Sig) -> ?l2b(?LOCAL_DOC_PREFIX ++ "purge-" ++ "dreyfus-" ++ Sig). +% Returns a map of `Sig => DocId` elements for all the purge view +% checkpoint docs. Sig is a hex-encoded binary. +% +get_purge_checkpoints(Db) -> + couch_index_util:get_purge_checkpoints(Db, <<"dreyfus">>). + +get_signatures_from_ddocs(DbName, DesignDocs) -> + SigList = lists:flatmap(fun(Doc) -> active_sigs(DbName, Doc) end, DesignDocs), + #{Sig => true || Sig <- SigList}. + +active_sigs(DbName, #doc{body = {Fields}} = Doc) -> + try + {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), + {IndexNames, _} = lists:unzip(RawIndexes), + [ + begin + {ok, Index} = dreyfus_index:design_doc_to_index(DbName, Doc, IndexName), + Index#index.sig + end + || IndexName <- IndexNames + ] + catch + error:{badmatch, _Error} -> + [] + end. + get_signature_from_idxdir(IdxDir) -> IdxDirList = filename:split(IdxDir), Sig = lists:last(IdxDirList), @@ -415,7 +443,7 @@ verify_index_exists(DbName, Props) -> case couch_db:get_design_doc(Db, DDocId) of {ok, #doc{} = DDoc} -> {ok, IdxState} = dreyfus_index:design_doc_to_index( - DDoc, IndexName + DbName, DDoc, IndexName ), IdxState#index.sig == Sig; {not_found, _} -> diff --git a/src/dreyfus/test/eunit/dreyfus_purge_test.erl b/src/dreyfus/test/eunit/dreyfus_purge_test.erl index bed1f79f8c..a7c0068e01 100644 --- a/src/dreyfus/test/eunit/dreyfus_purge_test.erl +++ b/src/dreyfus/test/eunit/dreyfus_purge_test.erl @@ -1085,7 +1085,7 @@ wait_for_replicate(DbName, DocIds, ExpectRevCount, TimeOut) when wait_for_replicate(DbName, DocId, ExpectRevCount, TimeOut) -> FDI = fabric:get_full_doc_info(DbName, DocId, []), #doc_info{revs = Revs} = couch_doc:to_doc_info(FDI), - case erlang:length(Revs) of + case length(Revs) of ExpectRevCount -> couch_log:notice( "[~p] wait end by expect, time used:~p, DocId:~p", @@ -1102,17 +1102,17 @@ get_sigs(DbName) -> {ok, DesignDocs} = fabric:design_docs(DbName), lists:usort( lists:flatmap( - fun active_sigs/1, + fun(Doc) -> active_sigs(DbName, Doc) end, [couch_doc:from_json_obj(DD) || DD <- DesignDocs] ) ). -active_sigs(#doc{body = {Fields}} = Doc) -> +active_sigs(DbName, #doc{body = {Fields}} = Doc) -> {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), {IndexNames, _} = lists:unzip(RawIndexes), [ begin - {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName), + {ok, Index} = dreyfus_index:design_doc_to_index(DbName, Doc, IndexName), Index#index.sig end || IndexName <- IndexNames diff --git a/src/ets_lru/src/ets_lru.erl b/src/ets_lru/src/ets_lru.erl index 080c8f7e2b..15a23f2773 100644 --- a/src/ets_lru/src/ets_lru.erl +++ b/src/ets_lru/src/ets_lru.erl @@ -349,7 +349,7 @@ next_timeout(Tab, Now, Max) -> infinity; {Time, _} -> TimeDiff = Now - Time, - erlang:max(Max - TimeDiff, 0) + max(Max - TimeDiff, 0) end. set_options(St, []) -> diff --git a/src/ets_lru/test/ets_lru_test.erl b/src/ets_lru/test/ets_lru_test.erl index 25a51fa9cd..6a7e5fcd0b 100644 --- a/src/ets_lru/test/ets_lru_test.erl +++ b/src/ets_lru/test/ets_lru_test.erl @@ -345,7 +345,7 @@ insert_kvs(Info, LRU, Count, Limit) -> timer:sleep(1), case ets:info(lru_objects, Info) > Limit of true -> - erlang:error(exceeded_limit); + error(exceeded_limit); false -> true end; @@ -355,7 +355,7 @@ insert_kvs(Info, LRU, Count, Limit) -> insert_kvs(Info, LRU, Count - 1, Limit). stop_lru({ok, LRU}) -> - Ref = erlang:monitor(process, LRU), + Ref = monitor(process, LRU), ets_lru:stop(LRU), receive {'DOWN', Ref, process, LRU, Reason} -> Reason diff --git a/src/exxhash/README.md b/src/exxhash/README.md index b99fa8605d..59ca6a2650 100644 --- a/src/exxhash/README.md +++ b/src/exxhash/README.md @@ -22,7 +22,7 @@ Updating xxHash was originally vendored from https://cyan4973.github.io/xxHash/ with commit SHA f4bef929aa854e9f52a303c5e58fd52855a0ecfa -Updated on 2025-04-30 from commit 41fea3d9ac7881c78fdc4003626977aa073bb906 +Updated on 2025-09-17 from commit c961fbe61ad1ee1e430b9c304735a0534fda1c6d Only these two files are used from the original library: `c_src/xxhash.h` diff --git a/src/exxhash/c_src/xxhash.h b/src/exxhash/c_src/xxhash.h index 66364b66f9..1b975455fa 100644 --- a/src/exxhash/c_src/xxhash.h +++ b/src/exxhash/c_src/xxhash.h @@ -791,18 +791,9 @@ XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canoni #endif /*! @endcond */ -/*! @cond Doxygen ignores this part */ -/* - * C23 __STDC_VERSION__ number hasn't been specified yet. For now - * leave as `201711L` (C17 + 1). - * TODO: Update to correct value when its been specified. - */ -#define XXH_C23_VN 201711L -/*! @endcond */ - /*! @cond Doxygen ignores this part */ /* C-language Attributes are added in C23. */ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 202311L) && defined(__has_c_attribute) # define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) #else # define XXH_HAS_C_ATTRIBUTE(x) 0 @@ -1126,7 +1117,7 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const # define XXH_SVE 6 /*!< SVE for some ARMv8-A and ARMv9-A */ # define XXH_LSX 7 /*!< LSX (128-bit SIMD) for LoongArch64 */ # define XXH_LASX 8 /*!< LASX (256-bit SIMD) for LoongArch64 */ - +# define XXH_RVV 9 /*!< RVV (RISC-V Vector) for RISC-V */ /*-********************************************************************** * XXH3 64-bit variant @@ -2661,7 +2652,7 @@ typedef union { xxh_u32 u32; } __attribute__((__packed__)) unalign; #endif static xxh_u32 XXH_read32(const void* ptr) { - typedef __attribute__((__aligned__(1))) xxh_u32 xxh_unalign32; + typedef __attribute__((__aligned__(1))) __attribute__((__may_alias__)) xxh_u32 xxh_unalign32; return *((const xxh_unalign32*)ptr); } @@ -2753,7 +2744,7 @@ static int XXH_isLittleEndian(void) * additional case: * * ``` - * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 202311L) * # include * # ifdef unreachable * # define XXH_UNREACHABLE() unreachable() @@ -3374,7 +3365,7 @@ typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((__packed__)) unalign6 #endif static xxh_u64 XXH_read64(const void* ptr) { - typedef __attribute__((__aligned__(1))) xxh_u64 xxh_unalign64; + typedef __attribute__((__aligned__(1))) __attribute__((__may_alias__)) xxh_u64 xxh_unalign64; return *((const xxh_unalign64*)ptr); } @@ -3882,6 +3873,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can # include # elif defined(__loongarch_sx) # include +# elif defined(__riscv_vector) +# include # endif #endif @@ -4020,6 +4013,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can # define XXH_VECTOR XXH_LASX # elif defined(__loongarch_sx) # define XXH_VECTOR XXH_LSX +# elif defined(__riscv_vector) +# define XXH_VECTOR XXH_RVV # else # define XXH_VECTOR XXH_SCALAR # endif @@ -4061,6 +4056,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can # define XXH_ACC_ALIGN 64 # elif XXH_VECTOR == XXH_LSX /* lsx */ # define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_RVV /* rvv */ +# define XXH_ACC_ALIGN 64 /* could be 8, but 64 may be faster */ # endif #endif @@ -4069,6 +4066,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can # define XXH_SEC_ALIGN XXH_ACC_ALIGN #elif XXH_VECTOR == XXH_SVE # define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_RVV +# define XXH_SEC_ALIGN XXH_ACC_ALIGN #else # define XXH_SEC_ALIGN 8 #endif @@ -5273,10 +5272,18 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTR (void)(&XXH_writeLE64); { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); -# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 - /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ - XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; - __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER <= 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 + * and some specific variants of 2015 may also lack it */ + /* Cast to unsigned 64-bit first to avoid signed arithmetic issues */ + xxh_u64 const seed64_unsigned = (xxh_u64)seed64; + xxh_u64 const neg_seed64 = (xxh_u64)(0ULL - seed64_unsigned); + __m128i const seed = _mm_set_epi32( + (int)(neg_seed64 >> 32), /* high 32 bits of negated seed */ + (int)(neg_seed64), /* low 32 bits of negated seed */ + (int)(seed64_unsigned >> 32), /* high 32 bits of original seed */ + (int)(seed64_unsigned) /* low 32 bits of original seed */ + ); # else __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); # endif @@ -5714,8 +5721,9 @@ XXH3_accumulate_512_lsx( void* XXH_RESTRICT acc, __m128i* const xacc = (__m128i *) acc; const __m128i* const xinput = (const __m128i *) input; const __m128i* const xsecret = (const __m128i *) secret; + size_t i; - for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { + for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { /* data_vec = xinput[i]; */ __m128i const data_vec = __lsx_vld(xinput + i, 0); /* key_vec = xsecret[i]; */ @@ -5745,8 +5753,9 @@ XXH3_scrambleAcc_lsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) __m128i* const xacc = (__m128i*) acc; const __m128i* const xsecret = (const __m128i *) secret; const __m128i prime32 = __lsx_vreplgr2vr_d(XXH_PRIME32_1); + size_t i; - for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { + for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { /* xacc[i] ^= (xacc[i] >> 47) */ __m128i const acc_vec = xacc[i]; __m128i const shifted = __lsx_vsrli_d(acc_vec, 47); @@ -5773,11 +5782,12 @@ XXH3_accumulate_512_lasx( void* XXH_RESTRICT acc, { XXH_ASSERT((((size_t)acc) & 31) == 0); { + size_t i; __m256i* const xacc = (__m256i *) acc; const __m256i* const xinput = (const __m256i *) input; const __m256i* const xsecret = (const __m256i *) secret; - for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { + for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { /* data_vec = xinput[i]; */ __m256i const data_vec = __lasx_xvld(xinput + i, 0); /* key_vec = xsecret[i]; */ @@ -5807,8 +5817,9 @@ XXH3_scrambleAcc_lasx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) __m256i* const xacc = (__m256i*) acc; const __m256i* const xsecret = (const __m256i *) secret; const __m256i prime32 = __lasx_xvreplgr2vr_d(XXH_PRIME32_1); + size_t i; - for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { + for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { /* xacc[i] ^= (xacc[i] >> 47) */ __m256i const acc_vec = xacc[i]; __m256i const shifted = __lasx_xvsrli_d(acc_vec, 47); @@ -5825,6 +5836,133 @@ XXH3_scrambleAcc_lasx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) #endif +#if (XXH_VECTOR == XXH_RVV) + #define XXH_CONCAT2(X, Y) X ## Y + #define XXH_CONCAT(X, Y) XXH_CONCAT2(X, Y) +#if ((defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 13) || \ + (defined(__clang__) && __clang_major__ < 16)) + #define XXH_RVOP(op) op + #define XXH_RVCAST(op) XXH_CONCAT(vreinterpret_v_, op) +#else + #define XXH_RVOP(op) XXH_CONCAT(__riscv_, op) + #define XXH_RVCAST(op) XXH_CONCAT(__riscv_vreinterpret_v_, op) +#endif +XXH_FORCE_INLINE void +XXH3_accumulate_512_rvv( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + { + // Try to set vector lenght to 512 bits. + // If this length is unavailable, then maximum available will be used + size_t vl = XXH_RVOP(vsetvl_e64m2)(8); + + uint64_t* xacc = (uint64_t*) acc; + const uint64_t* xinput = (const uint64_t*) input; + const uint64_t* xsecret = (const uint64_t*) secret; + static const uint64_t swap_mask[16] = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14}; + vuint64m2_t xswap_mask = XXH_RVOP(vle64_v_u64m2)(swap_mask, vl); + + size_t i; + for (i = 0; i < XXH_STRIPE_LEN/8; i += vl) { + /* data_vec = xinput[i]; */ + vuint64m2_t data_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)((const uint8_t*)(xinput + i), vl * 8)); + /* key_vec = xsecret[i]; */ + vuint64m2_t key_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)((const uint8_t*)(xsecret + i), vl * 8)); + /* acc_vec = xacc[i]; */ + vuint64m2_t acc_vec = XXH_RVOP(vle64_v_u64m2)(xacc + i, vl); + /* data_key = data_vec ^ key_vec; */ + vuint64m2_t data_key = XXH_RVOP(vxor_vv_u64m2)(data_vec, key_vec, vl); + /* data_key_hi = data_key >> 32; */ + vuint64m2_t data_key_hi = XXH_RVOP(vsrl_vx_u64m2)(data_key, 32, vl); + /* data_key_lo = data_key & 0xffffffff; */ + vuint64m2_t data_key_lo = XXH_RVOP(vand_vx_u64m2)(data_key, 0xffffffff, vl); + /* swap high and low halves */ + vuint64m2_t data_swap = XXH_RVOP(vrgather_vv_u64m2)(data_vec, xswap_mask, vl); + /* acc_vec += data_key_lo * data_key_hi; */ + acc_vec = XXH_RVOP(vmacc_vv_u64m2)(acc_vec, data_key_lo, data_key_hi, vl); + /* acc_vec += data_swap; */ + acc_vec = XXH_RVOP(vadd_vv_u64m2)(acc_vec, data_swap, vl); + /* xacc[i] = acc_vec; */ + XXH_RVOP(vse64_v_u64m2)(xacc + i, acc_vec, vl); + } + } +} + +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(rvv) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_rvv(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + size_t count = XXH_STRIPE_LEN/8; + uint64_t* xacc = (uint64_t*)acc; + const uint8_t* xsecret = (const uint8_t *)secret; + size_t vl; + for (; count > 0; count -= vl, xacc += vl, xsecret += vl*8) { + vl = XXH_RVOP(vsetvl_e64m2)(count); + { + /* key_vec = xsecret[i]; */ + vuint64m2_t key_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)(xsecret, vl*8)); + /* acc_vec = xacc[i]; */ + vuint64m2_t acc_vec = XXH_RVOP(vle64_v_u64m2)(xacc, vl); + /* acc_vec ^= acc_vec >> 47; */ + vuint64m2_t vsrl = XXH_RVOP(vsrl_vx_u64m2)(acc_vec, 47, vl); + acc_vec = XXH_RVOP(vxor_vv_u64m2)(acc_vec, vsrl, vl); + /* acc_vec ^= key_vec; */ + acc_vec = XXH_RVOP(vxor_vv_u64m2)(acc_vec, key_vec, vl); + /* acc_vec *= XXH_PRIME32_1; */ + acc_vec = XXH_RVOP(vmul_vx_u64m2)(acc_vec, XXH_PRIME32_1, vl); + /* xacc[i] *= acc_vec; */ + XXH_RVOP(vse64_v_u64m2)(xacc, acc_vec, vl); + } + } + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_rvv(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT(XXH_SEC_ALIGN >= 8); + XXH_ASSERT(((size_t)customSecret & 7) == 0); + (void)(&XXH_writeLE64); + { + size_t count = XXH_SECRET_DEFAULT_SIZE/8; + size_t vl; + size_t VLMAX = XXH_RVOP(vsetvlmax_e64m2)(); + int64_t* cSecret = (int64_t*)customSecret; + const int64_t* kSecret = (const int64_t*)(const void*)XXH3_kSecret; + +#if __riscv_v_intrinsic >= 1000000 + // ratified v1.0 intrinics version + vbool32_t mneg = XXH_RVCAST(u8m1_b32)( + XXH_RVOP(vmv_v_x_u8m1)(0xaa, XXH_RVOP(vsetvlmax_e8m1)())); +#else + // support pre-ratification intrinics, which lack mask to vector casts + size_t vlmax = XXH_RVOP(vsetvlmax_e8m1)(); + vbool32_t mneg = XXH_RVOP(vmseq_vx_u8mf4_b32)( + XXH_RVOP(vand_vx_u8mf4)( + XXH_RVOP(vid_v_u8mf4)(vlmax), 1, vlmax), 1, vlmax); +#endif + vint64m2_t seed = XXH_RVOP(vmv_v_x_i64m2)((int64_t)seed64, VLMAX); + seed = XXH_RVOP(vneg_v_i64m2_mu)(mneg, seed, seed, VLMAX); + + for (; count > 0; count -= vl, cSecret += vl, kSecret += vl) { + /* make sure vl=VLMAX until last iteration */ + vl = XXH_RVOP(vsetvl_e64m2)(count < VLMAX ? count : VLMAX); + { + vint64m2_t src = XXH_RVOP(vle64_v_i64m2)(kSecret, vl); + vint64m2_t res = XXH_RVOP(vadd_vv_i64m2)(src, seed, vl); + XXH_RVOP(vse64_v_i64m2)(cSecret, res, vl); + } + } + } +} +#endif + + /* scalar variants - universal */ #if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) @@ -6067,6 +6205,12 @@ typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); #define XXH3_scrambleAcc XXH3_scrambleAcc_lsx #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#elif (XXH_VECTOR == XXH_RVV) +#define XXH3_accumulate_512 XXH3_accumulate_512_rvv +#define XXH3_accumulate XXH3_accumulate_rvv +#define XXH3_scrambleAcc XXH3_scrambleAcc_rvv +#define XXH3_initCustomSecret XXH3_initCustomSecret_rvv + #else /* scalar */ #define XXH3_accumulate_512 XXH3_accumulate_512_scalar @@ -6563,6 +6707,16 @@ XXH3_update(XXH3_state_t* XXH_RESTRICT const state, } XXH_ASSERT(state != NULL); + state->totalLen += len; + + /* small input : just fill in tmp buffer */ + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + { const xxh_u8* const bEnd = input + len; const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 @@ -6575,15 +6729,6 @@ XXH3_update(XXH3_state_t* XXH_RESTRICT const state, #else xxh_u64* XXH_RESTRICT const acc = state->acc; #endif - state->totalLen += len; - XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); - - /* small input : just fill in tmp buffer */ - if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { - XXH_memcpy(state->buffer + state->bufferedSize, input, len); - state->bufferedSize += (XXH32_hash_t)len; - return XXH_OK; - } /* total input is now > XXH3_INTERNALBUFFER_SIZE */ #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl index d552a387dd..9cd8a07a5d 100644 --- a/src/fabric/src/fabric.erl +++ b/src/fabric/src/fabric.erl @@ -25,6 +25,8 @@ get_db_info/1, get_doc_count/1, get_doc_count/2, set_revs_limit/3, + update_props/3, + update_props/4, set_security/2, set_security/3, get_revs_limit/1, get_security/1, get_security/2, @@ -62,9 +64,10 @@ -export([ design_docs/1, reset_validation_funs/1, - cleanup_index_files/0, - cleanup_index_files/1, + cleanup_index_files_all_nodes/0, cleanup_index_files_all_nodes/1, + cleanup_index_files_this_node/0, + cleanup_index_files_this_node/1, dbname/1, db_uuids/1 ]). @@ -175,6 +178,16 @@ get_revs_limit(DbName) -> catch couch_db:close(Db) end. +%% @doc update shard property. Some properties like `partitioned` or `hash` are +%% static and cannot be updated. They will return an error. +-spec update_props(dbname(), atom() | binary(), any()) -> ok. +update_props(DbName, K, V) -> + update_props(DbName, K, V, [?ADMIN_CTX]). + +-spec update_props(dbname(), atom() | binary(), any(), [option()]) -> ok. +update_props(DbName, K, V, Options) when is_atom(K) orelse is_binary(K) -> + fabric_db_meta:update_props(dbname(DbName), K, V, opts(Options)). + %% @doc sets the readers/writers/admin permissions for a database -spec set_security(dbname(), SecObj :: json_obj()) -> ok. set_security(DbName, SecObj) -> @@ -570,54 +583,17 @@ reset_validation_funs(DbName) -> || #shard{node = Node, name = Name} <- mem3:shards(DbName) ]. -%% @doc clean up index files for all Dbs --spec cleanup_index_files() -> [ok]. -cleanup_index_files() -> - {ok, Dbs} = fabric:all_dbs(), - [cleanup_index_files(Db) || Db <- Dbs]. +cleanup_index_files_this_node() -> + fabric_index_cleanup:cleanup_this_node(). -%% @doc clean up index files for a specific db --spec cleanup_index_files(dbname()) -> ok. -cleanup_index_files(DbName) -> - try - ShardNames = [mem3:name(S) || S <- mem3:local_shards(dbname(DbName))], - cleanup_local_indices_and_purge_checkpoints(ShardNames) - catch - error:database_does_not_exist -> - ok - end. +cleanup_index_files_this_node(Db) -> + fabric_index_cleanup:cleanup_this_node(dbname(Db)). -cleanup_local_indices_and_purge_checkpoints([]) -> - ok; -cleanup_local_indices_and_purge_checkpoints([_ | _] = Dbs) -> - AllIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs), - AllPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs), - Sigs = couch_mrview_util:get_signatures(hd(Dbs)), - ok = cleanup_purges(Sigs, AllPurges, Dbs), - ok = cleanup_indices(Sigs, AllIndices). - -cleanup_purges(Sigs, AllPurges, Dbs) -> - Fun = fun(DbPurges, Db) -> - couch_mrview_cleanup:cleanup_purges(Db, Sigs, DbPurges) - end, - lists:zipwith(Fun, AllPurges, Dbs), - ok. +cleanup_index_files_all_nodes() -> + fabric_index_cleanup:cleanup_all_nodes(). -cleanup_indices(Sigs, AllIndices) -> - Fun = fun(DbIndices) -> - couch_mrview_cleanup:cleanup_indices(Sigs, DbIndices) - end, - lists:foreach(Fun, AllIndices). - -%% @doc clean up index files for a specific db on all nodes --spec cleanup_index_files_all_nodes(dbname()) -> [reference()]. -cleanup_index_files_all_nodes(DbName) -> - lists:foreach( - fun(Node) -> - rexi:cast(Node, {?MODULE, cleanup_index_files, [DbName]}) - end, - mem3:nodes() - ). +cleanup_index_files_all_nodes(Db) -> + fabric_index_cleanup:cleanup_all_nodes(dbname(Db)). %% some simple type validation and transcoding dbname(DbName) when is_list(DbName) -> @@ -629,7 +605,7 @@ dbname(Db) -> couch_db:name(Db) catch error:badarg -> - erlang:error({illegal_database_name, Db}) + error({illegal_database_name, Db}) end. %% @doc get db shard uuids @@ -648,7 +624,7 @@ docid(DocId) -> docs(Db, Docs) when is_list(Docs) -> [doc(Db, D) || D <- Docs]; docs(_Db, Docs) -> - erlang:error({illegal_docs_list, Docs}). + error({illegal_docs_list, Docs}). doc(_Db, #doc{} = Doc) -> Doc; @@ -658,14 +634,13 @@ doc(Db0, {_} = Doc) -> true -> Db0; false -> - Shard = hd(mem3:shards(Db0)), - Props = couch_util:get_value(props, Shard#shard.opts, []), + Props = mem3:props(Db0), {ok, Db1} = couch_db:clustered_db(Db0, [{props, Props}]), Db1 end, couch_db:doc_from_json_obj_validate(Db, Doc); doc(_Db, Doc) -> - erlang:error({illegal_doc_format, Doc}). + error({illegal_doc_format, Doc}). design_doc(#doc{} = DDoc) -> DDoc; diff --git a/src/fabric/src/fabric_db_create.erl b/src/fabric/src/fabric_db_create.erl index f7c6e54022..3f378dc73e 100644 --- a/src/fabric/src/fabric_db_create.erl +++ b/src/fabric/src/fabric_db_create.erl @@ -158,11 +158,7 @@ make_document([#shard{dbname = DbName} | _] = Shards, Suffix, Options) -> {RawOut, ByNodeOut, ByRangeOut} = lists:foldl( fun(#shard{node = N, range = [B, E]}, {Raw, ByNode, ByRange}) -> - Range = ?l2b([ - couch_util:to_hex(<>), - "-", - couch_util:to_hex(<>) - ]), + Range = mem3_util:range_to_hex([B, E]), Node = couch_util:to_binary(N), { [[<<"add">>, Range, Node] | Raw], @@ -227,7 +223,7 @@ db_exists_for_existing_db() -> db_exists_for_missing_db() -> Mock = fun(DbName) -> - erlang:error(database_does_not_exist, [DbName]) + error(database_does_not_exist, [DbName]) end, meck:expect(mem3, shards, Mock), ?assertEqual(false, db_exists(<<"foobar">>)), diff --git a/src/fabric/src/fabric_db_delete.erl b/src/fabric/src/fabric_db_delete.erl index 6e44c6af54..5a95b03246 100644 --- a/src/fabric/src/fabric_db_delete.erl +++ b/src/fabric/src/fabric_db_delete.erl @@ -29,7 +29,7 @@ go(DbName, _Options) -> {ok, accepted} -> accepted; {ok, not_found} -> - erlang:error(database_does_not_exist, [DbName]); + error(database_does_not_exist, [DbName]); Error -> Error after diff --git a/src/fabric/src/fabric_db_meta.erl b/src/fabric/src/fabric_db_meta.erl index 1013b958d4..af4a069d4c 100644 --- a/src/fabric/src/fabric_db_meta.erl +++ b/src/fabric/src/fabric_db_meta.erl @@ -16,7 +16,8 @@ set_revs_limit/3, set_security/3, get_all_security/2, - set_purge_infos_limit/3 + set_purge_infos_limit/3, + update_props/4 ]). -include_lib("fabric/include/fabric.hrl"). @@ -198,3 +199,25 @@ maybe_finish_get(#acc{workers = []} = Acc) -> {stop, Acc}; maybe_finish_get(Acc) -> {ok, Acc}. + +update_props(DbName, K, V, Options) -> + Shards = mem3:shards(DbName), + Workers = fabric_util:submit_jobs(Shards, update_props, [K, V, Options]), + Handler = fun handle_update_props_message/3, + Acc0 = {Workers, length(Workers) - 1}, + case fabric_util:recv(Workers, #shard.ref, Handler, Acc0) of + {ok, ok} -> + ok; + {timeout, {DefunctWorkers, _}} -> + fabric_util:log_timeout(DefunctWorkers, "update_props"), + {error, timeout}; + Error -> + Error + end. + +handle_update_props_message(ok, _, {_Workers, 0}) -> + {stop, ok}; +handle_update_props_message(ok, Worker, {Workers, Waiting}) -> + {ok, {lists:delete(Worker, Workers), Waiting - 1}}; +handle_update_props_message(Error, _, _Acc) -> + {error, Error}. diff --git a/src/fabric/src/fabric_db_purged_infos.erl b/src/fabric/src/fabric_db_purged_infos.erl index 45f5681b6c..b1c5a606eb 100644 --- a/src/fabric/src/fabric_db_purged_infos.erl +++ b/src/fabric/src/fabric_db_purged_infos.erl @@ -18,8 +18,7 @@ -record(pacc, { counters, - replies, - ring_opts + replies }). go(DbName) -> @@ -29,8 +28,7 @@ go(DbName) -> Fun = fun handle_message/3, Acc0 = #pacc{ counters = fabric_dict:init(Workers, nil), - replies = couch_util:new_set(), - ring_opts = [{any, Shards}] + replies = couch_util:new_set() }, try case fabric_util:recv(Workers, #shard.ref, Fun, Acc0) of @@ -48,17 +46,17 @@ go(DbName) -> end. handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Shard, #pacc{} = Acc) -> - #pacc{counters = Counters, ring_opts = RingOpts} = Acc, - case fabric_util:remove_down_workers(Counters, NodeRef, RingOpts) of + #pacc{counters = Counters} = Acc, + case fabric_util:remove_down_workers(Counters, NodeRef, [all]) of {ok, NewCounters} -> {ok, Acc#pacc{counters = NewCounters}}; error -> {error, {nodedown, <<"progress not possible">>}} end; handle_message({rexi_EXIT, Reason}, Shard, #pacc{} = Acc) -> - #pacc{counters = Counters, ring_opts = RingOpts} = Acc, + #pacc{counters = Counters} = Acc, NewCounters = fabric_dict:erase(Shard, Counters), - case fabric_ring:is_progress_possible(NewCounters, RingOpts) of + case fabric_ring:is_progress_possible(NewCounters, [all]) of true -> {ok, Acc#pacc{counters = NewCounters}}; false -> @@ -92,8 +90,7 @@ make_shards() -> init_acc(Shards) -> #pacc{ counters = fabric_dict:init(Shards, nil), - replies = couch_util:new_set(), - ring_opts = [{any, Shards}] + replies = couch_util:new_set() }. first_result_ok_test() -> diff --git a/src/fabric/src/fabric_db_update_listener.erl b/src/fabric/src/fabric_db_update_listener.erl index 4f3c30a252..e91f2ec5af 100644 --- a/src/fabric/src/fabric_db_update_listener.erl +++ b/src/fabric/src/fabric_db_update_listener.erl @@ -60,8 +60,8 @@ go(Parent, ParentRef, DbName, Timeout, ClientReq) -> end, case Resp of {ok, _} -> ok; - {error, Error} -> erlang:error(Error); - Error -> erlang:error(Error) + {error, Error} -> error(Error); + Error -> error(Error) end. start_update_notifiers(Shards) -> @@ -99,7 +99,7 @@ handle_db_event(_DbName, _Event, St) -> start_cleanup_monitor(Parent, Notifiers, ClientReq) -> spawn(fun() -> - Ref = erlang:monitor(process, Parent), + Ref = monitor(process, Parent), cleanup_monitor(Parent, Ref, Notifiers, ClientReq) end). @@ -129,11 +129,11 @@ stop({Pid, Ref}) -> erlang:send(Pid, {Ref, done}). wait_db_updated({Pid, Ref}) -> - MonRef = erlang:monitor(process, Pid), + MonRef = monitor(process, Pid), erlang:send(Pid, {Ref, get_state}), receive {state, Pid, State} -> - erlang:demonitor(MonRef, [flush]), + demonitor(MonRef, [flush]), State; {'DOWN', MonRef, process, Pid, _Reason} -> changes_feed_died diff --git a/src/fabric/src/fabric_doc_open.erl b/src/fabric/src/fabric_doc_open.erl index 4946a26bfa..138a3f2bbb 100644 --- a/src/fabric/src/fabric_doc_open.erl +++ b/src/fabric/src/fabric_doc_open.erl @@ -41,11 +41,11 @@ go(DbName, Id, Options) -> ), SuppressDeletedDoc = not lists:member(deleted, Options), N = mem3:n(DbName), - R = couch_util:get_value(r, Options, integer_to_list(mem3:quorum(DbName))), + R = fabric_util:r_from_opts(DbName, Options), Acc0 = #acc{ dbname = DbName, workers = Workers, - r = erlang:min(N, list_to_integer(R)), + r = min(N, R), state = r_not_met, replies = [] }, @@ -317,8 +317,8 @@ t_handle_message_down(_) -> t_handle_message_exit(_) -> Exit = {rexi_EXIT, nil}, - Worker0 = #shard{ref = erlang:make_ref()}, - Worker1 = #shard{ref = erlang:make_ref()}, + Worker0 = #shard{ref = make_ref()}, + Worker1 = #shard{ref = make_ref()}, % Only removes the specified worker ?assertEqual( @@ -338,9 +338,9 @@ t_handle_message_exit(_) -> ). t_handle_message_reply(_) -> - Worker0 = #shard{ref = erlang:make_ref()}, - Worker1 = #shard{ref = erlang:make_ref()}, - Worker2 = #shard{ref = erlang:make_ref()}, + Worker0 = #shard{ref = make_ref()}, + Worker1 = #shard{ref = make_ref()}, + Worker2 = #shard{ref = make_ref()}, Workers = [Worker0, Worker1, Worker2], Acc0 = #acc{workers = Workers, r = 2, replies = []}, @@ -426,9 +426,9 @@ t_handle_message_reply(_) -> ). t_store_node_revs(_) -> - W1 = #shard{node = w1, ref = erlang:make_ref()}, - W2 = #shard{node = w2, ref = erlang:make_ref()}, - W3 = #shard{node = w3, ref = erlang:make_ref()}, + W1 = #shard{node = w1, ref = make_ref()}, + W2 = #shard{node = w2, ref = make_ref()}, + W3 = #shard{node = w3, ref = make_ref()}, Foo1 = {ok, #doc{id = <<"bar">>, revs = {1, [<<"foo">>]}}}, Foo2 = {ok, #doc{id = <<"bar">>, revs = {2, [<<"foo2">>, <<"foo">>]}}}, NFM = {not_found, missing}, diff --git a/src/fabric/src/fabric_doc_open_revs.erl b/src/fabric/src/fabric_doc_open_revs.erl index b0ff994b01..93ec7e71ef 100644 --- a/src/fabric/src/fabric_doc_open_revs.erl +++ b/src/fabric/src/fabric_doc_open_revs.erl @@ -37,12 +37,11 @@ go(DbName, Id, Revs, Options) -> open_revs, [Id, Revs, Options] ), - R = couch_util:get_value(r, Options, integer_to_list(mem3:quorum(DbName))), State = #state{ dbname = DbName, worker_count = length(Workers), workers = Workers, - r = list_to_integer(R), + r = fabric_util:r_from_opts(DbName, Options), revs = Revs, latest = lists:member(latest, Options), replies = [] @@ -213,7 +212,7 @@ maybe_read_repair(Db, IsTree, Replies, NodeRevs, ReplyCount, DoRepair) -> [] -> ok; _ -> - erlang:spawn(fun() -> read_repair(Db, Docs, NodeRevs) end) + spawn(fun() -> read_repair(Db, Docs, NodeRevs) end) end. tree_repair_docs(_Replies, false) -> diff --git a/src/fabric/src/fabric_doc_purge.erl b/src/fabric/src/fabric_doc_purge.erl index 5405ceb600..a4fc9e76fe 100644 --- a/src/fabric/src/fabric_doc_purge.erl +++ b/src/fabric/src/fabric_doc_purge.erl @@ -28,44 +28,41 @@ go(_, [], _) -> {ok, []}; go(DbName, IdsRevs, Options) -> - % Generate our purge requests of {UUID, DocId, Revs} - {UUIDs, Reqs} = create_reqs(IdsRevs, [], []), - - % Fire off rexi workers for each shard. - {Workers, WorkerUUIDs} = dict:fold( - fun(Shard, ShardReqs, {Ws, WUUIDs}) -> + % Generate our purge requests of {UUID, DocId, Revs}. Return: + % * Reqs : [{UUID, DocId, Revs}] + % * UUIDs : [UUID] in the same order as Reqs + % * Responses : #{UUID => []} initial response accumulator + % + {UUIDs, Reqs, Responses} = create_requests_and_responses(IdsRevs), + + % Fire off rexi workers for each shard. Return: + % * Workers : [#shard{ref = Ref}] + % * WorkerUUIDs : #{Worker => [UUID]} + % * UUIDCounts : #{UUID => Counter} + % + {Workers, WorkerUUIDs, UUIDCounts} = maps:fold( + fun(Shard, ShardReqs, {WorkersAcc, WorkersUUIDsAcc, CountsAcc}) -> #shard{name = ShardDbName, node = Node} = Shard, Args = [ShardDbName, ShardReqs, Options], Ref = rexi:cast(Node, {fabric_rpc, purge_docs, Args}), Worker = Shard#shard{ref = Ref}, ShardUUIDs = [UUID || {UUID, _Id, _Revs} <- ShardReqs], - {[Worker | Ws], [{Worker, ShardUUIDs} | WUUIDs]} + Fun = fun(UUID, Acc) -> update_counter(UUID, Acc) end, + CountsAcc1 = lists:foldl(Fun, CountsAcc, ShardUUIDs), + WorkersUUIDAcc1 = WorkersUUIDsAcc#{Worker => ShardUUIDs}, + {[Worker | WorkersAcc], WorkersUUIDAcc1, CountsAcc1} end, - {[], []}, + {[], #{}, #{}}, group_reqs_by_shard(DbName, Reqs) ), - UUIDCounts = lists:foldl( - fun({_Worker, WUUIDs}, CountAcc) -> - lists:foldl( - fun(UUID, InnerCountAcc) -> - dict:update_counter(UUID, 1, InnerCountAcc) - end, - CountAcc, - WUUIDs - ) - end, - dict:new(), - WorkerUUIDs - ), - RexiMon = fabric_util:create_monitors(Workers), Timeout = fabric_util:request_timeout(), Acc0 = #acc{ worker_uuids = WorkerUUIDs, - resps = dict:from_list([{UUID, []} || UUID <- UUIDs]), + resps = Responses, uuid_counts = UUIDCounts, - w = w(DbName, Options) + w = fabric_util:w_from_opts(DbName, Options) }, Callback = fun handle_message/3, Acc2 = @@ -85,8 +82,9 @@ handle_message({rexi_DOWN, _, {_, Node}, _}, _Worker, Acc) -> worker_uuids = WorkerUUIDs, resps = Resps } = Acc, - Pred = fun({#shard{node = N}, _}) -> N == Node end, - {Failed, Rest} = lists:partition(Pred, WorkerUUIDs), + Pred = fun(#shard{node = N}, _) -> N == Node end, + Failed = maps:filter(Pred, WorkerUUIDs), + Rest = maps:without(maps:keys(Failed), WorkerUUIDs), NewResps = append_errors(internal_server_error, Failed, Resps), maybe_stop(Acc#acc{worker_uuids = Rest, resps = NewResps}); handle_message({rexi_EXIT, _}, Worker, Acc) -> @@ -94,60 +92,47 @@ handle_message({rexi_EXIT, _}, Worker, Acc) -> worker_uuids = WorkerUUIDs, resps = Resps } = Acc, - {value, WorkerPair, Rest} = lists:keytake(Worker, 1, WorkerUUIDs), - NewResps = append_errors(internal_server_error, [WorkerPair], Resps), - maybe_stop(Acc#acc{worker_uuids = Rest, resps = NewResps}); + {FailedUUIDs, WorkerUUIDs1} = maps:take(Worker, WorkerUUIDs), + NewResps = append_errors(internal_server_error, #{Worker => FailedUUIDs}, Resps), + maybe_stop(Acc#acc{worker_uuids = WorkerUUIDs1, resps = NewResps}); handle_message({ok, Replies}, Worker, Acc) -> #acc{ worker_uuids = WorkerUUIDs, resps = Resps } = Acc, - {value, {_W, UUIDs}, Rest} = lists:keytake(Worker, 1, WorkerUUIDs), + {UUIDs, WorkerUUIDs1} = maps:take(Worker, WorkerUUIDs), NewResps = append_resps(UUIDs, Replies, Resps), - maybe_stop(Acc#acc{worker_uuids = Rest, resps = NewResps}); + maybe_stop(Acc#acc{worker_uuids = WorkerUUIDs1, resps = NewResps}); handle_message({bad_request, Msg}, _, _) -> throw({bad_request, Msg}). handle_timeout(#acc{worker_uuids = DefunctWorkerUUIDs, resps = Resps} = Acc) -> - DefunctWorkers = [Worker || {Worker, _} <- DefunctWorkerUUIDs], + DefunctWorkers = maps:keys(DefunctWorkerUUIDs), fabric_util:log_timeout(DefunctWorkers, "purge_docs"), NewResps = append_errors(timeout, DefunctWorkerUUIDs, Resps), - Acc#acc{worker_uuids = [], resps = NewResps}. + Acc#acc{worker_uuids = #{}, resps = NewResps}. -create_reqs([], UUIDs, Reqs) -> - {lists:reverse(UUIDs), lists:reverse(Reqs)}; -create_reqs([{Id, Revs} | RestIdsRevs], UUIDs, Reqs) -> - UUID = couch_uuids:new(), - NewUUIDs = [UUID | UUIDs], - NewReqs = [{UUID, Id, lists:usort(Revs)} | Reqs], - create_reqs(RestIdsRevs, NewUUIDs, NewReqs). +create_requests_and_responses(IdsRevs) -> + Fun = fun({Id, Revs}, {UUIDsAcc, RespAcc}) -> + UUID = couch_uuids:v7_bin(), + {{UUID, Id, lists:usort(Revs)}, {[UUID | UUIDsAcc], RespAcc#{UUID => []}}} + end, + {IdRevs1, {UUIDs, Resps}} = lists:mapfoldl(Fun, {[], #{}}, IdsRevs), + {lists:reverse(UUIDs), IdRevs1, Resps}. group_reqs_by_shard(DbName, Reqs) -> - lists:foldl( - fun({_UUID, Id, _Revs} = Req, D0) -> - lists:foldl( - fun(Shard, D1) -> - dict:append(Shard, Req, D1) - end, - D0, - mem3:shards(DbName, Id) - ) + ReqFoldFun = + fun({_UUID, Id, _Revs} = Req, #{} = Map0) -> + AppendFun = fun(Shard, Map1) -> map_append(Shard, Req, Map1) end, + lists:foldl(AppendFun, Map0, mem3:shards(DbName, Id)) end, - dict:new(), - Reqs - ). + lists:foldl(ReqFoldFun, #{}, Reqs). -w(DbName, Options) -> - try - list_to_integer(couch_util:get_value(w, Options)) - catch - _:_ -> - mem3:quorum(DbName) - end. - -append_errors(Type, WorkerUUIDs, Resps) -> - lists:foldl( - fun({_Worker, UUIDs}, RespAcc) -> +% Failed WorkerUUIDs = #{#shard{} => [UUIDs, ...]} +% Resps = #{UUID => [{ok, ...} | {error, ...}] +append_errors(Type, #{} = WorkerUUIDs, #{} = Resps) -> + maps:fold( + fun(_Worker, UUIDs, RespAcc) -> Errors = [{error, Type} || _UUID <- UUIDs], append_resps(UUIDs, Errors, RespAcc) end, @@ -155,59 +140,54 @@ append_errors(Type, WorkerUUIDs, Resps) -> WorkerUUIDs ). -append_resps([], [], Resps) -> +append_resps([], [], #{} = Resps) -> Resps; -append_resps([UUID | RestUUIDs], [Reply | RestReplies], Resps) -> - NewResps = dict:append(UUID, Reply, Resps), +append_resps([UUID | RestUUIDs], [Reply | RestReplies], #{} = Resps) -> + NewResps = map_append(UUID, Reply, Resps), append_resps(RestUUIDs, RestReplies, NewResps). -maybe_stop(#acc{worker_uuids = []} = Acc) -> +maybe_stop(#acc{worker_uuids = #{} = Map} = Acc) when map_size(Map) == 0 -> {stop, Acc}; -maybe_stop(#acc{resps = Resps, uuid_counts = Counts, w = W} = Acc) -> +maybe_stop(#acc{resps = #{} = Resps, uuid_counts = #{} = Counts, w = W} = Acc) -> try - dict:fold( - fun(UUID, UUIDResps, _) -> - UUIDCount = dict:fetch(UUID, Counts), + Fun = + fun(UUID, UUIDResps) -> + #{UUID := UUIDCount} = Counts, case has_quorum(UUIDResps, UUIDCount, W) of true -> ok; false -> throw(keep_going) end end, - nil, - Resps - ), + maps:foreach(Fun, Resps), {stop, Acc} catch throw:keep_going -> {ok, Acc} end. -format_resps(UUIDs, #acc{} = Acc) -> - #acc{ - resps = Resps, - w = W - } = Acc, - FoldFun = fun(UUID, Replies, ReplyAcc) -> +format_resps(UUIDs, #acc{resps = Resps, w = W}) -> + Fun = fun(_UUID, Replies) -> OkReplies = [Reply || {ok, Reply} <- Replies], case OkReplies of [] -> [Error | _] = lists:usort(Replies), - [{UUID, Error} | ReplyAcc]; - _ -> + Error; + [_ | _] -> AllRevs = lists:usort(lists:flatten(OkReplies)), - IsOk = - length(OkReplies) >= W andalso - length(lists:usort(OkReplies)) == 1, + IsOk = length(OkReplies) >= W andalso length(lists:usort(OkReplies)) == 1, Health = if IsOk -> ok; true -> accepted end, - [{UUID, {Health, AllRevs}} | ReplyAcc] + {Health, AllRevs} end end, - FinalReplies = dict:fold(FoldFun, [], Resps), - couch_util:reorder_results(UUIDs, FinalReplies); + FinalReplies = maps:map(Fun, Resps), + % Reorder results in the same order as the the initial IdRevs + % this also implicitly asserts that the all UUIDs should have + % a matching reply + [map_get(UUID, FinalReplies) || UUID <- UUIDs]; format_resps(_UUIDs, Else) -> Else. @@ -225,22 +205,45 @@ resp_health(Resps) -> has_quorum(Resps, Count, W) -> OkResps = [R || {ok, _} = R <- Resps], - OkCounts = lists:foldl( - fun(R, Acc) -> - orddict:update_counter(R, 1, Acc) - end, - orddict:new(), - OkResps - ), - MaxOk = lists:max([0 | element(2, lists:unzip(OkCounts))]), + OkCounts = lists:foldl(fun(R, Acc) -> update_counter(R, Acc) end, #{}, OkResps), + MaxOk = lists:max([0 | maps:values(OkCounts)]), if MaxOk >= W -> true; length(Resps) >= Count -> true; true -> false end. +map_append(Key, Val, #{} = Map) -> + maps:update_with(Key, fun(V) -> [Val | V] end, [Val], Map). + +update_counter(Key, #{} = Map) -> + maps:update_with(Key, fun(V) -> V + 1 end, 1, Map). + -ifdef(TEST). + -include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +response_health_test() -> + ?assertEqual(error, resp_health([])), + ?assertEqual(error, resp_health([{potato, x}])), + ?assertEqual(ok, resp_health([{ok, x}, {ok, y}])), + ?assertEqual(accepted, resp_health([{accepted, x}])), + ?assertEqual(accepted, resp_health([{ok, x}, {accepted, y}])), + ?assertEqual(error, resp_health([{error, x}])), + ?assertEqual(error, resp_health([{ok, x}, {error, y}])), + ?assertEqual(error, resp_health([{error, x}, {accepted, y}, {ok, z}])). + +has_quorum_test() -> + ?assertEqual(true, has_quorum([], 0, 0)), + ?assertEqual(true, has_quorum([], 1, 0)), + ?assertEqual(true, has_quorum([], 0, 1)), + ?assertEqual(false, has_quorum([], 1, 1)), + ?assertEqual(true, has_quorum([{ok, x}], 1, 1)), + ?assertEqual(true, has_quorum([{accepted, x}], 1, 1)), + ?assertEqual(false, has_quorum([{accepted, x}], 2, 1)), + ?assertEqual(false, has_quorum([{accepted, x}, {ok, y}], 3, 2)), + ?assertEqual(true, has_quorum([{accepted, x}, {ok, y}], 2, 2)). purge_test_() -> { @@ -248,6 +251,8 @@ purge_test_() -> fun setup/0, fun teardown/1, with([ + ?TDEF(t_create_reqs), + ?TDEF(t_w2_ok), ?TDEF(t_w3_ok), @@ -262,29 +267,53 @@ purge_test_() -> ?TDEF(t_mixed_ok_accepted), ?TDEF(t_mixed_errors), + ?TDEF(t_rexi_down_error), ?TDEF(t_timeout) ]) }. setup() -> - meck:new(couch_log), - meck:expect(couch_log, warning, fun(_, _) -> ok end), - meck:expect(couch_log, notice, fun(_, _) -> ok end), - meck:expect(couch_log, error, fun(_, _) -> ok end). + test_util:start_couch(). -teardown(_) -> - meck:unload(). +teardown(Ctx) -> + test_util:stop_couch(Ctx). + +t_create_reqs(_) -> + ?assertEqual({[], [], #{}}, create_requests_and_responses([])), + IdRevs = [ + {<<"3">>, []}, + {<<"1">>, [<<"2-b">>, <<"1-a">>]}, + {<<"2">>, [<<"3-c">>, <<"1-d">>, <<"3-c">>]} + ], + Res = create_requests_and_responses(IdRevs), + ?assertMatch({[<<_/binary>> | _], [{<<_/binary>>, _, _} | _], #{}}, Res), + {UUIDs, IdRevs1, Resps} = Res, + ?assertEqual(3, length(UUIDs)), + ?assertEqual(3, length(IdRevs1)), + ?assertEqual(3, map_size(Resps)), + ?assertEqual(lists:sort(UUIDs), lists:sort(maps:keys(Resps))), + {IdRevsUUIDs, DocIds, Revs} = lists:unzip3(IdRevs1), + ?assertEqual(UUIDs, IdRevsUUIDs), + ?assertEqual([<<"3">>, <<"1">>, <<"2">>], DocIds), + ?assertEqual( + [ + [], + [<<"1-a">>, <<"2-b">>], + [<<"1-d">>, <<"3-c">>] + ], + Revs + ). t_w2_ok(_) -> Acc0 = create_init_acc(2), Msg = {ok, [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}]}, {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), - ?assertEqual(2, length(Acc1#acc.worker_uuids)), + ?assertEqual(2, map_size(Acc1#acc.worker_uuids)), check_quorum(Acc1, false), {stop, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), - ?assertEqual(1, length(Acc2#acc.worker_uuids)), + ?assertEqual(1, map_size(Acc2#acc.worker_uuids)), check_quorum(Acc2, true), Expect = [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}], @@ -300,11 +329,11 @@ t_w3_ok(_) -> check_quorum(Acc1, false), {ok, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), - ?assertEqual(1, length(Acc2#acc.worker_uuids)), + ?assertEqual(1, map_size(Acc2#acc.worker_uuids)), check_quorum(Acc2, false), {stop, Acc3} = handle_message(Msg, worker(3, Acc0), Acc2), - ?assertEqual(0, length(Acc3#acc.worker_uuids)), + ?assertEqual(0, map_size(Acc3#acc.worker_uuids)), check_quorum(Acc3, true), Expect = [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}], @@ -318,15 +347,15 @@ t_w2_mixed_accepted(_) -> Msg2 = {ok, [{ok, [{1, <<"foo2">>}]}, {ok, [{2, <<"bar2">>}]}]}, {ok, Acc1} = handle_message(Msg1, worker(1, Acc0), Acc0), - ?assertEqual(2, length(Acc1#acc.worker_uuids)), + ?assertEqual(2, map_size(Acc1#acc.worker_uuids)), check_quorum(Acc1, false), {ok, Acc2} = handle_message(Msg2, worker(2, Acc0), Acc1), - ?assertEqual(1, length(Acc2#acc.worker_uuids)), + ?assertEqual(1, map_size(Acc2#acc.worker_uuids)), check_quorum(Acc2, false), {stop, Acc3} = handle_message(Msg1, worker(3, Acc0), Acc2), - ?assertEqual(0, length(Acc3#acc.worker_uuids)), + ?assertEqual(0, map_size(Acc3#acc.worker_uuids)), check_quorum(Acc3, true), Expect = [ @@ -343,15 +372,15 @@ t_w3_mixed_accepted(_) -> Msg2 = {ok, [{ok, [{1, <<"foo2">>}]}, {ok, [{2, <<"bar2">>}]}]}, {ok, Acc1} = handle_message(Msg1, worker(1, Acc0), Acc0), - ?assertEqual(2, length(Acc1#acc.worker_uuids)), + ?assertEqual(2, map_size(Acc1#acc.worker_uuids)), check_quorum(Acc1, false), {ok, Acc2} = handle_message(Msg2, worker(2, Acc0), Acc1), - ?assertEqual(1, length(Acc2#acc.worker_uuids)), + ?assertEqual(1, map_size(Acc2#acc.worker_uuids)), check_quorum(Acc2, false), {stop, Acc3} = handle_message(Msg2, worker(3, Acc0), Acc2), - ?assertEqual(0, length(Acc3#acc.worker_uuids)), + ?assertEqual(0, map_size(Acc3#acc.worker_uuids)), check_quorum(Acc3, true), Expect = [ @@ -368,15 +397,15 @@ t_w2_exit1_ok(_) -> ExitMsg = {rexi_EXIT, blargh}, {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), - ?assertEqual(2, length(Acc1#acc.worker_uuids)), + ?assertEqual(2, map_size(Acc1#acc.worker_uuids)), check_quorum(Acc1, false), {ok, Acc2} = handle_message(ExitMsg, worker(2, Acc0), Acc1), - ?assertEqual(1, length(Acc2#acc.worker_uuids)), + ?assertEqual(1, map_size(Acc2#acc.worker_uuids)), check_quorum(Acc2, false), {stop, Acc3} = handle_message(Msg, worker(3, Acc0), Acc2), - ?assertEqual(0, length(Acc3#acc.worker_uuids)), + ?assertEqual(0, map_size(Acc3#acc.worker_uuids)), check_quorum(Acc3, true), Expect = [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}], @@ -390,15 +419,15 @@ t_w2_exit2_accepted(_) -> ExitMsg = {rexi_EXIT, blargh}, {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), - ?assertEqual(2, length(Acc1#acc.worker_uuids)), + ?assertEqual(2, map_size(Acc1#acc.worker_uuids)), check_quorum(Acc1, false), {ok, Acc2} = handle_message(ExitMsg, worker(2, Acc0), Acc1), - ?assertEqual(1, length(Acc2#acc.worker_uuids)), + ?assertEqual(1, map_size(Acc2#acc.worker_uuids)), check_quorum(Acc2, false), {stop, Acc3} = handle_message(ExitMsg, worker(3, Acc0), Acc2), - ?assertEqual(0, length(Acc3#acc.worker_uuids)), + ?assertEqual(0, map_size(Acc3#acc.worker_uuids)), check_quorum(Acc3, true), Expect = [{accepted, [{1, <<"foo">>}]}, {accepted, [{2, <<"bar">>}]}], @@ -411,15 +440,15 @@ t_w2_exit3_error(_) -> ExitMsg = {rexi_EXIT, blargh}, {ok, Acc1} = handle_message(ExitMsg, worker(1, Acc0), Acc0), - ?assertEqual(2, length(Acc1#acc.worker_uuids)), + ?assertEqual(2, map_size(Acc1#acc.worker_uuids)), check_quorum(Acc1, false), {ok, Acc2} = handle_message(ExitMsg, worker(2, Acc0), Acc1), - ?assertEqual(1, length(Acc2#acc.worker_uuids)), + ?assertEqual(1, map_size(Acc2#acc.worker_uuids)), check_quorum(Acc2, false), {stop, Acc3} = handle_message(ExitMsg, worker(3, Acc0), Acc2), - ?assertEqual(0, length(Acc3#acc.worker_uuids)), + ?assertEqual(0, map_size(Acc3#acc.worker_uuids)), check_quorum(Acc3, true), Expect = [ @@ -439,15 +468,15 @@ t_w4_accepted(_) -> Msg = {ok, [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}]}, {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), - ?assertEqual(2, length(Acc1#acc.worker_uuids)), + ?assertEqual(2, map_size(Acc1#acc.worker_uuids)), check_quorum(Acc1, false), {ok, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), - ?assertEqual(1, length(Acc2#acc.worker_uuids)), + ?assertEqual(1, map_size(Acc2#acc.worker_uuids)), check_quorum(Acc2, false), {stop, Acc3} = handle_message(Msg, worker(3, Acc0), Acc2), - ?assertEqual(0, length(Acc3#acc.worker_uuids)), + ?assertEqual(0, map_size(Acc3#acc.worker_uuids)), check_quorum(Acc3, true), Expect = [{accepted, [{1, <<"foo">>}]}, {accepted, [{2, <<"bar">>}]}], @@ -456,20 +485,20 @@ t_w4_accepted(_) -> ?assertEqual(accepted, resp_health(Resps)). t_mixed_ok_accepted(_) -> - WorkerUUIDs = [ - {#shard{node = a, range = [1, 2]}, [<<"uuid1">>]}, - {#shard{node = b, range = [1, 2]}, [<<"uuid1">>]}, - {#shard{node = c, range = [1, 2]}, [<<"uuid1">>]}, - - {#shard{node = a, range = [3, 4]}, [<<"uuid2">>]}, - {#shard{node = b, range = [3, 4]}, [<<"uuid2">>]}, - {#shard{node = c, range = [3, 4]}, [<<"uuid2">>]} - ], + WorkerUUIDs = #{ + #shard{node = a, range = [1, 2]} => [<<"uuid1">>], + #shard{node = b, range = [1, 2]} => [<<"uuid1">>], + #shard{node = c, range = [1, 2]} => [<<"uuid1">>], + + #shard{node = a, range = [3, 4]} => [<<"uuid2">>], + #shard{node = b, range = [3, 4]} => [<<"uuid2">>], + #shard{node = c, range = [3, 4]} => [<<"uuid2">>] + }, Acc0 = #acc{ worker_uuids = WorkerUUIDs, - resps = dict:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), - uuid_counts = dict:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), + resps = maps:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), + uuid_counts = maps:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), w = 2 }, @@ -477,11 +506,11 @@ t_mixed_ok_accepted(_) -> Msg2 = {ok, [{ok, [{2, <<"bar">>}]}]}, ExitMsg = {rexi_EXIT, blargh}, - {ok, Acc1} = handle_message(Msg1, worker(1, Acc0), Acc0), - {ok, Acc2} = handle_message(Msg1, worker(2, Acc0), Acc1), - {ok, Acc3} = handle_message(ExitMsg, worker(4, Acc0), Acc2), - {ok, Acc4} = handle_message(ExitMsg, worker(5, Acc0), Acc3), - {stop, Acc5} = handle_message(Msg2, worker(6, Acc0), Acc4), + {ok, Acc1} = handle_message(Msg1, worker(a, [1, 2], Acc0), Acc0), + {ok, Acc2} = handle_message(Msg1, worker(b, [1, 2], Acc0), Acc1), + {ok, Acc3} = handle_message(ExitMsg, worker(a, [3, 4], Acc0), Acc2), + {ok, Acc4} = handle_message(ExitMsg, worker(b, [3, 4], Acc0), Acc3), + {stop, Acc5} = handle_message(Msg2, worker(c, [3, 4], Acc0), Acc4), Expect = [{ok, [{1, <<"foo">>}]}, {accepted, [{2, <<"bar">>}]}], Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc5), @@ -489,59 +518,94 @@ t_mixed_ok_accepted(_) -> ?assertEqual(accepted, resp_health(Resps)). t_mixed_errors(_) -> - WorkerUUIDs = [ - {#shard{node = a, range = [1, 2]}, [<<"uuid1">>]}, - {#shard{node = b, range = [1, 2]}, [<<"uuid1">>]}, - {#shard{node = c, range = [1, 2]}, [<<"uuid1">>]}, - - {#shard{node = a, range = [3, 4]}, [<<"uuid2">>]}, - {#shard{node = b, range = [3, 4]}, [<<"uuid2">>]}, - {#shard{node = c, range = [3, 4]}, [<<"uuid2">>]} - ], + WorkerUUIDs = #{ + #shard{node = a, range = [1, 2]} => [<<"uuid1">>], + #shard{node = b, range = [1, 2]} => [<<"uuid1">>], + #shard{node = c, range = [1, 2]} => [<<"uuid1">>], + + #shard{node = a, range = [3, 4]} => [<<"uuid2">>], + #shard{node = b, range = [3, 4]} => [<<"uuid2">>], + #shard{node = c, range = [3, 4]} => [<<"uuid2">>] + }, Acc0 = #acc{ worker_uuids = WorkerUUIDs, - resps = dict:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), - uuid_counts = dict:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), + resps = maps:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), + uuid_counts = maps:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), w = 2 }, Msg = {ok, [{ok, [{1, <<"foo">>}]}]}, ExitMsg = {rexi_EXIT, blargh}, - {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), - {ok, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), - {ok, Acc3} = handle_message(ExitMsg, worker(4, Acc0), Acc2), - {ok, Acc4} = handle_message(ExitMsg, worker(5, Acc0), Acc3), - {stop, Acc5} = handle_message(ExitMsg, worker(6, Acc0), Acc4), + {ok, Acc1} = handle_message(Msg, worker(a, [1, 2], Acc0), Acc0), + {ok, Acc2} = handle_message(Msg, worker(b, [1, 2], Acc0), Acc1), + {ok, Acc3} = handle_message(ExitMsg, worker(a, [3, 4], Acc0), Acc2), + {ok, Acc4} = handle_message(ExitMsg, worker(b, [3, 4], Acc0), Acc3), + {stop, Acc5} = handle_message(ExitMsg, worker(c, [3, 4], Acc0), Acc4), Expect = [{ok, [{1, <<"foo">>}]}, {error, internal_server_error}], Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc5), ?assertEqual(Expect, Resps), ?assertEqual(error, resp_health(Resps)). -t_timeout(_) -> - WorkerUUIDs = [ - {#shard{node = a, range = [1, 2]}, [<<"uuid1">>]}, - {#shard{node = b, range = [1, 2]}, [<<"uuid1">>]}, - {#shard{node = c, range = [1, 2]}, [<<"uuid1">>]}, - - {#shard{node = a, range = [3, 4]}, [<<"uuid2">>]}, - {#shard{node = b, range = [3, 4]}, [<<"uuid2">>]}, - {#shard{node = c, range = [3, 4]}, [<<"uuid2">>]} +t_rexi_down_error(_) -> + WorkerUUIDs = #{ + #shard{node = a, range = [1, 2]} => [<<"uuid1">>], + #shard{node = b, range = [1, 2]} => [<<"uuid1">>], + #shard{node = c, range = [1, 2]} => [<<"uuid1">>], + + #shard{node = a, range = [3, 4]} => [<<"uuid2">>], + #shard{node = b, range = [3, 4]} => [<<"uuid2">>], + #shard{node = c, range = [3, 4]} => [<<"uuid2">>] + }, + + Acc0 = #acc{ + worker_uuids = WorkerUUIDs, + resps = maps:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), + uuid_counts = maps:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), + w = 2 + }, + + Msg = {ok, [{ok, [{1, <<"foo">>}]}]}, + {ok, Acc1} = handle_message(Msg, worker(a, [1, 2], Acc0), Acc0), + + DownMsgB = {rexi_DOWN, nodedown, {nil, b}, nil}, + {ok, Acc2} = handle_message(DownMsgB, worker(b, [1, 2], Acc0), Acc1), + + DownMsgC = {rexi_DOWN, nodedown, {nil, c}, nil}, + {ok, Acc3} = handle_message(DownMsgC, worker(c, [3, 4], Acc0), Acc2), + + Expect = [ + {accepted, [{1, <<"foo">>}]}, + {error, internal_server_error} ], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc3), + ?assertEqual(Expect, Resps), + ?assertEqual(error, resp_health(Resps)). + +t_timeout(_) -> + WorkerUUIDs = #{ + #shard{node = a, range = [1, 2]} => [<<"uuid1">>], + #shard{node = b, range = [1, 2]} => [<<"uuid1">>], + #shard{node = c, range = [1, 2]} => [<<"uuid1">>], + + #shard{node = a, range = [3, 4]} => [<<"uuid2">>], + #shard{node = b, range = [3, 4]} => [<<"uuid2">>], + #shard{node = c, range = [3, 4]} => [<<"uuid2">>] + }, Acc0 = #acc{ worker_uuids = WorkerUUIDs, - resps = dict:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), - uuid_counts = dict:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), + resps = maps:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), + uuid_counts = maps:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), w = 2 }, Msg = {ok, [{ok, [{1, <<"foo">>}]}]}, - {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), - {ok, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), - {ok, Acc3} = handle_message(Msg, worker(3, Acc0), Acc2), + {ok, Acc1} = handle_message(Msg, worker(a, [1, 2], Acc0), Acc0), + {ok, Acc2} = handle_message(Msg, worker(b, [1, 2], Acc0), Acc1), + {ok, Acc3} = handle_message(Msg, worker(c, [1, 2], Acc0), Acc2), Acc4 = handle_timeout(Acc3), Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc4), ?assertEqual([{ok, [{1, <<"foo">>}]}, {error, timeout}], Resps). @@ -556,31 +620,124 @@ create_init_acc(W) -> % Create our worker_uuids. We're relying on the fact that % we're using a fake Q=1 db so we don't have to worry % about any hashing here. - WorkerUUIDs = lists:map( - fun(Shard) -> - {Shard#shard{ref = erlang:make_ref()}, [UUID1, UUID2]} - end, - Shards - ), + UUIDs = [UUID1, UUID2], + Workers = [{S#shard{ref = make_ref()}, UUIDs} || S <- Shards], + WorkerUUIDs = maps:from_list(Workers), #acc{ worker_uuids = WorkerUUIDs, - resps = dict:from_list([{UUID1, []}, {UUID2, []}]), - uuid_counts = dict:from_list([{UUID1, 3}, {UUID2, 3}]), + resps = #{UUID1 => [], UUID2 => []}, + uuid_counts = #{UUID1 => 3, UUID2 => 3}, w = W }. +worker(Node, Range, #acc{worker_uuids = WorkerUUIDs}) -> + Workers = maps:keys(WorkerUUIDs), + Pred = fun(#shard{node = N, range = R}) -> + Node =:= N andalso Range =:= R + end, + case lists:filter(Pred, Workers) of + [W] -> W; + _ -> error(not_found) + end. + worker(N, #acc{worker_uuids = WorkerUUIDs}) -> - {Worker, _} = lists:nth(N, WorkerUUIDs), - Worker. + Workers = maps:keys(WorkerUUIDs), + lists:nth(N, lists:sort(Workers)). check_quorum(Acc, Expect) -> - dict:fold( - fun(_Shard, Resps, _) -> + maps:map( + fun(_Shard, Resps) -> ?assertEqual(Expect, has_quorum(Resps, 3, Acc#acc.w)) end, - nil, Acc#acc.resps ). +purge_end_to_end_test_() -> + { + setup, + fun() -> + Ctx = test_util:start_couch([fabric]), + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [{q, 2}, {n, 1}]), + {Ctx, DbName} + end, + fun({Ctx, DbName}) -> + fabric:delete_db(DbName), + test_util:stop_couch(Ctx), + meck:unload() + end, + with([ + ?TDEF(t_purge), + ?TDEF(t_purge_missing_doc_id), + ?TDEF(t_purge_missing_rev) + ]) + }. + +t_purge({_Ctx, DbName}) -> + Rev1 = update_doc(DbName, <<"1">>), + Rev2 = update_doc(DbName, <<"2">>), + Rev3 = update_doc(DbName, <<"3">>), + Res = fabric:purge_docs( + DbName, + [ + {<<"3">>, [Rev3]}, + {<<"1">>, [Rev1]}, + {<<"2">>, [Rev2]} + ], + [] + ), + ?assertMatch({ok, [_, _, _]}, Res), + {ok, [Res3, Res1, Res2]} = Res, + ?assertMatch({ok, [Rev1]}, Res1), + ?assertMatch({ok, [Rev2]}, Res2), + ?assertMatch({ok, [Rev3]}, Res3). + +t_purge_missing_doc_id({_Ctx, DbName}) -> + ?assertMatch({ok, []}, fabric:purge_docs(DbName, [], [])), + Rev1 = update_doc(DbName, <<"1">>), + Rev2 = update_doc(DbName, <<"2">>), + Res = fabric:purge_docs( + DbName, + [ + {<<"3">>, [Rev1]}, + {<<"1">>, [Rev1]}, + {<<"2">>, [Rev2]} + ], + [] + ), + ?assertMatch({ok, [_, _, _]}, Res), + {ok, [Res3, Res1, Res2]} = Res, + ?assertMatch({ok, [Rev1]}, Res1), + ?assertMatch({ok, [Rev2]}, Res2), + ?assertMatch({ok, []}, Res3). + +t_purge_missing_rev({_Ctx, DbName}) -> + Rev1 = update_doc(DbName, <<"1">>), + Rev2 = update_doc(DbName, <<"2">>), + update_doc(DbName, <<"3">>), + Res = fabric:purge_docs( + DbName, + [ + {<<"1">>, [Rev2, Rev1]}, + {<<"2">>, [Rev1]}, + {<<"3">>, []} + ], + [] + ), + ?assertMatch({ok, [_, _, _]}, Res), + {ok, [Res1, Res2, Res3]} = Res, + ?assertMatch({ok, [Rev1]}, Res1), + ?assertMatch({ok, []}, Res2), + ?assertMatch({ok, []}, Res3). + +update_doc(DbName, Id) -> + fabric_util:isolate(fun() -> + Data = binary:encode_hex(crypto:strong_rand_bytes(10)), + Doc = #doc{id = Id, body = {[{<<"foo">>, Data}]}}, + case fabric:update_doc(DbName, Doc, []) of + {ok, Res} -> Res + end + end). + -endif. diff --git a/src/fabric/src/fabric_doc_update.erl b/src/fabric/src/fabric_doc_update.erl index 1f5755de09..a977180bc4 100644 --- a/src/fabric/src/fabric_doc_update.erl +++ b/src/fabric/src/fabric_doc_update.erl @@ -42,11 +42,10 @@ go(DbName, AllDocs0, Opts) -> ), {Workers, _} = lists:unzip(GroupedDocs), RexiMon = fabric_util:create_monitors(Workers), - W = couch_util:get_value(w, Options, integer_to_list(mem3:quorum(DbName))), Acc0 = #acc{ waiting_count = length(Workers), doc_count = length(AllDocs), - w = list_to_integer(W), + w = fabric_util:w_from_opts(DbName, Options), grouped_docs = GroupedDocs, reply = dict:new() }, diff --git a/src/fabric/src/fabric_index_cleanup.erl b/src/fabric/src/fabric_index_cleanup.erl new file mode 100644 index 0000000000..13759ba1d1 --- /dev/null +++ b/src/fabric/src/fabric_index_cleanup.erl @@ -0,0 +1,81 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(fabric_index_cleanup). + +-export([ + cleanup_all_nodes/0, + cleanup_all_nodes/1, + cleanup_this_node/0, + cleanup_this_node/1 +]). + +cleanup_all_nodes() -> + Fun = fun(DbName, _) -> cleanup_all_nodes(DbName) end, + mem3:fold_dbs(Fun, nil), + ok. + +cleanup_all_nodes(DbName) -> + cleanup_indexes(DbName, mem3_util:live_nodes()). + +cleanup_this_node() -> + Fun = fun(DbName, _) -> + case mem3:local_shards(DbName) of + [_ | _] -> cleanup_this_node(DbName); + [] -> ok + end + end, + mem3:fold_dbs(Fun, nil), + ok. + +cleanup_this_node(DbName) -> + cleanup_indexes(DbName, [config:node_name()]). + +cleanup_indexes(DbName, Nodes) -> + try fabric_util:get_design_doc_records(DbName) of + {ok, DDocs} when is_list(DDocs) -> + VSigs = couch_mrview_util:get_signatures_from_ddocs(DbName, DDocs), + DSigs = dreyfus_util:get_signatures_from_ddocs(DbName, DDocs), + NSigs = nouveau_util:get_signatures_from_ddocs(DbName, DDocs), + Shards = [S || S <- mem3:shards(DbName), lists:member(mem3:node(S), Nodes)], + ByNode = maps:groups_from_list(fun mem3:node/1, fun mem3:name/1, Shards), + Fun = fun(Node, Dbs, Acc) -> + Acc1 = send(Node, couch_mrview_cleanup, cleanup, [Dbs, VSigs], Acc), + Acc2 = send(Node, dreyfus_fabric_cleanup, go_local, [DbName, Dbs, DSigs], Acc1), + Acc3 = send(Node, nouveau_fabric_cleanup, go_local, [DbName, Dbs, NSigs], Acc2), + Acc3 + end, + Reqs = maps:fold(Fun, erpc:reqids_new(), ByNode), + recv(DbName, Reqs, fabric_util:abs_request_timeout()); + Error -> + couch_log:error("~p : error fetching ddocs db:~p ~p", [?MODULE, DbName, Error]), + Error + catch + error:database_does_not_exist -> + ok + end. + +send(Node, M, F, A, Reqs) -> + Label = {Node, M, F}, + erpc:send_request(Node, M, F, A, Label, Reqs). + +recv(DbName, Reqs, Timeout) -> + case erpc:receive_response(Reqs, Timeout, true) of + {ok, _Label, Reqs1} -> + recv(DbName, Reqs1, Timeout); + {Error, Label, Reqs1} -> + ErrMsg = "~p : error cleaning indexes db:~p req:~p error:~p", + couch_log:error(ErrMsg, [?MODULE, DbName, Label, Error]), + recv(DbName, Reqs1, Timeout); + no_request -> + ok + end. diff --git a/src/fabric/src/fabric_open_revs.erl b/src/fabric/src/fabric_open_revs.erl index b0a54645a1..b4f95df8a3 100644 --- a/src/fabric/src/fabric_open_revs.erl +++ b/src/fabric/src/fabric_open_revs.erl @@ -83,8 +83,7 @@ handle_message(Reason, Worker, #st{} = St) -> handle_error(Reason, St#st{workers = Workers1, reqs = Reqs1}). init_state(DbName, IdsRevsOpts, Options) -> - DefaultR = integer_to_list(mem3:quorum(DbName)), - R = list_to_integer(couch_util:get_value(r, Options, DefaultR)), + R = fabric_util:r_from_opts(DbName, Options), {ArgRefs, Reqs0} = build_req_map(IdsRevsOpts), ShardMap = build_worker_map(DbName, Reqs0), {Workers, Reqs} = spawn_workers(Reqs0, ShardMap, Options), diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index 67f529e093..31c42c2a94 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -33,6 +33,7 @@ reset_validation_funs/1, set_security/3, set_revs_limit/3, + update_props/4, create_shard_db_doc/2, delete_shard_db_doc/2, get_partition_info/2 @@ -146,9 +147,13 @@ all_docs(DbName, Options, Args0) -> case fabric_util:upgrade_mrargs(Args0) of #mrargs{keys = undefined} = Args -> set_io_priority(DbName, Options), - {ok, Db} = get_or_create_db(DbName, Options), - CB = get_view_cb(Args), - couch_mrview:query_all_docs(Db, Args, CB, Args) + case get_or_create_db(DbName, Options) of + {ok, Db} -> + CB = get_view_cb(Args), + couch_mrview:query_all_docs(Db, Args, CB, Args); + Error -> + rexi:reply(Error) + end end. update_mrview(DbName, {DDocId, Rev}, ViewName, Args0) -> @@ -171,9 +176,13 @@ map_view(DbName, {DDocId, Rev}, ViewName, Args0, DbOptions) -> map_view(DbName, DDoc, ViewName, Args0, DbOptions) -> set_io_priority(DbName, DbOptions), Args = fabric_util:upgrade_mrargs(Args0), - {ok, Db} = get_or_create_db(DbName, DbOptions), - CB = get_view_cb(Args), - couch_mrview:query_view(Db, DDoc, ViewName, Args, CB, Args). + case get_or_create_db(DbName, DbOptions) of + {ok, Db} -> + CB = get_view_cb(Args), + couch_mrview:query_view(Db, DDoc, ViewName, Args, CB, Args); + Error -> + rexi:reply(Error) + end. %% @equiv reduce_view(DbName, DDoc, ViewName, Args0) reduce_view(DbName, DDocInfo, ViewName, Args0) -> @@ -185,10 +194,14 @@ reduce_view(DbName, {DDocId, Rev}, ViewName, Args0, DbOptions) -> reduce_view(DbName, DDoc, ViewName, Args0, DbOptions) -> set_io_priority(DbName, DbOptions), Args = fabric_util:upgrade_mrargs(Args0), - {ok, Db} = get_or_create_db(DbName, DbOptions), - VAcc0 = #vacc{db = Db}, - Callback = fun(Msg, Acc) -> reduce_cb(Msg, Acc, Args#mrargs.extra) end, - couch_mrview:query_view(Db, DDoc, ViewName, Args, Callback, VAcc0). + case get_or_create_db(DbName, DbOptions) of + {ok, Db} -> + VAcc0 = #vacc{db = Db}, + Callback = fun(Msg, Acc) -> reduce_cb(Msg, Acc, Args#mrargs.extra) end, + couch_mrview:query_view(Db, DDoc, ViewName, Args, Callback, VAcc0); + Error -> + rexi:reply(Error) + end. create_db(DbName) -> create_db(DbName, []). @@ -262,6 +275,9 @@ set_revs_limit(DbName, Limit, Options) -> set_purge_infos_limit(DbName, Limit, Options) -> with_db(DbName, Options, {couch_db, set_purge_infos_limit, [Limit]}). +update_props(DbName, K, V, Options) -> + with_db(DbName, Options, {couch_db, update_props, [K, V]}). + open_doc(DbName, DocId, Options) -> with_db(DbName, Options, {couch_db, open_doc, [DocId, Options]}). @@ -341,7 +357,7 @@ compact(ShardName, DesignName) -> {ok, Pid} = couch_index_server:get_index( couch_mrview_index, ShardName, <<"_design/", DesignName/binary>> ), - Ref = erlang:make_ref(), + Ref = make_ref(), Pid ! {'$gen_call', {self(), Ref}, compact}. get_uuid(DbName) -> @@ -458,7 +474,7 @@ get_node_seqs(Db, Nodes) -> PurgeSeq = couch_util:get_value(<<"purge_seq">>, Props), case lists:keyfind(TgtNode, 1, Acc) of {_, OldSeq} -> - NewSeq = erlang:max(OldSeq, PurgeSeq), + NewSeq = max(OldSeq, PurgeSeq), NewEntry = {TgtNode, NewSeq}, NewAcc = lists:keyreplace(TgtNode, 1, Acc, NewEntry), {ok, NewAcc}; @@ -470,13 +486,18 @@ get_node_seqs(Db, Nodes) -> {stop, Acc} end end, - InitAcc = [{list_to_binary(atom_to_list(Node)), 0} || Node <- Nodes], + InitAcc = [{atom_to_binary(Node), 0} || Node <- Nodes], Opts = [{start_key, <>}], {ok, NodeBinSeqs} = couch_db:fold_local_docs(Db, FoldFun, InitAcc, Opts), - [{list_to_existing_atom(binary_to_list(N)), S} || {N, S} <- NodeBinSeqs]. + [{binary_to_existing_atom(N), S} || {N, S} <- NodeBinSeqs]. get_or_create_db(DbName, Options) -> - mem3_util:get_or_create_db_int(DbName, Options). + try + mem3_util:get_or_create_db_int(DbName, Options) + catch + throw:{error, missing_target} -> + error(database_does_not_exist, [DbName]) + end. get_view_cb(#mrargs{extra = Options}) -> case couch_util:get_value(callback, Options) of @@ -621,7 +642,7 @@ make_att_reader({follows, Parser, Ref}) when is_pid(Parser) -> % First time encountering a particular parser pid. Monitor it, % in case it dies, and notify it about us, so it could monitor % us in case we die. - PRef = erlang:monitor(process, Parser), + PRef = monitor(process, Parser), put({mp_parser_ref, Parser}, PRef), Parser ! {hello_from_writer, Ref, WriterPid}, PRef; diff --git a/src/fabric/src/fabric_streams.erl b/src/fabric/src/fabric_streams.erl index 3f9cbdc494..df96ae67cd 100644 --- a/src/fabric/src/fabric_streams.erl +++ b/src/fabric/src/fabric_streams.erl @@ -198,7 +198,7 @@ spawn_worker_cleaner(Coordinator, Workers, ClientReq) when case get(?WORKER_CLEANER) of undefined -> Pid = spawn(fun() -> - erlang:monitor(process, Coordinator), + monitor(process, Coordinator), NodeRefSet = couch_util:set_from_list(shards_to_node_refs(Workers)), cleaner_loop(Coordinator, NodeRefSet, ClientReq) end), @@ -269,7 +269,7 @@ should_clean_workers(_) -> end end), Cleaner = spawn_worker_cleaner(Coord, Workers, undefined), - Ref = erlang:monitor(process, Cleaner), + Ref = monitor(process, Cleaner), Coord ! die, receive {'DOWN', Ref, _, Cleaner, _} -> ok @@ -289,7 +289,7 @@ does_not_fire_if_cleanup_called(_) -> end end), Cleaner = spawn_worker_cleaner(Coord, Workers, undefined), - Ref = erlang:monitor(process, Cleaner), + Ref = monitor(process, Cleaner), cleanup(Workers), Coord ! die, receive @@ -312,7 +312,7 @@ should_clean_additional_worker_too(_) -> end), Cleaner = spawn_worker_cleaner(Coord, Workers, undefined), add_worker_to_cleaner(Coord, #shard{node = 'n2', ref = make_ref()}), - Ref = erlang:monitor(process, Cleaner), + Ref = monitor(process, Cleaner), Coord ! die, receive {'DOWN', Ref, _, Cleaner, _} -> ok @@ -337,7 +337,7 @@ coordinator_is_killed_if_client_disconnects(_) -> % Close the socket and then expect coordinator to be killed ok = gen_tcp:close(Sock), Cleaner = spawn_worker_cleaner(Coord, Workers, ClientReq), - CleanerRef = erlang:monitor(process, Cleaner), + CleanerRef = monitor(process, Cleaner), % Assert the correct behavior on the support platforms (all except Windows so far) case os:type() of {unix, Type} when @@ -378,7 +378,7 @@ coordinator_is_not_killed_if_client_is_connected(_) -> {ok, Sock} = gen_tcp:listen(0, [{active, false}]), ClientReq = mochiweb_request:new(Sock, 'GET', "/foo", {1, 1}, Headers), Cleaner = spawn_worker_cleaner(Coord, Workers, ClientReq), - CleanerRef = erlang:monitor(process, Cleaner), + CleanerRef = monitor(process, Cleaner), % Coordinator should stay up receive {'DOWN', CoordRef, _, Coord, _} -> @@ -430,7 +430,7 @@ submit_jobs_sets_up_cleaner(_) -> meck:wait(2, rexi, cast_ref, '_', 1000), % If we kill the coordinator, the cleaner should kill the workers meck:reset(rexi), - CleanupMon = erlang:monitor(process, Cleaner), + CleanupMon = monitor(process, Cleaner), exit(Coord, kill), receive {'DOWN', CoordRef, _, _, WorkerReason} -> diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl index d0961533f3..1da214f7fe 100644 --- a/src/fabric/src/fabric_util.erl +++ b/src/fabric/src/fabric_util.erl @@ -26,6 +26,7 @@ doc_id_and_rev/1 ]). -export([request_timeout/0, attachments_timeout/0, all_docs_timeout/0, view_timeout/1, timeout/2]). +-export([abs_request_timeout/0]). -export([log_timeout/2, remove_done_workers/2]). -export([is_users_db/1, is_replicator_db/1]). -export([open_cluster_db/1, open_cluster_db/2]). @@ -35,13 +36,14 @@ -export([worker_ranges/1]). -export([get_uuid_prefix_len/0]). -export([isolate/1, isolate/2]). +-export([get_design_doc_records/1]). +-export([w_from_opts/2, r_from_opts/2]). -compile({inline, [{doc_id_and_rev, 1}]}). -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). --include_lib("eunit/include/eunit.hrl"). remove_down_workers(Workers, BadNode) -> remove_down_workers(Workers, BadNode, []). @@ -107,6 +109,15 @@ log_timeout(Workers, EndPoint) -> Workers ). +% Return {abs, MonotonicMSec}. This is a format used by erpc to +% provide an absolute time limit for a collection or requests +% See https://www.erlang.org/doc/apps/kernel/erpc.html#t:timeout_time/0 +% +abs_request_timeout() -> + Timeout = fabric_util:request_timeout(), + NowMSec = erlang:monotonic_time(millisecond), + {abs, NowMSec + Timeout}. + remove_done_workers(Workers, WaitingIndicator) -> [W || {W, WI} <- fabric_dict:to_list(Workers), WI == WaitingIndicator]. @@ -120,7 +131,7 @@ get_db(DbName, Options) -> Shards = Local ++ lists:keysort(#shard.name, SameZone) ++ lists:keysort(#shard.name, DifferentZone), % suppress shards from down nodes - Nodes = [node() | erlang:nodes()], + Nodes = [node() | nodes()], Live = [S || #shard{node = N} = S <- Shards, lists:member(N, Nodes)], % Only accept factors > 1, otherwise our math breaks further down Factor = max(2, config:get_integer("fabric", "shard_timeout_factor", 2)), @@ -130,7 +141,7 @@ get_db(DbName, Options) -> get_shard(Live, Options, Timeout, Factor). get_shard([], _Opts, _Timeout, _Factor) -> - erlang:error({internal_server_error, "No DB shards could be opened."}); + error({internal_server_error, "No DB shards could be opened."}); get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) -> Mon = rexi_monitor:start([rexi_utils:server_pid(Node)]), MFA = {fabric_rpc, open_shard, [Name, [{timeout, Timeout} | Opts]]}, @@ -226,7 +237,7 @@ remove_ancestors([{_, {{not_found, _}, Count}} = Head | Tail], Acc) -> remove_ancestors([{_, {{ok, #doc{revs = {Pos, Revs}}}, Count}} = Head | Tail], Acc) -> Descendants = lists:dropwhile( fun({_, {{ok, #doc{revs = {Pos2, Revs2}}}, _}}) -> - case lists:nthtail(erlang:min(Pos2 - Pos, length(Revs2)), Revs2) of + case lists:nthtail(min(Pos2 - Pos, length(Revs2)), Revs2) of [] -> % impossible to tell if Revs2 is a descendant - assume no true; @@ -250,38 +261,6 @@ create_monitors(Shards) -> MonRefs = lists:usort([rexi_utils:server_pid(N) || #shard{node = N} <- Shards]), rexi_monitor:start(MonRefs). -%% verify only id and rev are used in key. -update_counter_test() -> - Reply = - {ok, #doc{ - id = <<"id">>, - revs = <<"rev">>, - body = <<"body">>, - atts = <<"atts">> - }}, - ?assertEqual( - [{{<<"id">>, <<"rev">>}, {Reply, 1}}], - update_counter(Reply, 1, []) - ). - -remove_ancestors_test() -> - Foo1 = {ok, #doc{revs = {1, [<<"foo">>]}}}, - Foo2 = {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}}, - Bar1 = {ok, #doc{revs = {1, [<<"bar">>]}}}, - Bar2 = {not_found, {1, <<"bar">>}}, - ?assertEqual( - [kv(Bar1, 1), kv(Foo1, 1)], - remove_ancestors([kv(Bar1, 1), kv(Foo1, 1)], []) - ), - ?assertEqual( - [kv(Bar1, 1), kv(Foo2, 2)], - remove_ancestors([kv(Bar1, 1), kv(Foo1, 1), kv(Foo2, 1)], []) - ), - ?assertEqual( - [kv(Bar1, 2)], - remove_ancestors([kv(Bar2, 1), kv(Bar1, 1)], []) - ). - is_replicator_db(DbName) -> path_ends_with(DbName, <<"_replicator">>). @@ -296,15 +275,12 @@ is_users_db(DbName) -> path_ends_with(Path, Suffix) -> Suffix =:= couch_db:dbname_suffix(Path). -open_cluster_db(#shard{dbname = DbName, opts = Options}) -> - case couch_util:get_value(props, Options) of - Props when is_list(Props) -> - {ok, Db} = couch_db:clustered_db(DbName, [{props, Props}]), - Db; - _ -> - {ok, Db} = couch_db:clustered_db(DbName, []), - Db - end. +open_cluster_db(#shard{dbname = DbName}) -> + open_cluster_db(DbName); +open_cluster_db(DbName) when is_binary(DbName) -> + Props = mem3:props(DbName), + {ok, Db} = couch_db:clustered_db(DbName, [{props, Props}]), + Db. open_cluster_db(DbName, Opts) -> % as admin @@ -320,25 +296,22 @@ kv(Item, Count) -> doc_id_and_rev(#doc{id = DocId, revs = {RevNum, [RevHash | _]}}) -> {DocId, {RevNum, RevHash}}. -is_partitioned(DbName0) when is_binary(DbName0) -> - Shards = mem3:shards(fabric:dbname(DbName0)), - is_partitioned(open_cluster_db(hd(Shards))); +is_partitioned(DbName) when is_binary(DbName) -> + is_partitioned(open_cluster_db(DbName)); is_partitioned(Db) -> couch_db:is_partitioned(Db). validate_all_docs_args(DbName, Args) when is_list(DbName) -> validate_all_docs_args(list_to_binary(DbName), Args); validate_all_docs_args(DbName, Args) when is_binary(DbName) -> - Shards = mem3:shards(fabric:dbname(DbName)), - Db = open_cluster_db(hd(Shards)), + Db = open_cluster_db(DbName), validate_all_docs_args(Db, Args); validate_all_docs_args(Db, Args) -> true = couch_db:is_clustered(Db), couch_mrview_util:validate_all_docs_args(Db, Args). validate_args(DbName, DDoc, Args) when is_binary(DbName) -> - Shards = mem3:shards(fabric:dbname(DbName)), - Db = open_cluster_db(hd(Shards)), + Db = open_cluster_db(DbName), validate_args(Db, DDoc, Args); validate_args(Db, DDoc, Args) -> true = couch_db:is_clustered(Db), @@ -397,6 +370,43 @@ worker_ranges(Workers) -> get_uuid_prefix_len() -> config:get_integer("fabric", "uuid_prefix_len", 7). +% Get design #doc{} records. Run in an isolated process. This is often used +% when computing signatures of various indexes +% +get_design_doc_records(DbName) -> + fabric_util:isolate(fun() -> + case fabric:design_docs(DbName) of + {ok, DDocs} when is_list(DDocs) -> + Fun = fun({[_ | _]} = Doc) -> couch_doc:from_json_obj(Doc) end, + {ok, lists:map(Fun, DDocs)}; + Else -> + Else + end + end). + +w_from_opts(Db, Options) -> + quorum_from_opts(Db, couch_util:get_value(w, Options)). + +r_from_opts(Db, Options) -> + quorum_from_opts(Db, couch_util:get_value(r, Options)). + +quorum_from_opts(Db, Val) -> + try + if + is_integer(Val) -> + Val; + is_list(Val) -> + % Compatibility clause. Keep as long as chttpd parses r and w + % request parameters as lists (strings). + list_to_integer(Val); + true -> + mem3:quorum(Db) + end + catch + _:_ -> + mem3:quorum(Db) + end. + % If we issue multiple fabric calls from the same process we have to isolate % them so in case of error they don't pollute the processes dictionary or the % mailbox @@ -405,16 +415,16 @@ isolate(Fun) -> isolate(Fun, infinity). isolate(Fun, Timeout) -> - {Pid, Ref} = erlang:spawn_monitor(fun() -> exit(do_isolate(Fun)) end), + {Pid, Ref} = spawn_monitor(fun() -> exit(do_isolate(Fun)) end), receive {'DOWN', Ref, _, _, {'$isolres', Res}} -> Res; {'DOWN', Ref, _, _, {'$isolerr', Tag, Reason, Stack}} -> erlang:raise(Tag, Reason, Stack) after Timeout -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), exit(Pid, kill), - erlang:error(timeout) + error(timeout) end. do_isolate(Fun) -> @@ -425,6 +435,41 @@ do_isolate(Fun) -> {'$isolerr', Tag, Reason, Stack} end. +-ifdef(TEST). +-include_lib("couch/include/couch_eunit.hrl"). + +%% verify only id and rev are used in key. +update_counter_test() -> + Reply = + {ok, #doc{ + id = <<"id">>, + revs = <<"rev">>, + body = <<"body">>, + atts = <<"atts">> + }}, + ?assertEqual( + [{{<<"id">>, <<"rev">>}, {Reply, 1}}], + update_counter(Reply, 1, []) + ). + +remove_ancestors_test() -> + Foo1 = {ok, #doc{revs = {1, [<<"foo">>]}}}, + Foo2 = {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}}, + Bar1 = {ok, #doc{revs = {1, [<<"bar">>]}}}, + Bar2 = {not_found, {1, <<"bar">>}}, + ?assertEqual( + [kv(Bar1, 1), kv(Foo1, 1)], + remove_ancestors([kv(Bar1, 1), kv(Foo1, 1)], []) + ), + ?assertEqual( + [kv(Bar1, 1), kv(Foo2, 2)], + remove_ancestors([kv(Bar1, 1), kv(Foo1, 1), kv(Foo2, 1)], []) + ), + ?assertEqual( + [kv(Bar1, 2)], + remove_ancestors([kv(Bar2, 1), kv(Bar1, 1)], []) + ). + get_db_timeout_test() -> % Q=1, N=1 ?assertEqual(20000, get_db_timeout(1, 2, 100, 60000)), @@ -468,3 +513,32 @@ get_db_timeout_test() -> % request_timeout was set to infinity, with enough shards it still gets to % 100 min timeout at the start from the exponential logic ?assertEqual(100, get_db_timeout(64, 2, 100, infinity)). + +rw_opts_test_() -> + { + foreach, + fun() -> meck:new(mem3, [passthrough]) end, + fun(_) -> meck:unload() end, + [ + ?TDEF_FE(t_w_opts_get), + ?TDEF_FE(t_r_opts_get) + ] + }. + +t_w_opts_get(_) -> + meck:expect(mem3, quorum, 1, 3), + ?assertEqual(5, w_from_opts(any_db, [{w, 5}])), + ?assertEqual(5, w_from_opts(any_db, [{w, "5"}])), + ?assertEqual(3, w_from_opts(any_db, [{w, some_other_type}])), + ?assertEqual(3, w_from_opts(any_db, [{w, "five"}])), + ?assertEqual(3, w_from_opts(any_db, [])). + +t_r_opts_get(_) -> + meck:expect(mem3, quorum, 1, 3), + ?assertEqual(5, r_from_opts(any_db, [{other_opt, 42}, {r, 5}])), + ?assertEqual(5, r_from_opts(any_db, [{r, "5"}, {something_else, "xyz"}])), + ?assertEqual(3, r_from_opts(any_db, [{r, some_other_type}])), + ?assertEqual(3, r_from_opts(any_db, [{r, "five"}])), + ?assertEqual(3, r_from_opts(any_db, [])). + +-endif. diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl index 2d0133acb5..973e82e98d 100644 --- a/src/fabric/src/fabric_view_all_docs.erl +++ b/src/fabric/src/fabric_view_all_docs.erl @@ -144,7 +144,7 @@ go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) -> fun handle_message/3, State, fabric_util:view_timeout(QueryArgs), - 5000 + fabric_util:timeout("all_docs_view_permsg", "5000") ) of {ok, NewState} -> @@ -239,7 +239,7 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> FinalOffset = case Offset of null -> null; - _ -> erlang:min(Total, Offset + State#collector.skip) + _ -> min(Total, Offset + State#collector.skip) end, Meta = [{total, Total}, {offset, FinalOffset}] ++ @@ -373,7 +373,7 @@ cancel_read_pids(Pids) -> case queue:out(Pids) of {{value, {Pid, Ref}}, RestPids} -> exit(Pid, kill), - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), cancel_read_pids(RestPids); {empty, _} -> ok diff --git a/src/fabric/src/fabric_view_changes.erl b/src/fabric/src/fabric_view_changes.erl index f6695f1630..40e0522688 100644 --- a/src/fabric/src/fabric_view_changes.erl +++ b/src/fabric/src/fabric_view_changes.erl @@ -388,7 +388,7 @@ pack_seqs(Workers) -> SeqList = [{N, R, S} || {#shard{node = N, range = R}, S} <- Workers], SeqSum = lists:sum([fake_packed_seq(S) || {_, _, S} <- SeqList]), Opaque = couch_util:encodeBase64Url(?term_to_bin(SeqList, [compressed])), - ?l2b([integer_to_list(SeqSum), $-, Opaque]). + <<(integer_to_binary(SeqSum))/binary, $-, Opaque/binary>>. % Generate the sequence number used to build the emitted N-... prefix. % @@ -536,7 +536,7 @@ get_old_seq(#shard{range = R} = Shard, SinceSeqs) -> get_db_uuid_shards(DbName) -> % Need to use an isolated process as we are performing a fabric call from - % another fabric call and there is a good chance we'd polute the mailbox + % another fabric call and there is a good chance we'd pollute the mailbox % with returned messages Timeout = fabric_util:request_timeout(), IsolatedFun = fun() -> fabric:db_uuids(DbName) end, diff --git a/src/fabric/src/fabric_view_map.erl b/src/fabric/src/fabric_view_map.erl index cc8ed6cf1c..56389e7df1 100644 --- a/src/fabric/src/fabric_view_map.erl +++ b/src/fabric/src/fabric_view_map.erl @@ -181,7 +181,7 @@ handle_message({meta, Meta0}, {Worker, From}, State) -> offset = Offset }}; false -> - FinalOffset = erlang:min(Total, Offset + State#collector.skip), + FinalOffset = min(Total, Offset + State#collector.skip), Meta = [{total, Total}, {offset, FinalOffset}] ++ case UpdateSeq of diff --git a/src/fabric/test/eunit/fabric_bench_test.erl b/src/fabric/test/eunit/fabric_bench_test.erl index ea514cce8c..f055d24da0 100644 --- a/src/fabric/test/eunit/fabric_bench_test.erl +++ b/src/fabric/test/eunit/fabric_bench_test.erl @@ -59,7 +59,7 @@ t_old_db_deletion_works(_Ctx) -> % Quick db creation and deletion is racy so % we have to wait until the db is gone before proceeding. WaitFun = fun() -> - try mem3_shards:opts_for_db(Db) of + try mem3:props(Db) of _ -> wait catch error:database_does_not_exist -> diff --git a/src/fabric/test/eunit/fabric_db_info_tests.erl b/src/fabric/test/eunit/fabric_db_info_tests.erl index e7df560a1a..9a133ace58 100644 --- a/src/fabric/test/eunit/fabric_db_info_tests.erl +++ b/src/fabric/test/eunit/fabric_db_info_tests.erl @@ -20,7 +20,8 @@ main_test_() -> fun setup/0, fun teardown/1, with([ - ?TDEF(t_update_seq_has_uuids) + ?TDEF(t_update_seq_has_uuids), + ?TDEF(t_update_and_get_props) ]) }. @@ -55,3 +56,38 @@ t_update_seq_has_uuids(_) -> ?assertEqual(UuidFromShard, SeqUuid), ok = fabric:delete_db(DbName, []). + +t_update_and_get_props(_) -> + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [{q, 1}, {n, 1}]), + + {ok, Info} = fabric:get_db_info(DbName), + Props = couch_util:get_value(props, Info), + ?assertEqual({[]}, Props), + + ?assertEqual(ok, fabric:update_props(DbName, <<"foo">>, 100)), + {ok, Info1} = fabric:get_db_info(DbName), + Props1 = couch_util:get_value(props, Info1), + ?assertEqual({[{<<"foo">>, 100}]}, Props1), + + ?assertEqual(ok, fabric:update_props(DbName, bar, 101)), + {ok, Info2} = fabric:get_db_info(DbName), + Props2 = couch_util:get_value(props, Info2), + ?assertEqual( + {[ + {<<"foo">>, 100}, + {bar, 101} + ]}, + Props2 + ), + + ?assertEqual(ok, fabric:update_props(DbName, <<"foo">>, undefined)), + {ok, Info3} = fabric:get_db_info(DbName), + ?assertEqual({[{bar, 101}]}, couch_util:get_value(props, Info3)), + + Res = fabric:update_props(DbName, partitioned, true), + ?assertMatch({error, {bad_request, _}}, Res), + {ok, Info4} = fabric:get_db_info(DbName), + ?assertEqual({[{bar, 101}]}, couch_util:get_value(props, Info4)), + + ok = fabric:delete_db(DbName, []). diff --git a/src/fabric/test/eunit/fabric_moved_shards_seq_tests.erl b/src/fabric/test/eunit/fabric_moved_shards_seq_tests.erl index c3bb0c880e..f46ff927be 100644 --- a/src/fabric/test/eunit/fabric_moved_shards_seq_tests.erl +++ b/src/fabric/test/eunit/fabric_moved_shards_seq_tests.erl @@ -39,7 +39,7 @@ t_shard_moves_avoid_sequence_rewinds(_) -> ok = fabric:create_db(DbName, [{q, 1}, {n, 1}]), lists:foreach( fun(I) -> - update_doc(DbName, #doc{id = erlang:integer_to_binary(I)}) + update_doc(DbName, #doc{id = integer_to_binary(I)}) end, lists:seq(1, DocCnt) ), diff --git a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl index 07e6b1d422..fa6123dfcc 100644 --- a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl @@ -152,7 +152,7 @@ t_no_filter_different_node({DbName, DocId, OldDoc, PSeq}) -> create_purge_checkpoint(DbName, PSeq), % Create a valid purge for a different node - TgtNode = list_to_binary(atom_to_list('notfoo@127.0.0.1')), + TgtNode = atom_to_binary('notfoo@127.0.0.1'), create_purge_checkpoint(DbName, 0, TgtNode), rpc_update_doc(DbName, OldDoc), @@ -164,7 +164,7 @@ t_filter_local_node({DbName, DocId, OldDoc, PSeq}) -> create_purge_checkpoint(DbName, PSeq), % Create a valid purge for a different node - TgtNode = list_to_binary(atom_to_list('notfoo@127.0.0.1')), + TgtNode = atom_to_binary('notfoo@127.0.0.1'), create_purge_checkpoint(DbName, 0, TgtNode), % Add a local node rev to the list of node revs. It should @@ -257,7 +257,7 @@ rpc_update_doc(DbName, Doc) -> rpc_update_doc(DbName, Doc, [RROpt]). rpc_update_doc(DbName, Doc, Opts) -> - Ref = erlang:make_ref(), + Ref = make_ref(), put(rexi_from, {self(), Ref}), fabric_rpc:update_docs(DbName, [Doc], Opts), Reply = test_util:wait(fun() -> @@ -274,4 +274,4 @@ tgt_node() -> 'foo@127.0.0.1'. tgt_node_bin() -> - iolist_to_binary(atom_to_list(tgt_node())). + atom_to_binary(tgt_node()). diff --git a/src/fabric/test/eunit/fabric_rpc_tests.erl b/src/fabric/test/eunit/fabric_rpc_tests.erl index 16bb66bada..39c86e4c47 100644 --- a/src/fabric/test/eunit/fabric_rpc_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_tests.erl @@ -101,7 +101,7 @@ t_no_config_db_create_fails_for_shard_rpc(DbName) -> receive Resp0 -> Resp0 end, - ?assertMatch({Ref, {'rexi_EXIT', {{error, missing_target}, _}}}, Resp). + ?assertMatch({Ref, {'rexi_EXIT', {database_does_not_exist, _}}}, Resp). t_db_create_with_config(DbName) -> MDbName = mem3:dbname(DbName), diff --git a/src/fabric/test/eunit/fabric_tests.erl b/src/fabric/test/eunit/fabric_tests.erl index 1ba5d1bc6a..77327f4458 100644 --- a/src/fabric/test/eunit/fabric_tests.erl +++ b/src/fabric/test/eunit/fabric_tests.erl @@ -47,17 +47,20 @@ teardown({Ctx, DbName}) -> test_util:stop_couch(Ctx). t_cleanup_index_files(_) -> - CheckFun = fun(Res) -> Res =:= ok end, - ?assert(lists:all(CheckFun, fabric:cleanup_index_files())). + ?assertEqual(ok, fabric:cleanup_index_files_this_node()), + ?assertEqual(ok, fabric:cleanup_index_files_all_nodes()). t_cleanup_index_files_with_existing_db({_, DbName}) -> - ?assertEqual(ok, fabric:cleanup_index_files(DbName)). + ?assertEqual(ok, fabric:cleanup_index_files_this_node(DbName)), + ?assertEqual(ok, fabric:cleanup_index_files_all_nodes(DbName)), + ?assertEqual(ok, fabric:cleanup_index_files_this_node(<<"non_existent">>)), + ?assertEqual(ok, fabric:cleanup_index_files_all_nodes(<<"non_existent">>)). t_cleanup_index_files_with_view_data({_, DbName}) -> Sigs = sigs(DbName), Indices = indices(DbName), Purges = purges(DbName), - ok = fabric:cleanup_index_files(DbName), + ok = fabric:cleanup_index_files_all_nodes(DbName), % We haven't inadvertently removed any active index bits ?assertEqual(Sigs, sigs(DbName)), ?assertEqual(Indices, indices(DbName)), @@ -65,7 +68,7 @@ t_cleanup_index_files_with_view_data({_, DbName}) -> t_cleanup_index_files_with_deleted_db(_) -> SomeDb = ?tempdb(), - ?assertEqual(ok, fabric:cleanup_index_files(SomeDb)). + ?assertEqual(ok, fabric:cleanup_index_files_all_nodes(SomeDb)). t_cleanup_index_file_after_ddoc_update({_, DbName}) -> ?assertEqual( @@ -84,7 +87,7 @@ t_cleanup_index_file_after_ddoc_update({_, DbName}) -> ), update_ddoc(DbName, <<"_design/foo">>, <<"bar1">>), - ok = fabric:cleanup_index_files(DbName), + ok = fabric:cleanup_index_files_all_nodes(DbName), {ok, _} = fabric:query_view(DbName, <<"foo">>, <<"bar1">>), % One 4bc stays, da8 should gone and 9e3 is added @@ -120,7 +123,7 @@ t_cleanup_index_file_after_ddoc_delete({_, DbName}) -> ), delete_ddoc(DbName, <<"_design/foo">>), - ok = fabric:cleanup_index_files(DbName), + ok = fabric:cleanup_index_files_all_nodes(DbName), % 4bc stays the same, da8 should be gone ?assertEqual( @@ -137,13 +140,13 @@ t_cleanup_index_file_after_ddoc_delete({_, DbName}) -> ), delete_ddoc(DbName, <<"_design/boo">>), - ok = fabric:cleanup_index_files(DbName), + ok = fabric:cleanup_index_files_all_nodes(DbName), ?assertEqual([], indices(DbName)), ?assertEqual([], purges(DbName)), % cleaning a db with all deleted indices should still work - ok = fabric:cleanup_index_files(DbName), + ok = fabric:cleanup_index_files_all_nodes(DbName), ?assertEqual([], indices(DbName)), ?assertEqual([], purges(DbName)). @@ -298,7 +301,7 @@ t_query_view_configuration({_Ctx, DbName}) -> view_type = map, start_key_docid = <<>>, end_key_docid = <<255>>, - extra = [{view_row_map, true}] + extra = [{validated, true}, {view_row_map, true}] }, Options = [], Accumulator = [], diff --git a/src/global_changes/src/global_changes_server.erl b/src/global_changes/src/global_changes_server.erl index 8be0552f1b..8f502df8ee 100644 --- a/src/global_changes/src/global_changes_server.erl +++ b/src/global_changes/src/global_changes_server.erl @@ -69,7 +69,7 @@ init([]) -> pending_updates = couch_util:new_set(), max_write_delay = MaxWriteDelay, dbname = GlobalChangesDbName, - handler_ref = erlang:monitor(process, Handler) + handler_ref = monitor(process, Handler) }, {ok, State}. @@ -122,7 +122,7 @@ handle_info(flush_updates, State) -> handle_info(start_listener, State) -> {ok, Handler} = global_changes_listener:start(), NewState = State#state{ - handler_ref = erlang:monitor(process, Handler) + handler_ref = monitor(process, Handler) }, {noreply, NewState}; handle_info({'DOWN', Ref, _, _, Reason}, #state{handler_ref = Ref} = State) -> diff --git a/src/ioq/src/ioq.erl b/src/ioq/src/ioq.erl index 8e38c2a001..031626cfed 100644 --- a/src/ioq/src/ioq.erl +++ b/src/ioq/src/ioq.erl @@ -146,7 +146,7 @@ handle_cast(_Msg, State) -> handle_info({Ref, Reply}, State) -> case lists:keytake(Ref, #request.ref, State#state.running) of {value, Request, Remaining} -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), gen_server:reply(Request#request.from, Reply), {noreply, State#state{running = Remaining}, 0}; false -> @@ -225,6 +225,6 @@ choose_next_request(Index, State) -> end. submit_request(#request{} = Request, #state{} = State) -> - Ref = erlang:monitor(process, Request#request.fd), + Ref = monitor(process, Request#request.fd), Request#request.fd ! {'$gen_call', {self(), Ref}, Request#request.msg}, State#state{running = [Request#request{ref = Ref} | State#state.running]}. diff --git a/src/ken/rebar.config.script b/src/ken/rebar.config.script index d219b606ac..61af64c6d9 100644 --- a/src/ken/rebar.config.script +++ b/src/ken/rebar.config.script @@ -12,19 +12,9 @@ % License for the specific language governing permissions and limitations under % the License. -HaveDreyfus = element(1, file:list_dir("../dreyfus")) == ok. - -HastingsHome = os:getenv("HASTINGS_HOME", "../hastings"). -HaveHastings = element(1, file:list_dir(HastingsHome)) == ok. - -CurrOpts = case lists:keyfind(erl_opts, 1, CONFIG) of +ErlOpts = [{i, "../"}] ++ case lists:keyfind(erl_opts, 1, CONFIG) of {erl_opts, Opts} -> Opts; false -> [] end, -NewOpts = - if HaveDreyfus -> [{d, 'HAVE_DREYFUS'}]; true -> [] end ++ - if HaveHastings -> [{d, 'HAVE_HASTINGS'}]; true -> [] end ++ - [{i, "../"}] ++ CurrOpts. - -lists:keystore(erl_opts, 1, CONFIG, {erl_opts, NewOpts}). +lists:keystore(erl_opts, 1, CONFIG, {erl_opts, ErlOpts}). diff --git a/src/ken/src/ken.app.src.script b/src/ken/src/ken.app.src.script index aad00a7b9c..f88d05944e 100644 --- a/src/ken/src/ken.app.src.script +++ b/src/ken/src/ken.app.src.script @@ -10,23 +10,16 @@ % License for the specific language governing permissions and limitations under % the License. -HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}. -HaveHastings = code:lib_dir(hastings) /= {error, bad_name}. - -BaseApplications = [ +Applications = [ kernel, stdlib, couch_log, couch_event, couch, - config + config, + dreyfus ]. -Applications = - if HaveDreyfus -> [dreyfus]; true -> [] end ++ - if HaveHastings -> [hastings]; true -> [] end ++ - BaseApplications. - {application, ken, [ {description, "Ken builds views and search indexes automatically"}, {vsn, git}, diff --git a/src/ken/src/ken_server.erl b/src/ken/src/ken_server.erl index 72c0db8efe..1bc7faea19 100644 --- a/src/ken/src/ken_server.erl +++ b/src/ken/src/ken_server.erl @@ -51,13 +51,7 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/include/mem3.hrl"). --ifdef(HAVE_DREYFUS). -include_lib("dreyfus/include/dreyfus.hrl"). --endif. - --ifdef(HAVE_HASTINGS). --include_lib("hastings/src/hastings.hrl"). --endif. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). @@ -145,20 +139,9 @@ handle_cast({remove, DbName}, State) -> handle_cast({resubmit, DbName}, State) -> ets:delete(ken_resubmit, DbName), handle_cast({add, DbName}, State); -% st index job names have 3 elements, 3rd being 'hastings'. See job record definition. -handle_cast({trigger_update, #job{name = {_, _, hastings}, server = GPid, seq = Seq} = Job}, State) -> - % hastings_index:await will trigger a hastings index update - {Pid, _} = erlang:spawn_monitor( - hastings_index, - await, - [GPid, Seq] - ), - Now = erlang:monotonic_time(), - ets:insert(ken_workers, Job#job{worker_pid = Pid, lru = Now}), - {noreply, State, 0}; handle_cast({trigger_update, #job{name = {_, Index, nouveau}} = Job}, State) -> % nouveau_index_manager:update_index will trigger a search index update. - {Pid, _} = erlang:spawn_monitor( + {Pid, _} = spawn_monitor( nouveau_index_manager, update_index, [Index] @@ -169,7 +152,7 @@ handle_cast({trigger_update, #job{name = {_, Index, nouveau}} = Job}, State) -> % search index job names have 3 elements. See job record definition. handle_cast({trigger_update, #job{name = {_, _, _}, server = GPid, seq = Seq} = Job}, State) -> % dreyfus_index:await will trigger a search index update. - {Pid, _} = erlang:spawn_monitor( + {Pid, _} = spawn_monitor( dreyfus_index, await, [GPid, Seq] @@ -179,7 +162,7 @@ handle_cast({trigger_update, #job{name = {_, _, _}, server = GPid, seq = Seq} = {noreply, State, 0}; handle_cast({trigger_update, #job{name = {_, _}, server = SrvPid, seq = Seq} = Job}, State) -> % couch_index:get_state/2 will trigger a view group index update. - {Pid, _} = erlang:spawn_monitor(couch_index, get_state, [SrvPid, Seq]), + {Pid, _} = spawn_monitor(couch_index, get_state, [SrvPid, Seq]), Now = erlang:monotonic_time(), ets:insert(ken_workers, Job#job{worker_pid = Pid, lru = Now}), {noreply, State, 0}; @@ -294,6 +277,12 @@ design_docs(Name) -> case fabric:design_docs(mem3:dbname(Name)) of {error, {maintenance_mode, _, _Node}} -> {ok, []}; + {error, {nodedown, _Reason}} -> + {ok, []}; + {ok, DDocs} when is_list(DDocs) -> + {ok, DDocs}; + {ok, _Resp} -> + {ok, []}; Else -> Else end @@ -324,18 +313,16 @@ update_ddoc_indexes(Name, #doc{} = Doc, State) -> ok end, SearchUpdated = search_updated(Name, Doc, Seq, State), - STUpdated = st_updated(Name, Doc, Seq, State), NouveauUpdated = nouveau_updated(Name, Doc, Seq, State), - case {ViewUpdated, SearchUpdated, STUpdated, NouveauUpdated} of - {ok, ok, ok, ok} -> ok; + case {ViewUpdated, SearchUpdated, NouveauUpdated} of + {ok, ok, ok} -> ok; _ -> resubmit end. --ifdef(HAVE_DREYFUS). search_updated(Name, Doc, Seq, State) -> case should_update(Doc, <<"indexes">>) of true -> - try dreyfus_index:design_doc_to_indexes(Doc) of + try dreyfus_index:design_doc_to_indexes(Name, Doc) of SIndexes -> update_ddoc_search_indexes(Name, SIndexes, Seq, State) catch _:_ -> @@ -344,28 +331,6 @@ search_updated(Name, Doc, Seq, State) -> false -> ok end. --else. -search_updated(_Name, _Doc, _Seq, _State) -> - ok. --endif. - --ifdef(HAVE_HASTINGS). -st_updated(Name, Doc, Seq, State) -> - case should_update(Doc, <<"st_indexes">>) of - true -> - try hastings_index:design_doc_to_indexes(Doc) of - STIndexes -> update_ddoc_st_indexes(Name, STIndexes, Seq, State) - catch - _:_ -> - ok - end; - false -> - ok - end. --else. -st_updated(_Name, _Doc, _Seq, _State) -> - ok. --endif. nouveau_updated(Name, Doc, Seq, State) -> case should_update(Doc, <<"nouveau">>) of @@ -408,7 +373,6 @@ update_ddoc_views(Name, MRSt, Seq, State) -> ok end. --ifdef(HAVE_DREYFUS). update_ddoc_search_indexes(DbName, Indexes, Seq, State) -> if Indexes =/= [] -> @@ -432,34 +396,6 @@ update_ddoc_search_indexes(DbName, Indexes, Seq, State) -> true -> ok end. --endif. - --ifdef(HAVE_HASTINGS). -update_ddoc_st_indexes(DbName, Indexes, Seq, State) -> - if - Indexes =/= [] -> - % The record name in hastings is #h_idx rather than #index as it is for dreyfus - % Spawn a job for each spatial index in the ddoc - lists:foldl( - fun(#h_idx{ddoc_id = DDocName} = Index, Acc) -> - case hastings_index_manager:get_index(DbName, Index) of - {ok, Pid} -> - case maybe_start_job({DbName, DDocName, hastings}, Pid, Seq, State) of - resubmit -> resubmit; - _ -> Acc - end; - _ -> - % If any job fails, retry the db. - resubmit - end - end, - ok, - Indexes - ); - true -> - ok - end. --endif. update_ddoc_nouveau_indexes(DbName, Indexes, Seq, State) -> if @@ -498,10 +434,6 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) -> true; A < TotalChannels -> case Name of - % st_index name has three elements - {_, _, hastings} -> - {ok, CurrentSeq} = hastings_index:await(Pid, 0), - (Seq - CurrentSeq) < Threshold; % View name has two elements. _ when IsView -> % Since seq is 0, couch_index:get_state/2 won't diff --git a/src/ken/test/ken_server_test.erl b/src/ken/test/ken_server_test.erl index ba2f12da8d..e2cfb2c8f1 100644 --- a/src/ken/test/ken_server_test.erl +++ b/src/ken/test/ken_server_test.erl @@ -79,7 +79,7 @@ start_server(Module, Config) -> stop_server(Key, Cfg) -> {Key, Pid} = lists:keyfind(Key, 1, Cfg), - MRef = erlang:monitor(process, Pid), + MRef = monitor(process, Pid), true = exit(Pid, kill), receive {'DOWN', MRef, _, _, _} -> ok diff --git a/src/mango/rebar.config.script b/src/mango/rebar.config.script index c3083b3ee4..497fd4305b 100644 --- a/src/mango/rebar.config.script +++ b/src/mango/rebar.config.script @@ -13,16 +13,10 @@ % the License. -HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}. +ErlOpts = case lists:keyfind(erl_opts, 1, CONFIG) of + {erl_opts, Opts} -> Opts; + false -> [] +end, -if not HaveDreyfus -> CONFIG; true -> - CurrOpts = case lists:keyfind(erl_opts, 1, CONFIG) of - {erl_opts, Opts} -> Opts; - false -> [] - end, - NewOpts = [ - {d, 'HAVE_DREYFUS'} - ] ++ CurrOpts, - lists:keystore(erl_opts, 1, CONFIG, {erl_opts, NewOpts}) -end. +lists:keystore(erl_opts, 1, CONFIG, {erl_opts, ErlOpts}). diff --git a/src/mango/requirements.txt b/src/mango/requirements.txt index 123824330e..c8e8cff379 100644 --- a/src/mango/requirements.txt +++ b/src/mango/requirements.txt @@ -1,5 +1,5 @@ # nose2 0.13.0, requests, hypothesis version are driven # by the minimum version for python on centos 8 currently nose2==0.13.0 -requests==2.27.1 +requests==2.32.4 hypothesis==6.31.6 diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index 3e9849a093..ee20eaabab 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -26,20 +26,12 @@ -include("mango.hrl"). -include("mango_cursor.hrl"). --ifdef(HAVE_DREYFUS). -define(CURSOR_MODULES, [ mango_cursor_view, mango_cursor_text, mango_cursor_nouveau, mango_cursor_special ]). --else. --define(CURSOR_MODULES, [ - mango_cursor_view, - mango_cursor_nouveau, - mango_cursor_special -]). --endif. -define(SUPERVISOR, mango_cursor_sup). diff --git a/src/mango/src/mango_cursor_nouveau.erl b/src/mango/src/mango_cursor_nouveau.erl index ea9b1640d6..629c91a53a 100644 --- a/src/mango/src/mango_cursor_nouveau.erl +++ b/src/mango/src/mango_cursor_nouveau.erl @@ -49,7 +49,7 @@ create(Db, {Indexes, Trace}, Selector, Opts) -> end, NouveauLimit = get_nouveau_limit(), - Limit = erlang:min(NouveauLimit, couch_util:get_value(limit, Opts, mango_opts:default_limit())), + Limit = min(NouveauLimit, couch_util:get_value(limit, Opts, mango_opts:default_limit())), Skip = couch_util:get_value(skip, Opts, 0), Fields = couch_util:get_value(fields, Opts, all_fields), @@ -297,7 +297,7 @@ update_query_args(CAcc) -> }. get_limit(CAcc) -> - erlang:min(get_nouveau_limit(), CAcc#cacc.limit + CAcc#cacc.skip). + min(get_nouveau_limit(), CAcc#cacc.limit + CAcc#cacc.skip). get_nouveau_limit() -> config:get_integer("nouveau", "max_limit", 200). diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl index 1a79b59761..764d32d845 100644 --- a/src/mango/src/mango_cursor_text.erl +++ b/src/mango/src/mango_cursor_text.erl @@ -12,8 +12,6 @@ -module(mango_cursor_text). --ifdef(HAVE_DREYFUS). - -export([ create/4, explain/1, @@ -55,7 +53,7 @@ create(Db, {Indexes, Trace}, Selector, Opts0) -> Stats = mango_execution_stats:stats_init(DbName), DreyfusLimit = get_dreyfus_limit(), - Limit = erlang:min(DreyfusLimit, couch_util:get_value(limit, Opts, mango_opts:default_limit())), + Limit = min(DreyfusLimit, couch_util:get_value(limit, Opts, mango_opts:default_limit())), Skip = couch_util:get_value(skip, Opts, 0), Fields = couch_util:get_value(fields, Opts, all_fields), @@ -327,7 +325,7 @@ update_query_args(CAcc) -> }. get_limit(CAcc) -> - erlang:min(get_dreyfus_limit(), CAcc#cacc.limit + CAcc#cacc.skip). + min(get_dreyfus_limit(), CAcc#cacc.limit + CAcc#cacc.skip). get_dreyfus_limit() -> config:get_integer("dreyfus", "max_limit", 200). @@ -1226,5 +1224,3 @@ t_explain(_) -> ?assertEqual(Response, explain(Cursor)). -endif. - --endif. diff --git a/src/mango/src/mango_doc.erl b/src/mango/src/mango_doc.erl index f8cb4c63bc..255debf9e6 100644 --- a/src/mango/src/mango_doc.erl +++ b/src/mango/src/mango_doc.erl @@ -399,7 +399,7 @@ get_field({Props}, [Name | Rest], Validator) -> get_field(Values, [Name | Rest], Validator) when is_list(Values) -> % Name might be an integer index into an array try - Pos = list_to_integer(binary_to_list(Name)), + Pos = binary_to_integer(Name), case Pos >= 0 andalso Pos < length(Values) of true -> % +1 because Erlang uses 1 based list indices @@ -441,7 +441,7 @@ rem_field({Props}, [Name | Rest]) -> rem_field(Values, [Name]) when is_list(Values) -> % Name might be an integer index into an array try - Pos = list_to_integer(binary_to_list(Name)), + Pos = binary_to_integer(Name), case Pos >= 0 andalso Pos < length(Values) of true -> % +1 because Erlang uses 1 based list indices @@ -456,7 +456,7 @@ rem_field(Values, [Name]) when is_list(Values) -> rem_field(Values, [Name | Rest]) when is_list(Values) -> % Name might be an integer index into an array try - Pos = list_to_integer(binary_to_list(Name)), + Pos = binary_to_integer(Name), case Pos >= 0 andalso Pos < length(Values) of true -> % +1 because Erlang uses 1 based list indices @@ -494,7 +494,7 @@ set_field({Props}, [Name | Rest], Value) -> set_field(Values, [Name], Value) when is_list(Values) -> % Name might be an integer index into an array try - Pos = list_to_integer(binary_to_list(Name)), + Pos = binary_to_integer(Name), case Pos >= 0 andalso Pos < length(Values) of true -> % +1 because Erlang uses 1 based list indices @@ -509,7 +509,7 @@ set_field(Values, [Name], Value) when is_list(Values) -> set_field(Values, [Name | Rest], Value) when is_list(Values) -> % Name might be an integer index into an array try - Pos = list_to_integer(binary_to_list(Name)), + Pos = binary_to_integer(Name), case Pos >= 0 andalso Pos < length(Values) of true -> % +1 because Erlang uses 1 based list indices diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl index caa85edba4..ea3beda9c4 100644 --- a/src/mango/src/mango_idx_special.erl +++ b/src/mango/src/mango_idx_special.erl @@ -28,16 +28,16 @@ -include("mango.hrl"). validate(_) -> - erlang:exit(invalid_call). + exit(invalid_call). add(_, _) -> - erlang:exit(invalid_call). + exit(invalid_call). remove(_, _) -> - erlang:exit(invalid_call). + exit(invalid_call). from_ddoc(_) -> - erlang:exit(invalid_call). + exit(invalid_call). to_json(#idx{def = all_docs}) -> {[ diff --git a/src/mango/src/mango_json.erl b/src/mango/src/mango_json.erl index ca18d88982..36815feca1 100644 --- a/src/mango/src/mango_json.erl +++ b/src/mango/src/mango_json.erl @@ -105,7 +105,7 @@ to_binary(true) -> to_binary(false) -> false; to_binary(Data) when is_atom(Data) -> - list_to_binary(atom_to_list(Data)); + atom_to_binary(Data); to_binary(Data) when is_number(Data) -> Data; to_binary(Data) when is_binary(Data) -> diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl index f7f6156eae..edcecd4b6f 100644 --- a/src/mango/src/mango_native_proc.erl +++ b/src/mango/src/mango_native_proc.erl @@ -29,6 +29,7 @@ -record(st, { indexes = [], + validators = [], timeout = 5000 }). @@ -94,11 +95,37 @@ handle_call({prompt, [<<"nouveau_index_doc">>, Doc]}, _From, St) -> Else end, {reply, Vals, St}; +handle_call({prompt, [<<"ddoc">>, <<"new">>, DDocId, {DDoc}]}, _From, St) -> + NewSt = + case couch_util:get_value(<<"validate_doc_update">>, DDoc) of + undefined -> + St; + Selector0 -> + Selector = mango_selector:normalize(Selector0), + Validators = couch_util:set_value(DDocId, St#st.validators, Selector), + St#st{validators = Validators} + end, + {reply, true, NewSt}; +handle_call({prompt, [<<"ddoc">>, DDocId, [<<"validate_doc_update">>], Args]}, _From, St) -> + case couch_util:get_value(DDocId, St#st.validators) of + undefined -> + Msg = [<<"validate_doc_update">>, DDocId], + {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}; + Selector -> + [NewDoc, OldDoc, _Ctx, _SecObj] = Args, + Struct = {[{<<"newDoc">>, NewDoc}, {<<"oldDoc">>, OldDoc}]}, + Reply = + case mango_selector:match(Selector, Struct) of + true -> true; + _ -> {[{<<"forbidden">>, <<"document is not valid">>}]} + end, + {reply, Reply, St} + end; handle_call(Msg, _From, St) -> {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. handle_cast(garbage_collect, St) -> - erlang:garbage_collect(), + garbage_collect(), {noreply, St}; handle_cast(stop, St) -> {stop, normal, St}; diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index 42031b7569..9c5b7a96f7 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -563,7 +563,7 @@ match({[{Field, Cond}]}, Value, Cmp) -> match(Cond, SubValue, Cmp) end; match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) -> - erlang:error({unnormalized_selector, Sel}). + error({unnormalized_selector, Sel}). % Returns true if Selector requires all % fields in RequiredFields to exist in any matching documents. diff --git a/src/mango/src/mango_selector_text.erl b/src/mango/src/mango_selector_text.erl index 97e72061c6..fc30a57b61 100644 --- a/src/mango/src/mango_selector_text.erl +++ b/src/mango/src/mango_selector_text.erl @@ -214,7 +214,7 @@ convert(Path, Val) when is_binary(Val); is_number(Val); is_boolean(Val) -> {op_field, {make_field(Path, Val), value_str(Val)}}; % Anything else is a bad selector. convert(_Path, {Props} = Sel) when length(Props) > 1 -> - erlang:error({unnormalized_selector, Sel}). + error({unnormalized_selector, Sel}). to_query_nested(Args) -> QueryArgs = lists:map(fun to_query/1, Args), @@ -369,9 +369,9 @@ value_str(Value) when is_binary(Value) -> <<"\"", Escaped/binary, "\"">> end; value_str(Value) when is_integer(Value) -> - list_to_binary(integer_to_list(Value)); + integer_to_binary(Value); value_str(Value) when is_float(Value) -> - list_to_binary(float_to_list(Value)); + float_to_binary(Value); value_str(true) -> <<"true">>; value_str(false) -> @@ -427,7 +427,7 @@ replace_array_indexes([], NewPartsAcc, HasIntAcc) -> replace_array_indexes([Part | Rest], NewPartsAcc, HasIntAcc) -> {NewPart, HasInt} = try - _ = list_to_integer(binary_to_list(Part)), + _ = binary_to_integer(Part), {<<"[]">>, true} catch _:_ -> diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl index 32d75000b5..837cbf3dbe 100644 --- a/src/mango/src/mango_util.erl +++ b/src/mango/src/mango_util.erl @@ -114,32 +114,32 @@ load_ddoc(Db, DDocId, DbOpts) -> end. defer(Mod, Fun, Args) -> - {Pid, Ref} = erlang:spawn_monitor(?MODULE, do_defer, [Mod, Fun, Args]), + {Pid, Ref} = spawn_monitor(?MODULE, do_defer, [Mod, Fun, Args]), receive {'DOWN', Ref, process, Pid, {mango_defer_ok, Value}} -> Value; {'DOWN', Ref, process, Pid, {mango_defer_throw, Value}} -> erlang:throw(Value); {'DOWN', Ref, process, Pid, {mango_defer_error, Value}} -> - erlang:error(Value); + error(Value); {'DOWN', Ref, process, Pid, {mango_defer_exit, Value}} -> - erlang:exit(Value) + exit(Value) end. do_defer(Mod, Fun, Args) -> - try erlang:apply(Mod, Fun, Args) of + try apply(Mod, Fun, Args) of Resp -> - erlang:exit({mango_defer_ok, Resp}) + exit({mango_defer_ok, Resp}) catch throw:Error:Stack -> couch_log:error("Defered error: ~w~n ~p", [{throw, Error}, Stack]), - erlang:exit({mango_defer_throw, Error}); + exit({mango_defer_throw, Error}); error:Error:Stack -> couch_log:error("Defered error: ~w~n ~p", [{error, Error}, Stack]), - erlang:exit({mango_defer_error, Error}); + exit({mango_defer_error, Error}); exit:Error:Stack -> couch_log:error("Defered error: ~w~n ~p", [{exit, Error}, Stack]), - erlang:exit({mango_defer_exit, Error}) + exit({mango_defer_exit, Error}) end. assert_ejson({Props}) -> diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py index 9a701b06db..cae772b8a8 100644 --- a/src/mango/test/02-basic-find-test.py +++ b/src/mango/test/02-basic-find-test.py @@ -139,7 +139,7 @@ def test_multi_cond_duplicate_field(self): '"location.city":{"$exists":true}}}' ) r = self.db.sess.post(self.db.path("_find"), data=body) - r.raise_for_status() + mango.raise_for_status(r) docs = r.json()["docs"] # expectation is that only the second instance diff --git a/src/mango/test/04-key-tests.py b/src/mango/test/04-key-tests.py index 998f17ee52..1934d4266e 100644 --- a/src/mango/test/04-key-tests.py +++ b/src/mango/test/04-key-tests.py @@ -15,8 +15,9 @@ import unittest TEST_DOCS = [ - {"type": "complex_key", "title": "normal key"}, + {"_id": "100", "type": "complex_key", "title": "normal key"}, { + "_id": "200", "type": "complex_key", "title": "key with dot", "dot.key": "dot's value", @@ -24,14 +25,16 @@ "name.first": "Kvothe", }, { + "_id": "300", "type": "complex_key", "title": "key with peso", "$key": "peso", "deep": {"$key": "deep peso"}, "name": {"first": "Master Elodin"}, }, - {"type": "complex_key", "title": "unicode key", "": "apple"}, + {"_id": "400", "type": "complex_key", "title": "unicode key", "": "apple"}, { + "_id": "500", "title": "internal_fields_format", "utf8-1[]:string": "string", "utf8-2[]:boolean[]": True, diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py index 066b5a3296..fea92f55ce 100644 --- a/src/mango/test/mango.py +++ b/src/mango/test/mango.py @@ -26,6 +26,19 @@ import limit_docs +class MangoException(requests.exceptions.HTTPError): + def __init__(self, err: requests.exceptions.HTTPError): + super().__init__(str(err), response=err.response) + self.request = err.request + self.response = err.response + + def __str__(self): + formatted = super().__str__() + request = "request: {}".format(self.request.body) + response = "response: {}".format(self.response.text) + return "\n".join([formatted, request, response]) + + COUCH_HOST = os.environ.get("COUCH_HOST") or "http://127.0.0.1:15984" COUCH_USER = os.environ.get("COUCH_USER") COUCH_PASS = os.environ.get("COUCH_PASS") @@ -43,9 +56,17 @@ def random_string(n_max): return "".join(random.choice(string.ascii_letters) for _ in range(n)) +def requests_session(): + # use trust_env=False to disable possible .netrc usage + sess = requests.session() + sess.trust_env = False + return sess + + def has_text_service(): - features = requests.get(COUCH_HOST).json()["features"] - return "search" in features + with requests_session() as sess: + features = sess.get(COUCH_HOST).json()["features"] + return "search" in features def clean_up_dbs(): @@ -58,6 +79,13 @@ def delay(n=5, t=0.5): time.sleep(t) +def raise_for_status(resp): + try: + resp.raise_for_status() + except requests.exceptions.HTTPError as err: + raise MangoException(err) from None + + class Concurrently(object): def __init__(self, thread, thread_args, start=True): self.thread = threading.Thread(target=self.wrapper, args=(thread, thread_args)) @@ -82,7 +110,7 @@ def join(self): class Database(object): def __init__(self, dbname): self.dbname = dbname - self.sess = requests.session() + self.sess = requests_session() self.sess.auth = (COUCH_USER, COUCH_PASS) self.sess.headers["Content-Type"] = "application/json" @@ -100,11 +128,11 @@ def create(self, q=1, n=1, partitioned=False): if r.status_code == 404: p = str(partitioned).lower() r = self.sess.put(self.url, params={"q": q, "n": n, "partitioned": p}) - r.raise_for_status() + raise_for_status(r) def delete(self): r = self.sess.delete(self.url) - r.raise_for_status() + raise_for_status(r) def recreate(self): r = self.sess.get(self.url) @@ -124,32 +152,32 @@ def save_doc(self, doc): def save_docs_with_conflicts(self, docs, **kwargs): body = json.dumps({"docs": docs, "new_edits": False}) r = self.sess.post(self.path("_bulk_docs"), data=body, params=kwargs) - r.raise_for_status() + raise_for_status(r) def save_docs(self, docs, **kwargs): for offset in range(0, len(docs), BULK_BATCH_SIZE): chunk = docs[offset : (offset + BULK_BATCH_SIZE)] body = {"docs": chunk} r = self.sess.post(self.path("_bulk_docs"), json=body, params=kwargs) - r.raise_for_status() + raise_for_status(r) for doc, result in zip(chunk, r.json()): doc["_id"] = result["id"] doc["_rev"] = result["rev"] def open_doc(self, docid): r = self.sess.get(self.path(docid)) - r.raise_for_status() + raise_for_status(r) return r.json() def delete_doc(self, docid): r = self.sess.get(self.path(docid)) - r.raise_for_status() + raise_for_status(r) original_rev = r.json()["_rev"] self.sess.delete(self.path(docid), params={"rev": original_rev}) def ddoc_info(self, ddocid): r = self.sess.get(self.path([ddocid, "_info"])) - r.raise_for_status() + raise_for_status(r) return r.json() def create_index( @@ -172,7 +200,7 @@ def create_index( body["index"]["partial_filter_selector"] = partial_filter_selector body = json.dumps(body) r = self.sess.post(self.path("_index"), data=body) - r.raise_for_status() + raise_for_status(r) assert r.json()["id"] is not None assert r.json()["name"] is not None @@ -215,7 +243,7 @@ def create_text_index( body["ddoc"] = ddoc body = json.dumps(body) r = self.sess.post(self.path("_index"), data=body) - r.raise_for_status() + raise_for_status(r) return r.json()["result"] == "created" def list_indexes(self, limit="", skip=""): @@ -224,7 +252,7 @@ def list_indexes(self, limit="", skip=""): if skip != "": skip = "skip=" + str(skip) r = self.sess.get(self.path("_index?" + limit + ";" + skip)) - r.raise_for_status() + raise_for_status(r) return r.json()["indexes"] def get_index(self, ddocid, name): @@ -247,7 +275,7 @@ def get_index(self, ddocid, name): def delete_index(self, ddocid, name, idx_type="json"): path = ["_index", ddocid, idx_type, name] r = self.sess.delete(self.path(path), params={"w": "3"}) - r.raise_for_status() + raise_for_status(r) while len(self.get_index(ddocid, name)) == 1: delay(t=0.1) @@ -306,7 +334,7 @@ def find( else: path = self.path("{}_find".format(ppath)) r = self.sess.post(path, data=body) - r.raise_for_status() + raise_for_status(r) if explain or return_raw: return r.json() else: diff --git a/src/mem3/include/mem3.hrl b/src/mem3/include/mem3.hrl index d97b254696..fa232e8405 100644 --- a/src/mem3/include/mem3.hrl +++ b/src/mem3/include/mem3.hrl @@ -22,7 +22,7 @@ dbname :: binary() | 'undefined', range :: [non_neg_integer() | '$1' | '$2'] | '_' | 'undefined', ref :: reference() | '_' | 'undefined', - opts :: list() | 'undefined' + opts = []:: list() | 'undefined' }). %% Do not reference outside of mem3. @@ -33,7 +33,7 @@ range :: [non_neg_integer() | '$1' | '$2'] | '_', ref :: reference() | 'undefined' | '_', order :: non_neg_integer() | 'undefined' | '_', - opts :: list() + opts = []:: list() }). %% types diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl index c0d64d7e46..f748ff7a0a 100644 --- a/src/mem3/src/mem3.erl +++ b/src/mem3/src/mem3.erl @@ -18,6 +18,7 @@ restart/0, nodes/0, node_info/2, + props/1, shards/1, shards/2, choose_shards/2, n/1, n/2, @@ -38,9 +39,10 @@ -export([db_is_current/1]). -export([shard_creation_time/1]). -export([generate_shard_suffix/0]). +-export([get_db_doc/1, update_db_doc/1]). %% For mem3 use only. --export([name/1, node/1, range/1, engine/1]). +-export([name/1, node/1, range/1]). -include_lib("mem3/include/mem3.hrl"). @@ -72,7 +74,7 @@ restart() -> ]. compare_nodelists() -> Nodes = mem3:nodes(), - AllNodes = erlang:nodes([this, visible]), + AllNodes = nodes([this, visible]), {Replies, BadNodes} = gen_server:multi_call(Nodes, mem3_nodes, get_nodelist), Dict = lists:foldl( fun({Node, Nodelist}, D) -> @@ -115,6 +117,11 @@ nodes() -> node_info(Node, Key) -> mem3_nodes:get_node_info(Node, Key). +-spec props(DbName :: iodata()) -> []. +props(DbName) -> + Opts = mem3_shards:opts_for_db(DbName), + couch_util:get_value(props, Opts, []). + -spec shards(DbName :: iodata()) -> [#shard{}]. shards(DbName) -> shards_int(DbName, []). @@ -135,8 +142,7 @@ shards_int(DbName, Options) -> name = ShardDbName, dbname = ShardDbName, range = [0, (2 bsl 31) - 1], - order = undefined, - opts = [] + order = undefined } ]; ShardDbName -> @@ -147,8 +153,7 @@ shards_int(DbName, Options) -> node = config:node_name(), name = ShardDbName, dbname = ShardDbName, - range = [0, (2 bsl 31) - 1], - opts = [] + range = [0, (2 bsl 31) - 1] } ]; _ -> @@ -260,7 +265,7 @@ choose_shards(DbName, Nodes, Options) -> Suffix = couch_util:get_value(shard_suffix, Options, ""), N = mem3_util:n_val(couch_util:get_value(n, Options), NodeCount), if - N =:= 0 -> erlang:error(no_nodes_in_zone); + N =:= 0 -> error(no_nodes_in_zone); true -> ok end, Q = mem3_util:q_val( @@ -310,7 +315,7 @@ dbname(DbName) when is_list(DbName) -> dbname(DbName) when is_binary(DbName) -> DbName; dbname(_) -> - erlang:error(badarg). + error(badarg). %% @doc Determine if DocId belongs in shard (identified by record or filename) belongs(#shard{} = Shard, DocId) when is_binary(DocId) -> @@ -416,25 +421,13 @@ name(#ordered_shard{name = Name}) -> owner(DbName, DocId, Nodes) -> hd(mem3_util:rotate_list({DbName, DocId}, lists:usort(Nodes))). -engine(#shard{opts = Opts}) -> - engine(Opts); -engine(#ordered_shard{opts = Opts}) -> - engine(Opts); -engine(Opts) when is_list(Opts) -> - case couch_util:get_value(engine, Opts) of - Engine when is_binary(Engine) -> - [{engine, Engine}]; - _ -> - [] - end. - %% Check whether a node is up or down %% side effect: set up a connection to Node if there not yet is one. -spec ping(node()) -> pos_integer() | Error :: term(). ping(Node) -> - [{Node, Res}] = ping_nodes([Node]), + [{Node, Res}] = ping_nodes([Node], ?PING_TIMEOUT_IN_MS), Res. -spec ping(node(), Timeout :: pos_integer()) -> pos_integer() | Error :: term(). @@ -480,7 +473,7 @@ gather_ping_results(Refs, Until, Results) -> end; false -> Fun = fun(Ref, true, Acc) -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), Acc#{Ref => timeout} end, maps:fold(Fun, Results, Refs) @@ -507,7 +500,7 @@ do_ping(Node, Timeout) -> {Tag, Err} end. --spec dead_nodes() -> [node() | Error :: term()]. +-spec dead_nodes() -> [{node(), [node()]}]. %% @doc Returns a list of dead nodes from the cluster. %% @@ -525,32 +518,21 @@ do_ping(Node, Timeout) -> dead_nodes() -> dead_nodes(?PING_TIMEOUT_IN_MS). --spec dead_nodes(Timeout :: pos_integer()) -> [node() | Error :: term()]. +-spec dead_nodes(Timeout :: pos_integer()) -> [{node(), [node()]}]. dead_nodes(Timeout) when is_integer(Timeout), Timeout > 0 -> % Here we are trying to detect overlapping partitions where not all the % nodes connect to each other. For example: n1 connects to n2 and n3, but % n2 and n3 are not connected. - DeadFun = fun() -> - Expected = ordsets:from_list(mem3:nodes()), - Live = ordsets:from_list(mem3_util:live_nodes()), - Dead = ordsets:subtract(Expected, Live), - ordsets:to_list(Dead) + Nodes = [node() | erlang:nodes()], + Expected = erpc:multicall(Nodes, mem3, nodes, [], Timeout), + Live = erpc:multicall(Nodes, mem3_util, live_nodes, [], Timeout), + ZipF = fun + (N, {ok, E}, {ok, L}) -> {N, E -- L}; + (N, _, _) -> {N, Nodes} end, - {Responses, BadNodes} = multicall(DeadFun, Timeout), - AccF = lists:foldl( - fun - (Dead, Acc) when is_list(Dead) -> ordsets:union(Acc, Dead); - (Error, Acc) -> ordsets:union(Acc, [Error]) - end, - ordsets:from_list(BadNodes), - Responses - ), - ordsets:to_list(AccF). - -multicall(Fun, Timeout) when is_integer(Timeout), Timeout > 0 -> - F = fun() -> catch Fun() end, - rpc:multicall(erlang, apply, [F, []], Timeout). + DeadPerNode = lists:zipwith3(ZipF, Nodes, Expected, Live), + lists:sort([{N, lists:sort(D)} || {N, D} <- DeadPerNode, D =/= []]). db_is_current(#shard{name = Name}) -> db_is_current(Name); @@ -587,6 +569,12 @@ strip_shard_suffix(DbName) when is_binary(DbName) -> filename:rootname(DbName) end. +get_db_doc(DocId) -> + mem3_db_doc_updater:get_db_doc(DocId). + +update_db_doc(Doc) -> + mem3_db_doc_updater:update_db_doc(Doc). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/mem3/src/mem3_db_doc_updater.erl b/src/mem3/src/mem3_db_doc_updater.erl new file mode 100644 index 0000000000..f5537f8c9d --- /dev/null +++ b/src/mem3/src/mem3_db_doc_updater.erl @@ -0,0 +1,107 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(mem3_db_doc_updater). + +-behaviour(gen_server). + +-export([ + get_db_doc/1, + update_db_doc/1, + + start_link/0, + + init/1, + handle_call/3, + handle_cast/2 +]). + +-include_lib("couch/include/couch_db.hrl"). + +% Early return shortcut +% +-define(THROW(RES), throw({reply, RES, nil})). + +get_db_doc(DocId) when is_binary(DocId) -> + Timeout = shard_update_timeout_msec(), + gen_server:call(first_node(), {get_db_doc, DocId}, Timeout). + +update_db_doc(#doc{} = Doc) -> + Timeout = shard_update_timeout_msec(), + gen_server:call(first_node(), {update_db_doc, Doc}, Timeout). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init(_) -> + {ok, nil}. + +handle_call({get_db_doc, DocId}, _From, nil = St) -> + {reply, get_db_doc_int(DocId), St}; +handle_call({update_db_doc, #doc{} = Doc}, _From, nil = St) -> + {reply, update_db_doc_int(Doc), St}; +handle_call(Msg, _From, nil = St) -> + {stop, {invalid_call, Msg}, invalid_call, St}. + +handle_cast(Msg, nil = St) -> + {stop, {invalid_cast, Msg}, St}. + +% Private + +update_db_doc_int(#doc{} = Doc) -> + ok = validate_coordinator(), + couch_util:with_db(mem3_sync:shards_db(), fun(Db) -> + try + Res = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]), + ok = replicate_to_all_nodes(shard_update_timeout_msec()), + Res + catch + conflict -> + ?THROW({error, conflict}) + end + end). + +get_db_doc_int(DocId) -> + ok = validate_coordinator(), + ok = replicate_from_all_nodes(shard_update_timeout_msec()), + couch_util:with_db(mem3_sync:shards_db(), fun(Db) -> + case couch_db:open_doc(Db, DocId, [ejson_body]) of + {ok, #doc{deleted = true}} -> ?THROW({error, not_found}); + {ok, #doc{} = Doc} -> {ok, Doc}; + {not_found, _} -> ?THROW({error, not_found}) + end + end). + +validate_coordinator() -> + case hd(mem3_util:live_nodes()) =:= node() of + true -> ok; + false -> ?THROW({error, coordinator_changed}) + end. + +replicate_from_all_nodes(TimeoutMSec) -> + case mem3_util:replicate_dbs_from_all_nodes(TimeoutMSec) of + ok -> ok; + Error -> ?THROW({error, Error}) + end. + +replicate_to_all_nodes(TimeoutMSec) -> + case mem3_util:replicate_dbs_to_all_nodes(TimeoutMSec) of + ok -> ok; + Error -> ?THROW({error, Error}) + end. + +shard_update_timeout_msec() -> + config:get_integer("mem3", "shard_update_timeout_msec", 300000). + +first_node() -> + FirstNode = hd(mem3_util:live_nodes()), + {?MODULE, FirstNode}. diff --git a/src/mem3/src/mem3_hash.erl b/src/mem3/src/mem3_hash.erl index 6dfe3f45ae..a8119dd70e 100644 --- a/src/mem3/src/mem3_hash.erl +++ b/src/mem3/src/mem3_hash.erl @@ -23,33 +23,35 @@ -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). -calculate(#shard{opts = Opts}, DocId) -> - Props = couch_util:get_value(props, Opts, []), - MFA = get_hash_fun_int(Props), +calculate(#shard{dbname = DbName}, DocId) -> + MFA = get_hash_fun(DbName), calculate(MFA, DocId); -calculate(#ordered_shard{opts = Opts}, DocId) -> - Props = couch_util:get_value(props, Opts, []), - MFA = get_hash_fun_int(Props), +calculate(#ordered_shard{dbname = DbName}, DocId) -> + MFA = get_hash_fun(DbName), calculate(MFA, DocId); calculate(DbName, DocId) when is_binary(DbName) -> MFA = get_hash_fun(DbName), calculate(MFA, DocId); +calculate(Props, DocId) when is_list(Props) -> + MFA = get_hash_fun(Props), + calculate(MFA, DocId); calculate({Mod, Fun, Args}, DocId) -> - erlang:apply(Mod, Fun, [DocId | Args]). + apply(Mod, Fun, [DocId | Args]). -get_hash_fun(#shard{opts = Opts}) -> - get_hash_fun_int(Opts); -get_hash_fun(#ordered_shard{opts = Opts}) -> - get_hash_fun_int(Opts); +get_hash_fun(#shard{dbname = DbName}) -> + get_hash_fun(DbName); +get_hash_fun(#ordered_shard{dbname = DbName}) -> + get_hash_fun(DbName); get_hash_fun(DbName0) when is_binary(DbName0) -> DbName = mem3:dbname(DbName0), try - [#shard{opts = Opts} | _] = mem3_shards:for_db(DbName), - get_hash_fun_int(couch_util:get_value(props, Opts, [])) + get_hash_fun_int(mem3:props(DbName)) catch error:database_does_not_exist -> {?MODULE, crc32, []} - end. + end; +get_hash_fun(Props) when is_list(Props) -> + get_hash_fun_int(Props). crc32(Item) when is_binary(Item) -> erlang:crc32(Item); diff --git a/src/mem3/src/mem3_httpd.erl b/src/mem3/src/mem3_httpd.erl index 745fe815ca..fb58bd8d1d 100644 --- a/src/mem3/src/mem3_httpd.erl +++ b/src/mem3/src/mem3_httpd.erl @@ -104,9 +104,7 @@ json_shards([], AccIn) -> List = dict:to_list(AccIn), {lists:sort(List)}; json_shards([#shard{node = Node, range = [B, E]} | Rest], AccIn) -> - HexBeg = couch_util:to_hex(<>), - HexEnd = couch_util:to_hex(<>), - Range = list_to_binary(HexBeg ++ "-" ++ HexEnd), + Range = mem3_util:range_to_hex([B, E]), json_shards(Rest, dict:append(Range, Node, AccIn)). sync_shard(ShardName) -> diff --git a/src/mem3/src/mem3_nodes.erl b/src/mem3/src/mem3_nodes.erl index 47bcd9cc64..334128456b 100644 --- a/src/mem3/src/mem3_nodes.erl +++ b/src/mem3/src/mem3_nodes.erl @@ -66,7 +66,7 @@ handle_call({get_node_info, Node, Key}, _From, State) -> {reply, Resp, State}; handle_call({add_node, Node, NodeInfo}, _From, State) -> gen_event:notify(mem3_events, {add_node, Node}), - ets:insert(?MODULE, {Node, NodeInfo}), + update_ets(Node, NodeInfo), {reply, ok, State}; handle_call({remove_node, Node}, _From, State) -> gen_event:notify(mem3_events, {remove_node, Node}), @@ -95,12 +95,26 @@ handle_info(_Info, State) -> {noreply, State}. %% internal functions - initialize_nodelist() -> DbName = mem3_sync:nodes_db(), {ok, Db} = mem3_util:ensure_exists(DbName), {ok, _} = couch_db:fold_docs(Db, fun first_fold/2, Db, []), + insert_if_missing(Db, [config:node_name() | mem3_seeds:get_seeds()]), + + % when creating the document for the local node, populate + % the placement zone as defined by the COUCHDB_ZONE environment + % variable. This is an additional update on top of the first, + % empty document so that we don't create conflicting revisions + % between different nodes in the cluster when using a seedlist. + case os:getenv("COUCHDB_ZONE") of + false -> + % do not support unsetting a zone. + ok; + Zone -> + set_zone(DbName, config:node_name(), ?l2b(Zone)) + end, + Seq = couch_db:get_update_seq(Db), couch_db:close(Db), Seq. @@ -111,7 +125,7 @@ first_fold(#full_doc_info{deleted = true}, Acc) -> {ok, Acc}; first_fold(#full_doc_info{id = Id} = DocInfo, Db) -> {ok, #doc{body = {Props}}} = couch_db:open_doc(Db, DocInfo, [ejson_body]), - ets:insert(?MODULE, {mem3_util:to_atom(Id), Props}), + update_ets(mem3_util:to_atom(Id), Props), {ok, Db}. listen_for_changes(Since) -> @@ -156,7 +170,7 @@ insert_if_missing(Db, Nodes) -> [_] -> Acc; [] -> - ets:insert(?MODULE, {Node, []}), + update_ets(Node, []), [#doc{id = couch_util:to_binary(Node)} | Acc] end end, @@ -169,3 +183,42 @@ insert_if_missing(Db, Nodes) -> true -> {ok, []} end. + +-spec update_ets(Node :: term(), NodeInfo :: [tuple()]) -> true. +update_ets(Node, NodeInfo) -> + ets:insert(?MODULE, {Node, NodeInfo}). + +% sets the placement zone for the given node document. +-spec set_zone(DbName :: binary(), Node :: string() | binary(), Zone :: binary()) -> ok. +set_zone(DbName, Node, Zone) -> + {ok, Db} = couch_db:open(DbName, [sys_db, ?ADMIN_CTX]), + Props = get_from_db(Db, Node), + CurrentZone = couch_util:get_value(<<"zone">>, Props), + case CurrentZone of + Zone -> + ok; + _ -> + couch_log:info("Setting node zone attribute to ~s~n", [Zone]), + Props1 = couch_util:set_value(<<"zone">>, Props, Zone), + save_to_db(Db, Node, Props1) + end, + couch_db:close(Db), + ok. + +% get a node document from the system nodes db as a property list +-spec get_from_db(Db :: any(), Node :: string() | binary()) -> [tuple()]. +get_from_db(Db, Node) -> + Id = couch_util:to_binary(Node), + {ok, Doc} = couch_db:open_doc(Db, Id, [ejson_body]), + {Props} = couch_doc:to_json_obj(Doc, []), + Props. + +% save a node document (represented as a property list) +% to the system nodes db and update the ETS cache. +-spec save_to_db(Db :: any(), Node :: string() | binary(), Props :: [tuple()]) -> ok. +save_to_db(Db, Node, Props) -> + Doc = couch_doc:from_json_obj({Props}), + #doc{body = {NodeInfo}} = Doc, + {ok, _} = couch_db:update_doc(Db, Doc, []), + update_ets(Node, NodeInfo), + ok. diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl index 3df07a9fdc..f01c12663f 100644 --- a/src/mem3/src/mem3_rep.erl +++ b/src/mem3/src/mem3_rep.erl @@ -217,20 +217,21 @@ verify_purge_checkpoint(DbName, Props) -> %% looking for our push replication history and choosing the %% largest source_seq that has a target_seq =< TgtSeq. find_source_seq(SrcDb, TgtNode, TgtUUIDPrefix, TgtSeq) -> + SrcDbName = couch_db:name(SrcDb), case find_repl_doc(SrcDb, TgtUUIDPrefix) of {ok, TgtUUID, Doc} -> SrcNode = atom_to_binary(config:node_name(), utf8), - find_source_seq_int(Doc, SrcNode, TgtNode, TgtUUID, TgtSeq); + find_source_seq_int(SrcDbName, Doc, SrcNode, TgtNode, TgtUUID, TgtSeq); {not_found, _} -> couch_log:warning( "~p find_source_seq repl doc not_found " "src_db: ~p, tgt_node: ~p, tgt_uuid_prefix: ~p, tgt_seq: ~p", - [?MODULE, SrcDb, TgtNode, TgtUUIDPrefix, TgtSeq] + [?MODULE, SrcDbName, TgtNode, TgtUUIDPrefix, TgtSeq] ), 0 end. -find_source_seq_int(#doc{body = {Props}}, SrcNode0, TgtNode0, TgtUUID, TgtSeq) -> +find_source_seq_int(SrcDbName, #doc{body = {Props}}, SrcNode0, TgtNode0, TgtUUID, TgtSeq) -> SrcNode = case is_atom(SrcNode0) of true -> atom_to_binary(SrcNode0, utf8); @@ -262,9 +263,9 @@ find_source_seq_int(#doc{body = {Props}}, SrcNode0, TgtNode0, TgtUUID, TgtSeq) - [] -> couch_log:warning( "~p find_source_seq_int nil useable history " - "src_node: ~p, tgt_node: ~p, tgt_uuid: ~p, tgt_seq: ~p, " + "src_db: ~s src_node: ~p, tgt_node: ~p, tgt_uuid: ~p, tgt_seq: ~p, " "src_history: ~p", - [?MODULE, SrcNode, TgtNode, TgtUUID, TgtSeq, SrcHistory] + [?MODULE, SrcDbName, SrcNode, TgtNode, TgtUUID, TgtSeq, SrcHistory] ), 0 end. @@ -434,7 +435,7 @@ push_purges(Db, BatchSize, SrcShard, Tgt, HashFun) -> couch_util:get_value(<<"purge_seq">>, Props); {not_found, _} -> Oldest = couch_db:get_oldest_purge_seq(Db), - erlang:max(0, Oldest - 1) + max(0, Oldest - 1) end, BelongsFun = fun(Id) when is_binary(Id) -> case TgtRange of @@ -954,31 +955,31 @@ find_source_seq_int_test_() -> t_unknown_node(_) -> ?assertEqual( - find_source_seq_int(doc_(), <<"foo">>, <<"bing">>, <<"bar_uuid">>, 10), + find_source_seq_int(<<"db">>, doc_(), <<"foo">>, <<"bing">>, <<"bar_uuid">>, 10), 0 ). t_unknown_uuid(_) -> ?assertEqual( - find_source_seq_int(doc_(), <<"foo">>, <<"bar">>, <<"teapot">>, 10), + find_source_seq_int(<<"db">>, doc_(), <<"foo">>, <<"bar">>, <<"teapot">>, 10), 0 ). t_ok(_) -> ?assertEqual( - find_source_seq_int(doc_(), <<"foo">>, <<"bar">>, <<"bar_uuid">>, 100), + find_source_seq_int(<<"db">>, doc_(), <<"foo">>, <<"bar">>, <<"bar_uuid">>, 100), 100 ). t_old_ok(_) -> ?assertEqual( - find_source_seq_int(doc_(), <<"foo">>, <<"bar">>, <<"bar_uuid">>, 84), + find_source_seq_int(<<"db">>, doc_(), <<"foo">>, <<"bar">>, <<"bar_uuid">>, 84), 50 ). t_different_node(_) -> ?assertEqual( - find_source_seq_int(doc_(), <<"foo2">>, <<"bar">>, <<"bar_uuid">>, 92), + find_source_seq_int(<<"db">>, doc_(), <<"foo2">>, <<"bar">>, <<"bar_uuid">>, 92), 31 ). diff --git a/src/mem3/src/mem3_reshard.erl b/src/mem3/src/mem3_reshard.erl index 78537ec38a..c67b74a9fe 100644 --- a/src/mem3/src/mem3_reshard.erl +++ b/src/mem3/src/mem3_reshard.erl @@ -512,7 +512,7 @@ kill_job_int(#job{pid = undefined} = Job) -> kill_job_int(#job{pid = Pid, ref = Ref} = Job) -> couch_log:info("~p kill_job_int ~p", [?MODULE, jobfmt(Job)]), demonitor(Ref, [flush]), - case erlang:is_process_alive(Pid) of + case is_process_alive(Pid) of true -> ok = mem3_reshard_job_sup:terminate_child(Pid); false -> @@ -799,7 +799,7 @@ db_exists(Name) -> -spec db_monitor(pid()) -> no_return(). db_monitor(Server) -> couch_log:notice("~p db monitor ~p starting", [?MODULE, self()]), - EvtRef = erlang:monitor(process, couch_event_server), + EvtRef = monitor(process, couch_event_server), couch_event:register_all(self()), db_monitor_loop(Server, EvtRef). diff --git a/src/mem3/src/mem3_reshard_dbdoc.erl b/src/mem3/src/mem3_reshard_dbdoc.erl index ffed94a8f5..e2f0d0c502 100644 --- a/src/mem3/src/mem3_reshard_dbdoc.erl +++ b/src/mem3/src/mem3_reshard_dbdoc.erl @@ -12,129 +12,48 @@ -module(mem3_reshard_dbdoc). --behaviour(gen_server). - -export([ - update_shard_map/1, - - start_link/0, - - init/1, - handle_call/3, - handle_cast/2, - handle_info/2 + update_shard_map/1 ]). -include_lib("couch/include/couch_db.hrl"). -include("mem3_reshard.hrl"). --spec update_shard_map(#job{}) -> no_return | ok. update_shard_map(#job{source = Source, target = Target} = Job) -> - Node = hd(mem3_util:live_nodes()), + DocId = mem3:dbname(Source#shard.name), JobStr = mem3_reshard_job:jobfmt(Job), - LogMsg1 = "~p : ~p calling update_shard_map node:~p", - couch_log:notice(LogMsg1, [?MODULE, JobStr, Node]), - ServerRef = {?MODULE, Node}, - CallArg = {update_shard_map, Source, Target}, - TimeoutMSec = shard_update_timeout_msec(), + LogMsg1 = "~p : ~p calling update_shard_map", + couch_log:notice(LogMsg1, [?MODULE, JobStr]), try - case gen_server:call(ServerRef, CallArg, TimeoutMSec) of - {ok, _} -> ok; - {error, CallError} -> throw({error, CallError}) + case mem3:get_db_doc(DocId) of + {ok, #doc{} = Doc} -> + #doc{body = Body} = Doc, + NewBody = update_shard_props(Body, Source, Target), + NewDoc = Doc#doc{body = NewBody}, + case mem3:update_db_doc(NewDoc) of + {ok, _} -> + ok; + {error, UpdateError} -> + exit(UpdateError) + end, + LogMsg2 = "~p : ~p update_shard_map returned", + couch_log:notice(LogMsg2, [?MODULE, JobStr]), + TimeoutMSec = shard_update_timeout_msec(), + UntilSec = mem3_reshard:now_sec() + (TimeoutMSec div 1000), + case wait_source_removed(Source, 5, UntilSec) of + true -> + ok; + false -> + exit(shard_update_did_not_propagate) + end; + Error -> + exit(Error) end catch _:Err -> exit(Err) - end, - LogMsg2 = "~p : ~p update_shard_map on node:~p returned", - couch_log:notice(LogMsg2, [?MODULE, JobStr, Node]), - UntilSec = mem3_reshard:now_sec() + (TimeoutMSec div 1000), - case wait_source_removed(Source, 5, UntilSec) of - true -> ok; - false -> exit(shard_update_did_not_propagate) - end. - --spec start_link() -> {ok, pid()} | ignore | {error, term()}. -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -init(_) -> - couch_log:notice("~p start init()", [?MODULE]), - {ok, nil}. - -handle_call({update_shard_map, Source, Target}, _From, State) -> - Res = - try - update_shard_map(Source, Target) - catch - throw:{error, Error} -> - {error, Error} - end, - {reply, Res, State}; -handle_call(Call, From, State) -> - couch_log:error("~p unknown call ~p from: ~p", [?MODULE, Call, From]), - {noreply, State}. - -handle_cast(Cast, State) -> - couch_log:error("~p unexpected cast ~p", [?MODULE, Cast]), - {noreply, State}. - -handle_info(Info, State) -> - couch_log:error("~p unexpected info ~p", [?MODULE, Info]), - {noreply, State}. - -% Private - -update_shard_map(Source, Target) -> - ok = validate_coordinator(), - ok = replicate_from_all_nodes(shard_update_timeout_msec()), - DocId = mem3:dbname(Source#shard.name), - OldDoc = - case mem3_util:open_db_doc(DocId) of - {ok, #doc{deleted = true}} -> - throw({error, missing_source}); - {ok, #doc{} = Doc} -> - Doc; - {not_found, deleted} -> - throw({error, missing_source}); - OpenErr -> - throw({error, {shard_doc_open_error, OpenErr}}) - end, - #doc{body = OldBody} = OldDoc, - NewBody = update_shard_props(OldBody, Source, Target), - {ok, _} = write_shard_doc(OldDoc, NewBody), - ok = replicate_to_all_nodes(shard_update_timeout_msec()), - {ok, NewBody}. - -validate_coordinator() -> - case hd(mem3_util:live_nodes()) =:= node() of - true -> ok; - false -> throw({error, coordinator_changed}) - end. - -replicate_from_all_nodes(TimeoutMSec) -> - case mem3_util:replicate_dbs_from_all_nodes(TimeoutMSec) of - ok -> ok; - Error -> throw({error, Error}) end. -replicate_to_all_nodes(TimeoutMSec) -> - case mem3_util:replicate_dbs_to_all_nodes(TimeoutMSec) of - ok -> ok; - Error -> throw({error, Error}) - end. - -write_shard_doc(#doc{id = Id} = Doc, Body) -> - UpdatedDoc = Doc#doc{body = Body}, - couch_util:with_db(mem3_sync:shards_db(), fun(Db) -> - try - {ok, _} = couch_db:update_doc(Db, UpdatedDoc, []) - catch - conflict -> - throw({error, {conflict, Id, Doc#doc.body, UpdatedDoc}}) - end - end). - update_shard_props({Props0}, #shard{} = Source, [#shard{} | _] = Targets) -> {ByNode0} = couch_util:get_value(<<"by_node">>, Props0, {[]}), ByNodeKV = {<<"by_node">>, {update_by_node(ByNode0, Source, Targets)}}, @@ -205,12 +124,10 @@ node_key(#shard{node = Node}) -> couch_util:to_binary(Node). range_key(#shard{range = [B, E]}) -> - BHex = couch_util:to_hex(<>), - EHex = couch_util:to_hex(<>), - list_to_binary([BHex, "-", EHex]). + mem3_util:range_to_hex([B, E]). shard_update_timeout_msec() -> - config:get_integer("reshard", "shard_upate_timeout_msec", 300000). + config:get_integer("reshard", "shard_update_timeout_msec", 300000). wait_source_removed(#shard{name = Name} = Source, SleepSec, UntilSec) -> case check_source_removed(Source) of diff --git a/src/mem3/src/mem3_reshard_index.erl b/src/mem3/src/mem3_reshard_index.erl index 41e225d221..61390588a9 100644 --- a/src/mem3/src/mem3_reshard_index.erl +++ b/src/mem3/src/mem3_reshard_index.erl @@ -21,7 +21,6 @@ -define(MRVIEW, mrview). -define(DREYFUS, dreyfus). --define(HASTINGS, hastings). -define(NOUVEAU, nouveau). -include_lib("mem3/include/mem3.hrl"). @@ -63,8 +62,7 @@ fabric_design_docs(DbName) -> indices(DbName, Doc) -> mrview_indices(DbName, Doc) ++ nouveau_indices(DbName, Doc) ++ - [dreyfus_indices(DbName, Doc) || has_app(dreyfus)] ++ - [hastings_indices(DbName, Doc) || has_app(hastings)]. + [dreyfus_indices(DbName, Doc) || has_app(dreyfus)]. mrview_indices(DbName, Doc) -> try @@ -101,7 +99,7 @@ nouveau_indices(DbName, Doc) -> dreyfus_indices(DbName, Doc) -> try - Indices = dreyfus_index:design_doc_to_indexes(Doc), + Indices = dreyfus_index:design_doc_to_indexes(DbName, Doc), [{?DREYFUS, DbName, Index} || Index <- Indices] catch Tag:Err -> @@ -110,17 +108,6 @@ dreyfus_indices(DbName, Doc) -> [] end. -hastings_indices(DbName, Doc) -> - try - Indices = hastings_index:design_doc_to_indexes(Doc), - [{?HASTINGS, DbName, Index} || Index <- Indices] - catch - Tag:Err -> - Msg = "~p couldn't get hasting indices ~p ~p ~p:~p", - couch_log:error(Msg, [?MODULE, DbName, Doc, Tag, Err]), - [] - end. - build_index({?MRVIEW, DbName, MRSt} = Ctx, Try) -> ioq:set_io_priority({reshard, DbName}), await_retry( @@ -138,13 +125,6 @@ build_index({?DREYFUS, DbName, DIndex} = Ctx, Try) -> fun dreyfus_index:await/2, Ctx, Try - ); -build_index({?HASTINGS, DbName, HIndex} = Ctx, Try) -> - await_retry( - hastings_index_manager:get_index(DbName, HIndex), - fun hastings_index:await/2, - Ctx, - Try ). await_retry({ok, Pid}, AwaitIndex, {_, DbName, _} = Ctx, Try) -> @@ -196,8 +176,6 @@ index_info({?MRVIEW, DbName, MRSt}) -> GroupName = couch_mrview_index:get(idx_name, MRSt), {DbName, GroupName}; index_info({?DREYFUS, DbName, Index}) -> - {DbName, Index}; -index_info({?HASTINGS, DbName, Index}) -> {DbName, Index}. has_app(App) -> diff --git a/src/mem3/src/mem3_reshard_job.erl b/src/mem3/src/mem3_reshard_job.erl index b8a18b1768..aaacea3c9d 100644 --- a/src/mem3/src/mem3_reshard_job.erl +++ b/src/mem3/src/mem3_reshard_job.erl @@ -180,7 +180,7 @@ set_start_state(#job{split_state = State} = Job) -> undefined -> Fmt1 = "~p recover : unknown state ~s", couch_log:error(Fmt1, [?MODULE, jobfmt(Job)]), - erlang:error({invalid_split_job_recover_state, Job}); + error({invalid_split_job_recover_state, Job}); StartState -> Job#job{split_state = StartState} end. @@ -371,7 +371,7 @@ parent() -> handle_unknown_msg(Job, When, RMsg) -> LogMsg = "~p ~s received an unknown message ~p when in ~s", couch_log:error(LogMsg, [?MODULE, jobfmt(Job), RMsg, When]), - erlang:error({invalid_split_job_message, Job#job.id, When, RMsg}). + error({invalid_split_job_message, Job#job.id, When, RMsg}). initial_copy(#job{} = Job) -> Pid = spawn_link(?MODULE, initial_copy_impl, [Job]), @@ -411,7 +411,7 @@ topoff_impl(#job{source = #shard{} = Source, target = Targets}) -> BatchSize = config:get_integer( "rexi", "shard_split_topoff_batch_size", ?INTERNAL_REP_BATCH_SIZE ), - TMap = maps:from_list([{R, T} || #shard{range = R} = T <- Targets]), + TMap = #{R => T || #shard{range = R} = T <- Targets}, Opts = [ {batch_size, BatchSize}, {batch_count, all}, @@ -552,7 +552,7 @@ check_state(#job{split_state = State} = Job) -> true -> Job; false -> - erlang:error({invalid_shard_split_state, State, Job}) + error({invalid_shard_split_state, State, Job}) end. create_artificial_mem3_rep_checkpoints(#job{} = Job, Seq) -> @@ -665,7 +665,7 @@ reset_target(#job{source = Source, target = Targets} = Job) -> % Should never get here but if we do crash and don't continue LogMsg = "~p : ~p target unexpectedly found in shard map ~p", couch_log:error(LogMsg, [?MODULE, jobfmt(Job), Name]), - erlang:error({target_present_in_shard_map, Name}); + error({target_present_in_shard_map, Name}); {true, false} -> LogMsg = "~p : ~p resetting ~p target", couch_log:warning(LogMsg, [?MODULE, jobfmt(Job), Name]), diff --git a/src/mem3/src/mem3_reshard_sup.erl b/src/mem3/src/mem3_reshard_sup.erl index 5a28359fbf..42da19f7c9 100644 --- a/src/mem3/src/mem3_reshard_sup.erl +++ b/src/mem3/src/mem3_reshard_sup.erl @@ -24,9 +24,6 @@ start_link() -> init(_Args) -> Children = [ - {mem3_reshard_dbdoc, {mem3_reshard_dbdoc, start_link, []}, permanent, infinity, worker, [ - mem3_reshard_dbdoc - ]}, {mem3_reshard_job_sup, {mem3_reshard_job_sup, start_link, []}, permanent, infinity, supervisor, [mem3_reshard_job_sup]}, {mem3_reshard, {mem3_reshard, start_link, []}, permanent, brutal_kill, worker, [ diff --git a/src/mem3/src/mem3_rpc.erl b/src/mem3/src/mem3_rpc.erl index 70fc797dad..a70f6e0362 100644 --- a/src/mem3/src/mem3_rpc.erl +++ b/src/mem3/src/mem3_rpc.erl @@ -197,7 +197,7 @@ load_purge_infos_rpc(DbName, SrcUUID, BatchSize) -> couch_util:get_value(<<"purge_seq">>, Props); {not_found, _} -> Oldest = couch_db:get_oldest_purge_seq(Db), - erlang:max(0, Oldest - 1) + max(0, Oldest - 1) end, FoldFun = fun({PSeq, UUID, Id, Revs}, {Count, Infos, _}) -> NewCount = Count + length(Revs), @@ -265,7 +265,7 @@ compare_rev_epochs([{_, SourceSeq} | _], []) -> SourceSeq; compare_rev_epochs([{_, SourceSeq} | _], [{_, TargetSeq} | _]) -> % The source was moved to a new location independently, take the minimum - erlang:min(SourceSeq, TargetSeq) - 1. + min(SourceSeq, TargetSeq) - 1. %% @doc This adds a new update sequence checkpoint to the replication %% history. Checkpoints are keyed by the source node so that we @@ -382,11 +382,11 @@ rexi_call(Node, MFA, Timeout) -> {Ref, {ok, Reply}} -> Reply; {Ref, Error} -> - erlang:error(Error); + error(Error); {rexi_DOWN, Mon, _, Reason} -> - erlang:error({rexi_DOWN, {Node, Reason}}) + error({rexi_DOWN, {Node, Reason}}) after Timeout -> - erlang:error(timeout) + error(timeout) end after rexi_monitor:stop(Mon) diff --git a/src/mem3/src/mem3_shards.erl b/src/mem3/src/mem3_shards.erl index d37539db42..ba31ffbeb9 100644 --- a/src/mem3/src/mem3_shards.erl +++ b/src/mem3/src/mem3_shards.erl @@ -39,6 +39,7 @@ -define(DBS, mem3_dbs). -define(SHARDS, mem3_shards). +-define(OPTS, mem3_opts). -define(ATIMES, mem3_atimes). -define(OPENERS, mem3_openers). -define(RELISTEN_DELAY, 5000). @@ -46,14 +47,19 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). -opts_for_db(DbName0) -> +opts_for_db(DbName) when is_list(DbName) -> + opts_for_db(list_to_binary(DbName)); +opts_for_db(DbName0) when is_binary(DbName0) -> DbName = mem3:dbname(DbName0), - {ok, Db} = mem3_util:ensure_exists(mem3_sync:shards_db()), - case couch_db:open_doc(Db, DbName, [ejson_body]) of - {ok, #doc{body = {Props}}} -> - mem3_util:get_shard_opts(Props); - {not_found, _} -> - erlang:error(database_does_not_exist, [DbName]) + try ets:lookup(?OPTS, DbName) of + [] -> + load_opts_from_disk(DbName); + [{_, Opts}] -> + gen_server:cast(?MODULE, {cache_hit, DbName}), + Opts + catch + error:badarg -> + load_opts_from_disk(DbName) end. for_db(DbName) -> @@ -80,7 +86,7 @@ for_docid(DbName, DocId) -> for_docid(DbName, DocId, []). for_docid(DbName, DocId, Options) -> - HashKey = mem3_hash:calculate(DbName, DocId), + HashKey = mem3_hash:calculate(mem3:props(DbName), DocId), ShardHead = #shard{ dbname = DbName, range = ['$1', '$2'], @@ -97,13 +103,13 @@ for_docid(DbName, DocId, Options) -> Shards = try ets:select(?SHARDS, [ShardSpec, OrderedShardSpec]) of [] -> - load_shards_from_disk(DbName, DocId); + load_shards_from_disk(DbName, HashKey); Else -> gen_server:cast(?MODULE, {cache_hit, DbName}), Else catch error:badarg -> - load_shards_from_disk(DbName, DocId) + load_shards_from_disk(DbName, HashKey) end, case lists:member(ordered, Options) of true -> Shards; @@ -225,13 +231,9 @@ handle_config_terminate(_Server, _Reason, _State) -> init([]) -> couch_util:set_mqd_off_heap(?MODULE), - ets:new(?SHARDS, [ - bag, - public, - named_table, - {keypos, #shard.dbname}, - {read_concurrency, true} - ]), + CacheEtsOpts = [public, named_table, {read_concurrency, true}, {write_concurrency, auto}], + ets:new(?OPTS, CacheEtsOpts), + ets:new(?SHARDS, [bag, {keypos, #shard.dbname}] ++ CacheEtsOpts), ets:new(?DBS, [set, protected, named_table]), ets:new(?ATIMES, [ordered_set, protected, named_table]), ets:new(?OPENERS, [bag, public, named_table]), @@ -239,6 +241,7 @@ init([]) -> SizeList = config:get("mem3", "shard_cache_size", "25000"), WriteTimeout = config:get_integer("mem3", "shard_write_timeout", 1000), DbName = mem3_sync:shards_db(), + cache_shards_db_props(), ioq:set_io_priority({system, DbName}), UpdateSeq = get_update_seq(), {ok, #st{ @@ -304,6 +307,7 @@ handle_info({'DOWN', _, _, Pid, Reason}, #st{changes_pid = Pid} = St) -> couch_log:notice("~p changes listener died ~p", [?MODULE, Reason]), {St, get_update_seq()} end, + cache_shards_db_props(), erlang:send_after(5000, self(), {start_listener, Seq}), {noreply, NewSt#st{changes_pid = undefined}}; handle_info({start_listener, Seq}, St) -> @@ -324,9 +328,9 @@ terminate(_Reason, #st{changes_pid = Pid}) -> start_changes_listener(SinceSeq) -> Self = self(), - {Pid, _} = erlang:spawn_monitor(fun() -> - erlang:spawn_link(fun() -> - Ref = erlang:monitor(process, Self), + {Pid, _} = spawn_monitor(fun() -> + spawn_link(fun() -> + Ref = monitor(process, Self), receive {'DOWN', Ref, _, _, _} -> ok @@ -361,6 +365,7 @@ listen_for_changes(Since) -> DbName = mem3_sync:shards_db(), ioq:set_io_priority({system, DbName}), {ok, Db} = mem3_util:ensure_exists(DbName), + Args = #changes_args{ feed = "continuous", since = Since, @@ -393,10 +398,11 @@ changes_callback({change, {Change}, _}, _) -> ); {Doc} -> Shards = mem3_util:build_ordered_shards(DbName, Doc), + DbOpts = mem3_util:get_shard_opts(Doc), IdleTimeout = config:get_integer( "mem3", "writer_idle_timeout", 30000 ), - Writer = spawn_shard_writer(DbName, Shards, IdleTimeout), + Writer = spawn_shard_writer(DbName, DbOpts, Shards, IdleTimeout), ets:insert(?OPENERS, {DbName, Writer}), Msg = {cache_insert_change, DbName, Writer, Seq}, gen_server:cast(?MODULE, Msg), @@ -417,18 +423,30 @@ load_shards_from_disk(DbName) when is_binary(DbName) -> couch_stats:increment_counter([mem3, shard_cache, miss]), {ok, Db} = mem3_util:ensure_exists(mem3_sync:shards_db()), try - load_shards_from_db(Db, DbName) + {Shards, _DbOpts} = load_from_db(Db, DbName), + Shards after couch_db:close(Db) end. -load_shards_from_db(ShardDb, DbName) -> +load_opts_from_disk(DbName) when is_binary(DbName) -> + couch_stats:increment_counter([mem3, shard_cache, miss]), + {ok, Db} = mem3_util:ensure_exists(mem3_sync:shards_db()), + try + {_Shards, DbOpts} = load_from_db(Db, DbName), + DbOpts + after + couch_db:close(Db) + end. + +load_from_db(ShardDb, DbName) -> case couch_db:open_doc(ShardDb, DbName, [ejson_body]) of {ok, #doc{body = {Props}}} -> Seq = couch_db:get_update_seq(ShardDb), Shards = mem3_util:build_ordered_shards(DbName, Props), + DbOpts = mem3_util:get_shard_opts(Props), IdleTimeout = config:get_integer("mem3", "writer_idle_timeout", 30000), - case maybe_spawn_shard_writer(DbName, Shards, IdleTimeout) of + case maybe_spawn_shard_writer(DbName, DbOpts, Shards, IdleTimeout) of Writer when is_pid(Writer) -> case ets:insert_new(?OPENERS, {DbName, Writer}) of true -> @@ -440,14 +458,13 @@ load_shards_from_db(ShardDb, DbName) -> ignore -> ok end, - Shards; + {Shards, DbOpts}; {not_found, _} -> - erlang:error(database_does_not_exist, [DbName]) + error(database_does_not_exist, [DbName]) end. -load_shards_from_disk(DbName, DocId) -> +load_shards_from_disk(DbName, HashKey) -> Shards = load_shards_from_disk(DbName), - HashKey = mem3_hash:calculate(hd(Shards), DocId), [S || S <- Shards, in_range(S, HashKey)]. in_range(Shard, HashKey) -> @@ -474,6 +491,7 @@ create_if_missing(ShardName) -> cache_insert(#st{cur_size = Cur} = St, DbName, Writer, Timeout) -> NewATime = couch_util:unique_monotonic_integer(), true = ets:delete(?SHARDS, DbName), + true = ets:delete(?OPTS, DbName), flush_write(DbName, Writer, Timeout), case ets:lookup(?DBS, DbName) of [{DbName, ATime}] -> @@ -489,6 +507,7 @@ cache_insert(#st{cur_size = Cur} = St, DbName, Writer, Timeout) -> cache_remove(#st{cur_size = Cur} = St, DbName) -> true = ets:delete(?SHARDS, DbName), + true = ets:delete(?OPTS, DbName), case ets:lookup(?DBS, DbName) of [{DbName, ATime}] -> true = ets:delete(?DBS, DbName), @@ -515,6 +534,7 @@ cache_free(#st{max_size = Max, cur_size = Cur} = St) when Max =< Cur -> true = ets:delete(?ATIMES, ATime), true = ets:delete(?DBS, DbName), true = ets:delete(?SHARDS, DbName), + true = ets:delete(?OPTS, DbName), cache_free(St#st{cur_size = Cur - 1}); cache_free(St) -> St. @@ -522,24 +542,32 @@ cache_free(St) -> cache_clear(St) -> true = ets:delete_all_objects(?DBS), true = ets:delete_all_objects(?SHARDS), + true = ets:delete_all_objects(?OPTS), true = ets:delete_all_objects(?ATIMES), St#st{cur_size = 0}. -maybe_spawn_shard_writer(DbName, Shards, IdleTimeout) -> - case ets:member(?OPENERS, DbName) of +maybe_spawn_shard_writer(DbName, DbOpts, Shards, IdleTimeout) -> + try ets:member(?OPENERS, DbName) of true -> ignore; false -> - spawn_shard_writer(DbName, Shards, IdleTimeout) + spawn_shard_writer(DbName, DbOpts, Shards, IdleTimeout) + catch + error:badarg -> + % We might have been called before mem3 finished initializing + % from the error:badarg clause in for_db/2, for instance, so + % we shouldn't expect ?OPENERS to exist yet + ignore end. -spawn_shard_writer(DbName, Shards, IdleTimeout) -> - erlang:spawn(fun() -> shard_writer(DbName, Shards, IdleTimeout) end). +spawn_shard_writer(DbName, DbOpts, Shards, IdleTimeout) -> + spawn(fun() -> shard_writer(DbName, DbOpts, Shards, IdleTimeout) end). -shard_writer(DbName, Shards, IdleTimeout) -> +shard_writer(DbName, DbOpts, Shards, IdleTimeout) -> try receive write -> + true = ets:insert(?OPTS, {DbName, DbOpts}), true = ets:insert(?SHARDS, Shards); cancel -> ok @@ -551,15 +579,15 @@ shard_writer(DbName, Shards, IdleTimeout) -> end. flush_write(DbName, Writer, WriteTimeout) -> - Ref = erlang:monitor(process, Writer), + Ref = monitor(process, Writer), Writer ! write, receive {'DOWN', Ref, _, _, normal} -> ok; {'DOWN', Ref, _, _, Error} -> - erlang:exit({mem3_shards_bad_write, Error}) + exit({mem3_shards_bad_write, Error}) after WriteTimeout -> - erlang:exit({mem3_shards_write_timeout, DbName}) + exit({mem3_shards_write_timeout, DbName}) end. filter_shards_by_range(Range, Shards) -> @@ -571,9 +599,23 @@ filter_shards_by_range(Range, Shards) -> Shards ). +cache_shards_db_props() -> + DbName = mem3_sync:shards_db(), + {ok, Db} = mem3_util:ensure_exists(DbName), + try + DbProps = couch_db:get_props(Db), + DbOpts = [{props, DbProps}], + ets:insert(?OPTS, {DbName, DbOpts}) + catch + error:badarg -> + ok + after + couch_db:close(Db) + end. + -ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_eunit.hrl"). -define(DB, <<"eunit_db_name">>). -define(INFINITY, 99999999). @@ -588,22 +630,23 @@ mem3_shards_test_() -> fun setup/0, fun teardown/1, [ - t_maybe_spawn_shard_writer_already_exists(), - t_maybe_spawn_shard_writer_new(), - t_flush_writer_exists_normal(), - t_flush_writer_times_out(), - t_flush_writer_crashes(), - t_writer_deletes_itself_when_done(), - t_writer_does_not_delete_other_writers_for_same_shard(), - t_spawn_writer_in_load_shards_from_db(), - t_cache_insert_takes_new_update(), - t_cache_insert_ignores_stale_update_and_kills_worker() + ?TDEF_FE(t_maybe_spawn_shard_writer_already_exists), + ?TDEF_FE(t_maybe_spawn_shard_writer_new), + ?TDEF_FE(t_flush_writer_exists_normal), + ?TDEF_FE(t_flush_writer_times_out), + ?TDEF_FE(t_flush_writer_crashes), + ?TDEF_FE(t_writer_deletes_itself_when_done), + ?TDEF_FE(t_writer_does_not_delete_other_writers_for_same_shard), + ?TDEF_FE(t_spawn_writer_in_load_shards_from_db), + ?TDEF_FE(t_cache_insert_takes_new_update), + ?TDEF_FE(t_cache_insert_ignores_stale_update_and_kills_worker) ] } }. setup_all() -> ets:new(?SHARDS, [bag, public, named_table, {keypos, #shard.dbname}]), + ets:new(?OPTS, [set, public, named_table, {read_concurrency, true}, {write_concurrency, auto}]), ets:new(?OPENERS, [bag, public, named_table]), ets:new(?DBS, [set, public, named_table]), ets:new(?ATIMES, [ordered_set, public, named_table]), @@ -615,142 +658,130 @@ teardown_all(_) -> ets:delete(?ATIMES), ets:delete(?DBS), ets:delete(?OPENERS), - ets:delete(?SHARDS). + ets:delete(?SHARDS), + ets:delete(?OPTS). setup() -> ets:delete_all_objects(?ATIMES), ets:delete_all_objects(?DBS), ets:delete_all_objects(?OPENERS), - ets:delete_all_objects(?SHARDS). + ets:delete_all_objects(?SHARDS), + ets:delete_all_objects(?OPTS). teardown(_) -> ok. -t_maybe_spawn_shard_writer_already_exists() -> - ?_test(begin - ets:insert(?OPENERS, {?DB, self()}), - Shards = mock_shards(), - WRes = maybe_spawn_shard_writer(?DB, Shards, ?INFINITY), - ?assertEqual(ignore, WRes) - end). - -t_maybe_spawn_shard_writer_new() -> - ?_test(begin - Shards = mock_shards(), - WPid = maybe_spawn_shard_writer(?DB, Shards, 1000), - WRef = erlang:monitor(process, WPid), - ?assert(is_pid(WPid)), - ?assert(is_process_alive(WPid)), - WPid ! write, - ?assertEqual(normal, wait_writer_result(WRef)), - ?assertEqual(Shards, ets:tab2list(?SHARDS)) - end). - -t_flush_writer_exists_normal() -> - ?_test(begin - Shards = mock_shards(), - WPid = spawn_link_mock_writer(?DB, Shards, ?INFINITY), - ?assertEqual(ok, flush_write(?DB, WPid, ?INFINITY)), - ?assertEqual(Shards, ets:tab2list(?SHARDS)) - end). - -t_flush_writer_times_out() -> - ?_test(begin - WPid = spawn(fun() -> - receive - will_never_receive_this -> ok - end - end), - Error = {mem3_shards_write_timeout, ?DB}, - ?assertExit(Error, flush_write(?DB, WPid, 100)), - exit(WPid, kill) - end). - -t_flush_writer_crashes() -> - ?_test(begin - WPid = spawn(fun() -> - receive - write -> exit('kapow!') - end - end), - Error = {mem3_shards_bad_write, 'kapow!'}, - ?assertExit(Error, flush_write(?DB, WPid, 1000)) - end). - -t_writer_deletes_itself_when_done() -> - ?_test(begin - Shards = mock_shards(), - WPid = spawn_link_mock_writer(?DB, Shards, ?INFINITY), - WRef = erlang:monitor(process, WPid), - ets:insert(?OPENERS, {?DB, WPid}), - WPid ! write, - ?assertEqual(normal, wait_writer_result(WRef)), - ?assertEqual(Shards, ets:tab2list(?SHARDS)), - ?assertEqual([], ets:tab2list(?OPENERS)) - end). - -t_writer_does_not_delete_other_writers_for_same_shard() -> - ?_test(begin - Shards = mock_shards(), - WPid = spawn_link_mock_writer(?DB, Shards, ?INFINITY), - WRef = erlang:monitor(process, WPid), - ets:insert(?OPENERS, {?DB, WPid}), - % should not be deleted - ets:insert(?OPENERS, {?DB, self()}), - WPid ! write, - ?assertEqual(normal, wait_writer_result(WRef)), - ?assertEqual(Shards, ets:tab2list(?SHARDS)), - ?assertEqual(1, ets:info(?OPENERS, size)), - ?assertEqual([{?DB, self()}], ets:tab2list(?OPENERS)) - end). - -t_spawn_writer_in_load_shards_from_db() -> - ?_test(begin - meck:expect(couch_db, open_doc, 3, {ok, #doc{body = {[]}}}), - meck:expect(couch_db, get_update_seq, 1, 1), - meck:expect(mem3_util, build_ordered_shards, 2, mock_shards()), - % register to get cache_insert cast - erlang:register(?MODULE, self()), - load_shards_from_db(test_util:fake_db([{name, <<"testdb">>}]), ?DB), - meck:validate(couch_db), - meck:validate(mem3_util), - Cast = - receive - {'$gen_cast', Msg} -> Msg - after 1000 -> - timeout - end, - ?assertMatch({cache_insert, ?DB, Pid, 1} when is_pid(Pid), Cast), - {cache_insert, _, WPid, _} = Cast, - exit(WPid, kill), - ?assertEqual([{?DB, WPid}], ets:tab2list(?OPENERS)), - meck:unload(couch_db), - meck:unload(mem3_util) - end). - -t_cache_insert_takes_new_update() -> - ?_test(begin - Shards = mock_shards(), - WPid = spawn_link_mock_writer(?DB, Shards, ?INFINITY), - Msg = {cache_insert, ?DB, WPid, 2}, - {noreply, NewState} = handle_cast(Msg, mock_state(1)), - ?assertMatch(#st{cur_size = 1}, NewState), - ?assertEqual(Shards, ets:tab2list(?SHARDS)), - ?assertEqual([], ets:tab2list(?OPENERS)) - end). - -t_cache_insert_ignores_stale_update_and_kills_worker() -> - ?_test(begin - Shards = mock_shards(), - WPid = spawn_link_mock_writer(?DB, Shards, ?INFINITY), - WRef = erlang:monitor(process, WPid), - Msg = {cache_insert, ?DB, WPid, 1}, - {noreply, NewState} = handle_cast(Msg, mock_state(2)), - ?assertEqual(normal, wait_writer_result(WRef)), - ?assertMatch(#st{cur_size = 0}, NewState), - ?assertEqual([], ets:tab2list(?SHARDS)), - ?assertEqual([], ets:tab2list(?OPENERS)) - end). +t_maybe_spawn_shard_writer_already_exists(_) -> + ets:insert(?OPENERS, {?DB, self()}), + Shards = mock_shards(), + WRes = maybe_spawn_shard_writer(?DB, [{x, y}], Shards, ?INFINITY), + ?assertEqual(ignore, WRes). + +t_maybe_spawn_shard_writer_new(_) -> + Shards = mock_shards(), + WPid = maybe_spawn_shard_writer(?DB, [{x, y}], Shards, 1000), + WRef = monitor(process, WPid), + ?assert(is_pid(WPid)), + ?assert(is_process_alive(WPid)), + WPid ! write, + ?assertEqual(normal, wait_writer_result(WRef)), + ?assertEqual(Shards, ets:tab2list(?SHARDS)), + ?assertEqual([{?DB, [{x, y}]}], ets:tab2list(?OPTS)). + +t_flush_writer_exists_normal(_) -> + Shards = mock_shards(), + WPid = spawn_link_mock_writer(?DB, [{x, y}], Shards, ?INFINITY), + ?assertEqual(ok, flush_write(?DB, WPid, ?INFINITY)), + ?assertEqual(Shards, ets:tab2list(?SHARDS)), + ?assertEqual([{?DB, [{x, y}]}], ets:tab2list(?OPTS)). + +t_flush_writer_times_out(_) -> + WPid = spawn(fun() -> + receive + will_never_receive_this -> ok + end + end), + Error = {mem3_shards_write_timeout, ?DB}, + ?assertExit(Error, flush_write(?DB, WPid, 100)), + exit(WPid, kill). + +t_flush_writer_crashes(_) -> + WPid = spawn(fun() -> + receive + write -> exit('kapow!') + end + end), + Error = {mem3_shards_bad_write, 'kapow!'}, + ?assertExit(Error, flush_write(?DB, WPid, 1000)). + +t_writer_deletes_itself_when_done(_) -> + Shards = mock_shards(), + WPid = spawn_link_mock_writer(?DB, [{x, y}], Shards, ?INFINITY), + WRef = monitor(process, WPid), + ets:insert(?OPENERS, {?DB, WPid}), + WPid ! write, + ?assertEqual(normal, wait_writer_result(WRef)), + ?assertEqual(Shards, ets:tab2list(?SHARDS)), + ?assertEqual([{?DB, [{x, y}]}], ets:tab2list(?OPTS)), + ?assertEqual([], ets:tab2list(?OPENERS)). + +t_writer_does_not_delete_other_writers_for_same_shard(_) -> + Shards = mock_shards(), + WPid = spawn_link_mock_writer(?DB, [{x, y}], Shards, ?INFINITY), + WRef = monitor(process, WPid), + ets:insert(?OPENERS, {?DB, WPid}), + % should not be deleted + ets:insert(?OPENERS, {?DB, self()}), + WPid ! write, + ?assertEqual(normal, wait_writer_result(WRef)), + ?assertEqual(Shards, ets:tab2list(?SHARDS)), + ?assertEqual([{?DB, [{x, y}]}], ets:tab2list(?OPTS)), + ?assertEqual(1, ets:info(?OPENERS, size)), + ?assertEqual([{?DB, self()}], ets:tab2list(?OPENERS)). + +t_spawn_writer_in_load_shards_from_db(_) -> + meck:expect(couch_db, open_doc, 3, {ok, #doc{body = {[]}}}), + meck:expect(couch_db, get_update_seq, 1, 1), + meck:expect(mem3_util, build_ordered_shards, 2, mock_shards()), + % register to get cache_insert cast + erlang:register(?MODULE, self()), + load_from_db(test_util:fake_db([{name, <<"testdb">>}]), ?DB), + meck:validate(couch_db), + meck:validate(mem3_util), + Cast = + receive + {'$gen_cast', Msg} -> Msg + after 1000 -> + timeout + end, + ?assertMatch({cache_insert, ?DB, Pid, 1} when is_pid(Pid), Cast), + {cache_insert, _, WPid, _} = Cast, + exit(WPid, kill), + ?assertEqual([{?DB, WPid}], ets:tab2list(?OPENERS)), + meck:unload(couch_db), + meck:unload(mem3_util). + +t_cache_insert_takes_new_update(_) -> + Shards = mock_shards(), + WPid = spawn_link_mock_writer(?DB, [{x, y}], Shards, ?INFINITY), + Msg = {cache_insert, ?DB, WPid, 2}, + {noreply, NewState} = handle_cast(Msg, mock_state(1)), + ?assertMatch(#st{cur_size = 1}, NewState), + ?assertEqual(Shards, ets:tab2list(?SHARDS)), + ?assertEqual([{?DB, [{x, y}]}], ets:tab2list(?OPTS)), + ?assertEqual([], ets:tab2list(?OPENERS)). + +t_cache_insert_ignores_stale_update_and_kills_worker(_) -> + Shards = mock_shards(), + WPid = spawn_link_mock_writer(?DB, [{x, y}], Shards, ?INFINITY), + WRef = monitor(process, WPid), + Msg = {cache_insert, ?DB, WPid, 1}, + {noreply, NewState} = handle_cast(Msg, mock_state(2)), + ?assertEqual(normal, wait_writer_result(WRef)), + ?assertMatch(#st{cur_size = 0}, NewState), + ?assertEqual([], ets:tab2list(?SHARDS)), + ?assertEqual([], ets:tab2list(?OPTS)), + ?assertEqual([], ets:tab2list(?OPENERS)). mock_state(UpdateSeq) -> #st{ @@ -778,8 +809,8 @@ wait_writer_result(WRef) -> timeout end. -spawn_link_mock_writer(Db, Shards, Timeout) -> - erlang:spawn_link(fun() -> shard_writer(Db, Shards, Timeout) end). +spawn_link_mock_writer(Db, DbOpts, Shards, Timeout) -> + spawn_link(fun() -> shard_writer(Db, DbOpts, Shards, Timeout) end). mem3_shards_changes_test_() -> { @@ -798,7 +829,7 @@ should_kill_changes_listener_on_shutdown() -> {ok, Pid} = ?MODULE:start_link(), {ok, ChangesPid} = get_changes_pid(), ?assert(is_process_alive(ChangesPid)), - true = erlang:unlink(Pid), + true = unlink(Pid), true = test_util:stop_sync_throw( ChangesPid, fun() -> exit(Pid, shutdown) end, wait_timeout ), diff --git a/src/mem3/src/mem3_sup.erl b/src/mem3/src/mem3_sup.erl index 862ef6b50b..dc8bd854ee 100644 --- a/src/mem3/src/mem3_sup.erl +++ b/src/mem3/src/mem3_sup.erl @@ -18,19 +18,40 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init(_Args) -> + % Some startup order constraints based on call dependencies: + % + % * mem3_events gen_event should be started before all the others + % + % * mem3_nodes gen_server is needed so everyone can call mem3:nodes() + % + % * mem3_sync_nodes needs to run before mem3_sync and + % mem3_sync_event_listener, so they can both can call + % mem3_sync_nodes:add/1 + % + % * mem3_distribution force connects nodes from mem3:nodes(), so start it + % before mem3_sync since mem3_sync:initial_sync/0 expects the connected + % nodes to be there when calling mem3_sync_nodes:add(nodes()) + % + % * mem3_sync_event_listener has to start after mem3_sync, so it can call + % mem3_sync:push/2 + % + % * mem3_seeds and mem3_reshard_sup can wait till the end, as they will + % spawn background work that can go on for a while: seeding system dbs + % from other nodes running resharding jobs + % Children = [ child(mem3_events), child(mem3_nodes), - child(mem3_distribution), - child(mem3_seeds), - % Order important? + child(mem3_shards), child(mem3_sync_nodes), + child(mem3_distribution), child(mem3_sync), - child(mem3_shards), child(mem3_sync_event_listener), + child(mem3_seeds), + child(mem3_db_doc_updater), child(mem3_reshard_sup) ], - {ok, {{one_for_one, 10, 1}, couch_epi:register_service(mem3_epi, Children)}}. + {ok, {{rest_for_one, 10, 1}, couch_epi:register_service(mem3_epi, Children)}}. child(mem3_events) -> MFA = {gen_event, start_link, [{local, mem3_events}]}, diff --git a/src/mem3/src/mem3_sync.erl b/src/mem3/src/mem3_sync.erl index 04e4d18893..67eb771816 100644 --- a/src/mem3/src/mem3_sync.erl +++ b/src/mem3/src/mem3_sync.erl @@ -162,7 +162,7 @@ handle_info({'EXIT', Active, Reason}, State) -> {pending_changes, Count} -> maybe_resubmit(State, Job#job{pid = nil, count = Count}); _ -> - case mem3:db_is_current(Job#job.name) of + case is_job_current(Job, nodes(), mem3:nodes()) of true -> timer:apply_after(5000, ?MODULE, push, [Job#job{pid = nil}]); false -> @@ -390,6 +390,14 @@ maybe_redirect(Node) -> list_to_existing_atom(Redirect) end. +% Check that the db exists and node is either connected or part of the cluster. +% +is_job_current(#job{name = Name, node = Node}, ConnectedNodes, Mem3Nodes) -> + DbCurrent = mem3:db_is_current(Name), + Connected = lists:member(Node, ConnectedNodes), + InMem3Nodes = lists:member(Node, Mem3Nodes), + DbCurrent andalso (Connected orelse InMem3Nodes). + -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). @@ -404,4 +412,37 @@ find_next_node_test() -> ?assertEqual(x, find_next_node(n, [n, x], [n, x])), ?assertEqual(a, find_next_node(n, [a, n, x], [a, n, y])). +is_job_current_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_is_job_current) + ] + }. + +setup() -> + Ctx = test_util:start_couch(), + DbName = ?tempdb(), + {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), + couch_db:close(Db), + {Ctx, DbName}. + +teardown({Ctx, DbName}) -> + ok = couch_server:delete(DbName, [?ADMIN_CTX]), + test_util:stop_couch(Ctx). + +t_is_job_current({_, DbName}) -> + Job = #job{name = DbName, node = n1}, + ?assert(is_job_current(Job, [], [n1])), + ?assert(is_job_current(Job, [n1], [])), + ?assert(is_job_current(Job, [n1], [n1])), + ?assertNot(is_job_current(Job, [n2], [])), + ?assertNot(is_job_current(Job, [], [n2])), + ?assertNot(is_job_current(Job, [], [])), + ?assertNot(is_job_current(Job, [n2], [n2])), + ?assertNot(is_job_current(Job#job{name = <<"x">>}, [n1], [n1])), + ?assertNot(is_job_current(Job#job{name = <<"x">>}, [], [])). + -endif. diff --git a/src/mem3/src/mem3_sync_event_listener.erl b/src/mem3/src/mem3_sync_event_listener.erl index 7f9b2d3b25..2e16fc2b61 100644 --- a/src/mem3/src/mem3_sync_event_listener.erl +++ b/src/mem3/src/mem3_sync_event_listener.erl @@ -240,7 +240,7 @@ teardown_all(_) -> setup() -> {ok, Pid} = ?MODULE:start_link(), - erlang:unlink(Pid), + unlink(Pid), wait_config_subscribed(Pid), Pid. @@ -300,7 +300,7 @@ should_terminate(Pid) -> EventMgr = whereis(config_event), EventMgrWasAlive = (catch is_process_alive(EventMgr)), - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), RestartFun = fun() -> exit(EventMgr, kill) end, {_, _} = test_util:with_process_restart(config_event, RestartFun), diff --git a/src/mem3/src/mem3_sync_security.erl b/src/mem3/src/mem3_sync_security.erl index fc1726901b..f7df4c4017 100644 --- a/src/mem3/src/mem3_sync_security.erl +++ b/src/mem3/src/mem3_sync_security.erl @@ -20,7 +20,7 @@ maybe_sync(#shard{} = Src, #shard{} = Dst) -> case is_local(Src#shard.name) of false -> - erlang:spawn(?MODULE, maybe_sync_int, [Src, Dst]); + spawn(?MODULE, maybe_sync_int, [Src, Dst]); true -> ok end. diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl index 520f54629d..f45ee4063d 100644 --- a/src/mem3/src/mem3_util.erl +++ b/src/mem3/src/mem3_util.erl @@ -29,8 +29,7 @@ ]). -export([get_or_create_db/2, get_or_create_db_int/2]). -export([is_deleted/1, rotate_list/2]). --export([get_shard_opts/1, get_engine_opt/1, get_props_opt/1]). --export([get_shard_props/1, find_dirty_shards/0]). +-export([get_shard_opts/1]). -export([ iso8601_timestamp/0, live_nodes/0, @@ -44,7 +43,8 @@ non_overlapping_shards/1, non_overlapping_shards/3, calculate_max_n/1, - calculate_max_n/3 + calculate_max_n/3, + range_to_hex/1 ]). %% do not use outside mem3. @@ -71,15 +71,7 @@ name_shard(#ordered_shard{dbname = DbName, range = Range} = Shard, Suffix) -> Shard#ordered_shard{name = ?l2b(Name)}. make_name(DbName, [B, E], Suffix) -> - [ - "shards/", - couch_util:to_hex(<>), - "-", - couch_util:to_hex(<>), - "/", - DbName, - Suffix - ]. + ["shards/", ?b2l(range_to_hex([B, E])), "/", DbName, Suffix]. create_partition_map(DbName, N, Q, Nodes) -> create_partition_map(DbName, N, Q, Nodes, ""). @@ -172,6 +164,12 @@ update_db_doc(DbName, #doc{id = Id, body = Body} = Doc, ShouldMutate) -> couch_db:close(Db) end. +-spec range_to_hex([non_neg_integer()]) -> binary(). +range_to_hex([B, E]) when is_integer(B), is_integer(E) -> + HexB = couch_util:to_hex(<>), + HexE = couch_util:to_hex(<>), + ?l2b(HexB ++ "-" ++ HexE). + delete_db_doc(DocId) -> gen_server:cast(mem3_shards, {cache_remove, DocId}), delete_db_doc(mem3_sync:shards_db(), DocId, true). @@ -231,8 +229,7 @@ build_shards_by_node(DbName, DocProps) -> #shard{ dbname = DbName, node = to_atom(Node), - range = [Beg, End], - opts = get_shard_opts(DocProps) + range = [Beg, End] }, Suffix ) @@ -258,8 +255,7 @@ build_shards_by_range(DbName, DocProps) -> dbname = DbName, node = to_atom(Node), range = [Beg, End], - order = Order, - opts = get_shard_opts(DocProps) + order = Order }, Suffix ) @@ -271,14 +267,14 @@ build_shards_by_range(DbName, DocProps) -> ). to_atom(Node) when is_binary(Node) -> - list_to_atom(binary_to_list(Node)); + binary_to_atom(Node); to_atom(Node) when is_atom(Node) -> Node. to_integer(N) when is_integer(N) -> N; to_integer(N) when is_binary(N) -> - list_to_integer(binary_to_list(N)); + binary_to_integer(N); to_integer(N) when is_list(N) -> list_to_integer(N). @@ -469,13 +465,15 @@ range_overlap([A, B], [X, Y]) when -> A =< Y andalso X =< B. -non_overlapping_shards(Shards) -> +non_overlapping_shards([]) -> + []; +non_overlapping_shards([_ | _] = Shards) -> {Start, End} = lists:foldl( fun(Shard, {Min, Max}) -> [B, E] = mem3:range(Shard), {min(B, Min), max(E, Max)} end, - {0, ?RING_END}, + {?RING_END, 0}, Shards ), non_overlapping_shards(Shards, Start, End). @@ -642,50 +640,6 @@ merge_opts(New, Old) -> New ). -get_shard_props(ShardName) -> - case couch_db:open_int(ShardName, []) of - {ok, Db} -> - Props = - case couch_db_engine:get_props(Db) of - undefined -> []; - Else -> Else - end, - %% We don't normally store the default engine name - EngineProps = - case couch_db_engine:get_engine(Db) of - couch_bt_engine -> - []; - EngineName -> - [{engine, EngineName}] - end, - [{props, Props} | EngineProps]; - {not_found, _} -> - not_found; - Else -> - Else - end. - -find_dirty_shards() -> - mem3_shards:fold( - fun(#shard{node = Node, name = Name, opts = Opts} = Shard, Acc) -> - case Opts of - [] -> - Acc; - [{props, []}] -> - Acc; - _ -> - Props = rpc:call(Node, ?MODULE, get_shard_props, [Name]), - case Props =:= Opts of - true -> - Acc; - false -> - [{Shard, Props} | Acc] - end - end - end, - [] - ). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -705,10 +659,11 @@ range_overlap_test_() -> ] ]. -non_overlapping_shards_test() -> +non_overlapping_shards_test_() -> [ ?_assertEqual(Res, non_overlapping_shards(Shards)) || {Shards, Res} <- [ + {[], []}, { [shard(0, ?RING_END)], [shard(0, ?RING_END)] @@ -719,7 +674,7 @@ non_overlapping_shards_test() -> }, { [shard(0, 1), shard(0, 1)], - [shard(0, 1)] + [shard(0, 1), shard(0, 1)] }, { [shard(0, 1), shard(3, 4)], @@ -731,15 +686,15 @@ non_overlapping_shards_test() -> }, { [shard(1, 2), shard(0, 1)], - [shard(0, 1), shard(1, 2)] + [] }, { [shard(0, 1), shard(0, 2), shard(2, 5), shard(3, 5)], - [shard(0, 2), shard(2, 5)] + [shard(0, 2), shard(3, 5)] }, { - [shard(0, 2), shard(4, 5), shard(1, 3)], - [] + [shard(1, 2), shard(3, 4), shard(1, 4), shard(5, 6)], + [shard(1, 4), shard(5, 6)] } ] ]. @@ -777,4 +732,8 @@ calculate_max_n_custom_range_test_() -> shard(Begin, End) -> #shard{range = [Begin, End]}. +range_to_hex_test() -> + Range = [2147483648, 4294967295], + ?assertEqual(<<"80000000-ffffffff">>, range_to_hex(Range)). + -endif. diff --git a/src/mem3/test/eunit/mem3_distribution_test.erl b/src/mem3/test/eunit/mem3_distribution_test.erl index 4bccc872b7..b04ff55679 100644 --- a/src/mem3/test/eunit/mem3_distribution_test.erl +++ b/src/mem3/test/eunit/mem3_distribution_test.erl @@ -136,11 +136,12 @@ ping_nodes_test(_) -> {n1, {nodedown, n1}}, {n2, {nodedown, n2}} ], - couch_debug:ping_nodes() + couch_debug:ping_live_cluster_nodes() ), ?assertEqual({nodedown, n3}, couch_debug:ping(n3, 100)). dead_nodes_test(_) -> meck:expect(mem3, nodes, 0, [n1, n2, n3]), meck:expect(mem3_util, live_nodes, 0, [n1, n2]), - ?assertEqual([n3], couch_debug:dead_nodes()). + Node = node(), + ?assertEqual([{Node, [n3]}], couch_debug:dead_nodes()). diff --git a/src/mem3/test/eunit/mem3_rep_test.erl b/src/mem3/test/eunit/mem3_rep_test.erl index 470fb208d1..814fd11b22 100644 --- a/src/mem3/test/eunit/mem3_rep_test.erl +++ b/src/mem3/test/eunit/mem3_rep_test.erl @@ -209,7 +209,7 @@ get_all_docs(DbName) -> get_all_docs(DbName, #mrargs{}). get_all_docs(DbName, #mrargs{} = QArgs0) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc( fun() -> Cb = fun @@ -235,11 +235,11 @@ to_map({[_ | _]} = EJson) -> jiffy:decode(jiffy:encode(EJson), [return_maps]). create_db(DbName, Opts) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc(fun() -> fabric:create_db(DbName, Opts) end, GL). delete_db(DbName) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc(fun() -> fabric:delete_db(DbName, [?ADMIN_CTX]) end, GL). create_local_db(DbName) -> @@ -259,7 +259,7 @@ with_proc(Fun, GroupLeader, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> case GroupLeader of undefined -> ok; - _ -> erlang:group_leader(GroupLeader, self()) + _ -> group_leader(GroupLeader, self()) end, exit({with_proc_res, Fun()}) end), @@ -269,7 +269,7 @@ with_proc(Fun, GroupLeader, Timeout) -> {'DOWN', Ref, process, Pid, Error} -> error(Error) after Timeout -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), exit(Pid, kill), error({with_proc_timeout, Fun, Timeout}) end. diff --git a/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl b/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl index 1f6f89f8a8..140b376358 100644 --- a/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl @@ -232,7 +232,7 @@ continuous_feed_should_work_during_split(#{db1 := Db}) -> {'DOWN', UpdaterRef, process, UpdaterPid, normal} -> ok; {'DOWN', UpdaterRef, process, UpdaterPid, Error} -> - erlang:error( + error( {test_context_failed, [ {module, ?MODULE}, {line, ?LINE}, @@ -295,11 +295,11 @@ changes_callback({stop, EndSeq, _Pending}, Acc) -> %% common helpers from here create_db(DbName, Opts) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc(fun() -> fabric:create_db(DbName, Opts) end, GL). delete_db(DbName) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc(fun() -> fabric:delete_db(DbName, [?ADMIN_CTX]) end, GL). with_proc(Fun) -> @@ -312,7 +312,7 @@ with_proc(Fun, GroupLeader, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> case GroupLeader of undefined -> ok; - _ -> erlang:group_leader(GroupLeader, self()) + _ -> group_leader(GroupLeader, self()) end, exit({with_proc_res, Fun()}) end), @@ -322,7 +322,7 @@ with_proc(Fun, GroupLeader, Timeout) -> {'DOWN', Ref, process, Pid, Error} -> error(Error) after Timeout -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), exit(Pid, kill), error({with_proc_timeout, Fun, Timeout}) end. diff --git a/src/mem3/test/eunit/mem3_reshard_test.erl b/src/mem3/test/eunit/mem3_reshard_test.erl index 2579649f97..d6c3877382 100644 --- a/src/mem3/test/eunit/mem3_reshard_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_test.erl @@ -22,17 +22,7 @@ -define(TIMEOUT, 60). setup() -> - HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}, - case HaveDreyfus of - false -> ok; - true -> mock_dreyfus_indices() - end, - - HaveHastings = code:lib_dir(hastings) /= {error, bad_name}, - case HaveHastings of - false -> ok; - true -> mock_hastings_indices() - end, + mock_dreyfus_indices(), {Db1, Db2} = {?tempdb(), ?tempdb()}, create_db(Db1, [{q, 1}, {n, 1}]), PartProps = [{partitioned, true}, {hash, [couch_partition, hash, []]}], @@ -276,42 +266,24 @@ update_docs_before_topoff1(#{db1 := Db}) -> indices_are_built(#{db1 := Db}) -> {timeout, ?TIMEOUT, ?_test(begin - HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}, - HaveHastings = code:lib_dir(hastings) /= {error, bad_name}, - - add_test_docs(Db, #{docs => 10, mrview => 2, search => 2, geo => 2}), + add_test_docs(Db, #{docs => 10, mrview => 2, search => 2}), [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), {ok, JobId} = mem3_reshard:start_split_job(Shard), wait_state(JobId, completed), Shards1 = lists:sort(mem3:local_shards(Db)), ?assertEqual(2, length(Shards1)), MRViewGroupInfo = get_group_info(Db, <<"_design/mrview00000">>), - ?assertMatch(#{<<"update_seq">> := 32}, MRViewGroupInfo), - - HaveDreyfus = code:lib_dir(dreyfus) /= {error, bad_name}, - case HaveDreyfus of - false -> - ok; - true -> - % 4 because there are 2 indices and 2 target shards - ?assertEqual(4, meck:num_calls(dreyfus_index, await, 2)) - end, + ?assertMatch(#{<<"update_seq">> := 28}, MRViewGroupInfo), - HaveHastings = code:lib_dir(hastings) /= {error, bad_name}, - case HaveHastings of - false -> - ok; - true -> - % 4 because there are 2 indices and 2 target shards - ?assertEqual(4, meck:num_calls(hastings_index, await, 2)) - end + % 4 because there are 2 indices and 2 target shards + ?assertEqual(4, meck:num_calls(dreyfus_index, await, 2)) end)}. % This test that indices are built despite intermittent errors. indices_can_be_built_with_errors(#{db1 := Db}) -> {timeout, ?TIMEOUT, ?_test(begin - add_test_docs(Db, #{docs => 10, mrview => 2, search => 2, geo => 2}), + add_test_docs(Db, #{docs => 10, mrview => 2, search => 2}), [#shard{name = Shard}] = lists:sort(mem3:local_shards(Db)), meck:expect( couch_index_server, @@ -343,11 +315,11 @@ indices_can_be_built_with_errors(#{db1 := Db}) -> Shards1 = lists:sort(mem3:local_shards(Db)), ?assertEqual(2, length(Shards1)), MRViewGroupInfo = get_group_info(Db, <<"_design/mrview00000">>), - ?assertMatch(#{<<"update_seq">> := 32}, MRViewGroupInfo) + ?assertMatch(#{<<"update_seq">> := 28}, MRViewGroupInfo) end)}. mock_dreyfus_indices() -> - meck:expect(dreyfus_index, design_doc_to_indexes, fun(Doc) -> + meck:expect(dreyfus_index, design_doc_to_indexes, fun(_, Doc) -> #doc{body = {BodyProps}} = Doc, case couch_util:get_value(<<"indexes">>, BodyProps) of undefined -> @@ -359,19 +331,6 @@ mock_dreyfus_indices() -> meck:expect(dreyfus_index_manager, get_index, fun(_, _) -> {ok, pid} end), meck:expect(dreyfus_index, await, fun(_, _) -> {ok, indexpid, someseq} end). -mock_hastings_indices() -> - meck:expect(hastings_index, design_doc_to_indexes, fun(Doc) -> - #doc{body = {BodyProps}} = Doc, - case couch_util:get_value(<<"st_indexes">>, BodyProps) of - undefined -> - []; - {[_]} -> - [{hastings, <<"db">>, hastings_index1}] - end - end), - meck:expect(hastings_index_manager, get_index, fun(_, _) -> {ok, pid} end), - meck:expect(hastings_index, await, fun(_, _) -> {ok, someseq} end). - % Split partitioned database split_partitioned_db(#{db2 := Db}) -> {timeout, ?TIMEOUT, @@ -824,7 +783,7 @@ get_all_docs(DbName) -> get_all_docs(DbName, #mrargs{}). get_all_docs(DbName, #mrargs{} = QArgs0) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc( fun() -> Cb = fun @@ -886,11 +845,11 @@ to_map({[_ | _]} = EJson) -> jiffy:decode(jiffy:encode(EJson), [return_maps]). create_db(DbName, Opts) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc(fun() -> fabric:create_db(DbName, Opts) end, GL). delete_db(DbName) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc(fun() -> fabric:delete_db(DbName, [?ADMIN_CTX]) end, GL). with_proc(Fun) -> @@ -903,7 +862,7 @@ with_proc(Fun, GroupLeader, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> case GroupLeader of undefined -> ok; - _ -> erlang:group_leader(GroupLeader, self()) + _ -> group_leader(GroupLeader, self()) end, exit({with_proc_res, Fun()}) end), @@ -913,7 +872,7 @@ with_proc(Fun, GroupLeader, Timeout) -> {'DOWN', Ref, process, Pid, Error} -> error(Error) after Timeout -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), exit(Pid, kill), error({with_proc_timeout, Fun, Timeout}) end. @@ -924,7 +883,6 @@ add_test_docs(DbName, #{} = DocSpec) -> pdocs(maps:get(pdocs, DocSpec, #{})) ++ ddocs(mrview, maps:get(mrview, DocSpec, [])) ++ ddocs(search, maps:get(search, DocSpec, [])) ++ - ddocs(geo, maps:get(geo, DocSpec, [])) ++ ldocs(maps:get(local, DocSpec, [])), Res = update_docs(DbName, Docs), Docs1 = lists:map( @@ -1021,17 +979,6 @@ ddprop(mrview) -> ]}} ]}} ]; -ddprop(geo) -> - [ - {<<"st_indexes">>, - {[ - {<<"area">>, - {[ - {<<"analyzer">>, <<"standard">>}, - {<<"index">>, <<"function(d){if(d.g){st_index(d.g)}}">>} - ]}} - ]}} - ]; ddprop(search) -> [ {<<"indexes">>, diff --git a/src/mem3/test/eunit/mem3_shards_test.erl b/src/mem3/test/eunit/mem3_shards_test.erl index 6d2766fa22..14f4bc0846 100644 --- a/src/mem3/test/eunit/mem3_shards_test.erl +++ b/src/mem3/test/eunit/mem3_shards_test.erl @@ -49,7 +49,8 @@ mem3_shards_db_create_props_test_() -> fun setup/0, fun teardown/1, [ - fun partitioned_shards_recreated_properly/1 + ?TDEF_FE(partitioned_shards_recreated_properly, ?TIMEOUT), + ?TDEF_FE(update_props, ?TIMEOUT) ] } } @@ -61,33 +62,45 @@ mem3_shards_db_create_props_test_() -> % properties. % SEE: apache/couchdb#3631 partitioned_shards_recreated_properly(#{dbname := DbName, dbdoc := DbDoc}) -> - {timeout, ?TIMEOUT, - ?_test(begin - #doc{body = {Body0}} = DbDoc, - Body1 = [{<<"foo">>, <<"bar">>} | Body0], - Shards = [Shard | _] = lists:sort(mem3:shards(DbName)), - ShardName = Shard#shard.name, - ?assert(is_partitioned(Shards)), - ok = with_proc(fun() -> couch_server:delete(ShardName, []) end), - ?assertThrow({not_found, no_db_file}, is_partitioned(Shard)), - ok = mem3_util:update_db_doc(DbDoc#doc{body = {Body1}}), - Shards = - [Shard | _] = test_util:wait_value( - fun() -> - lists:sort(mem3:shards(DbName)) - end, - Shards - ), - ?assertEqual( - true, - test_util:wait_value( - fun() -> - catch is_partitioned(Shard) - end, - true - ) - ) - end)}. + #doc{body = {Body0}} = DbDoc, + Body1 = [{<<"foo">>, <<"bar">>} | Body0], + Shards = [Shard | _] = lists:sort(mem3:shards(DbName)), + ShardName = Shard#shard.name, + ?assert(is_partitioned(Shards)), + ok = with_proc(fun() -> couch_server:delete(ShardName, []) end), + ?assertThrow({not_found, no_db_file}, is_partitioned(Shard)), + ok = mem3_util:update_db_doc(DbDoc#doc{body = {Body1}}), + Shards = + [Shard | _] = test_util:wait_value( + fun() -> + lists:sort(mem3:shards(DbName)) + end, + Shards + ), + ?assertEqual( + true, + test_util:wait_value( + fun() -> + catch is_partitioned(Shard) + end, + true + ) + ). + +update_props(#{dbname := DbName, dbdoc := DbDoc}) -> + {ok, Doc} = mem3:get_db_doc(DbName), + ?assertEqual(DbDoc, Doc), + #doc{body = {Body0}} = Doc, + {Props} = couch_util:get_value(<<"props">>, Body0, {[]}), + Props1 = couch_util:set_value(<<"baz">>, Props, <<"bar">>), + Body1 = couch_util:set_value(<<"props">>, Body0, {Props1}), + ResUpdate = mem3:update_db_doc(Doc#doc{body = {Body1}}), + ?assertMatch({ok, _}, ResUpdate), + {ok, Doc2} = mem3:get_db_doc(DbName), + #doc{body = {Body2}} = Doc2, + {Props2} = couch_util:get_value(<<"props">>, Body2, {[]}), + ?assertEqual(<<"bar">>, couch_util:get_value(<<"baz">>, Props2)), + ?assertEqual({error, conflict}, mem3:update_db_doc(Doc#doc{body = {Body1}})). is_partitioned([#shard{} | _] = Shards) -> lists:all(fun is_partitioned/1, Shards); @@ -97,11 +110,11 @@ is_partitioned(Db) -> couch_db:is_partitioned(Db). create_db(DbName, Opts) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc(fun() -> fabric:create_db(DbName, Opts) end, GL). delete_db(DbName) -> - GL = erlang:group_leader(), + GL = group_leader(), with_proc(fun() -> fabric:delete_db(DbName, [?ADMIN_CTX]) end, GL). with_proc(Fun) -> @@ -114,7 +127,7 @@ with_proc(Fun, GroupLeader, Timeout) -> {Pid, Ref} = spawn_monitor(fun() -> case GroupLeader of undefined -> ok; - _ -> erlang:group_leader(GroupLeader, self()) + _ -> group_leader(GroupLeader, self()) end, exit({with_proc_res, Fun()}) end), @@ -124,7 +137,7 @@ with_proc(Fun, GroupLeader, Timeout) -> {'DOWN', Ref, process, Pid, Error} -> error(Error) after Timeout -> - erlang:demonitor(Ref, [flush]), + demonitor(Ref, [flush]), exit(Pid, kill), error({with_proc_timeout, Fun, Timeout}) end. diff --git a/src/mem3/test/eunit/mem3_zone_test.erl b/src/mem3/test/eunit/mem3_zone_test.erl new file mode 100644 index 0000000000..55fef3342f --- /dev/null +++ b/src/mem3/test/eunit/mem3_zone_test.erl @@ -0,0 +1,77 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(mem3_zone_test). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +mem3_zone_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_empty_zone), + ?TDEF_FE(t_set_zone_from_env), + ?TDEF_FE(t_set_zone_when_node_in_seedlist), + ?TDEF_FE(t_zone_already_set) + ] + }. + +assertZoneEqual(Expected) -> + [Node | _] = mem3:nodes(), + Actual = mem3:node_info(Node, <<"zone">>), + ?assertEqual(Expected, Actual). + +t_empty_zone(_) -> + ok = application:start(mem3), + assertZoneEqual(undefined). + +t_set_zone_from_env(_) -> + Zone = "zone1", + os:putenv("COUCHDB_ZONE", Zone), + ok = application:start(mem3), + assertZoneEqual(iolist_to_binary(Zone)). + +t_set_zone_when_node_in_seedlist(_) -> + CfgSeeds = "nonode@nohost", + config:set("cluster", "seedlist", CfgSeeds, false), + Zone = "zone1", + os:putenv("COUCHDB_ZONE", Zone), + ok = application:start(mem3), + assertZoneEqual(iolist_to_binary(Zone)). + +t_zone_already_set(_) -> + Zone = "zone1", + os:putenv("COUCHDB_ZONE", Zone), + ok = application:start(mem3), + application:stop(mem3), + ok = application:start(mem3), + assertZoneEqual(iolist_to_binary(Zone)). + +setup() -> + meck:new(mem3_seeds, [passthrough]), + meck:new(mem3_rpc, [passthrough]), + test_util:start_couch([rexi]). + +teardown(Ctx) -> + catch application:stop(mem3), + os:unsetenv("COUCHDB_ZONE"), + Filename = config:get("mem3", "nodes_db", "_nodes") ++ ".couch", + file:delete(filename:join([?BUILDDIR(), "tmp", "data", Filename])), + case config:get("couch_httpd_auth", "authentication_db") of + undefined -> ok; + DbName -> couch_server:delete(list_to_binary(DbName), []) + end, + meck:unload(), + test_util:stop_couch(Ctx). diff --git a/src/nouveau/src/nouveau.app.src b/src/nouveau/src/nouveau.app.src index 0828437c18..e8ea54915c 100644 --- a/src/nouveau/src/nouveau.app.src +++ b/src/nouveau/src/nouveau.app.src @@ -18,7 +18,7 @@ {vsn, git}, {applications, [ config, - ibrowse, + gun, kernel, stdlib, mem3, diff --git a/src/nouveau/src/nouveau_api.erl b/src/nouveau/src/nouveau_api.erl index b700524f75..cfc88af4f7 100644 --- a/src/nouveau/src/nouveau_api.erl +++ b/src/nouveau/src/nouveau_api.erl @@ -23,13 +23,12 @@ create_index/2, delete_path/1, delete_path/2, - delete_doc_async/5, - purge_doc/5, - update_doc_async/7, + delete_doc/4, + purge_doc/4, + update_doc/6, search/2, - set_purge_seq/4, - set_update_seq/4, - drain_async_responses/2, + set_purge_seq/3, + set_update_seq/3, jaxrs_error/2 ]). @@ -40,13 +39,13 @@ analyze(Text, Analyzer) when -> ReqBody = {[{<<"text">>, Text}, {<<"analyzer">>, Analyzer}]}, Resp = send_if_enabled( - nouveau_util:nouveau_url() ++ "/analyze", + "/analyze", [?JSON_CONTENT_TYPE], - post, + <<"POST">>, jiffy:encode(ReqBody) ), case Resp of - {ok, "200", _, RespBody} -> + {ok, 200, _, RespBody} -> Json = jiffy:decode(RespBody, [return_maps]), {ok, maps:get(<<"tokens">>, Json)}; {ok, StatusCode, _, RespBody} -> @@ -58,9 +57,9 @@ analyze(_, _) -> {error, {bad_request, <<"'text' and 'analyzer' fields must be non-empty strings">>}}. index_info(#index{} = Index) -> - Resp = send_if_enabled(index_url(Index), [], get), + Resp = send_if_enabled(index_path(Index), [], <<"GET">>), case Resp of - {ok, "200", _, RespBody} -> + {ok, 200, _, RespBody} -> {ok, jiffy:decode(RespBody, [return_maps])}; {ok, StatusCode, _, RespBody} -> {error, jaxrs_error(StatusCode, RespBody)}; @@ -70,10 +69,10 @@ index_info(#index{} = Index) -> create_index(#index{} = Index, IndexDefinition) -> Resp = send_if_enabled( - index_url(Index), [?JSON_CONTENT_TYPE], put, jiffy:encode(IndexDefinition) + index_path(Index), [?JSON_CONTENT_TYPE], <<"PUT">>, jiffy:encode(IndexDefinition) ), case Resp of - {ok, "204", _, _} -> + {ok, 200, _, _} -> ok; {ok, StatusCode, _, RespBody} -> {error, jaxrs_error(StatusCode, RespBody)}; @@ -88,10 +87,10 @@ delete_path(Path, Exclusions) when is_binary(Path), is_list(Exclusions) -> Resp = send_if_enabled( - index_path(Path), [?JSON_CONTENT_TYPE], delete, jiffy:encode(Exclusions) + index_path(Path), [?JSON_CONTENT_TYPE], <<"DELETE">>, jiffy:encode(Exclusions) ), case Resp of - {ok, "204", _, _} -> + {ok, 200, _, _} -> ok; {ok, StatusCode, _, RespBody} -> {error, jaxrs_error(StatusCode, RespBody)}; @@ -99,8 +98,7 @@ delete_path(Path, Exclusions) when send_error(Reason) end. -delete_doc_async(ConnPid, #index{} = Index, DocId, MatchSeq, UpdateSeq) when - is_pid(ConnPid), +delete_doc(#index{} = Index, DocId, MatchSeq, UpdateSeq) when is_binary(DocId), is_integer(MatchSeq), MatchSeq >= 0, @@ -108,19 +106,22 @@ delete_doc_async(ConnPid, #index{} = Index, DocId, MatchSeq, UpdateSeq) when UpdateSeq > 0 -> ReqBody = #{match_seq => MatchSeq, seq => UpdateSeq, purge => false}, - send_direct_if_enabled( - ConnPid, - doc_url(Index, DocId), + Resp = send_if_enabled( + doc_path(Index, DocId), [?JSON_CONTENT_TYPE], - delete, - jiffy:encode(ReqBody), - [ - {stream_to, self()} - ] - ). + <<"DELETE">>, + jiffy:encode(ReqBody) + ), + case Resp of + {ok, 200, _, _} -> + ok; + {ok, StatusCode, _, RespBody} -> + {error, jaxrs_error(StatusCode, RespBody)}; + {error, Reason} -> + send_error(Reason) + end. -purge_doc(ConnPid, #index{} = Index, DocId, MatchSeq, PurgeSeq) when - is_pid(ConnPid), +purge_doc(#index{} = Index, DocId, MatchSeq, PurgeSeq) when is_binary(DocId), is_integer(MatchSeq), MatchSeq >= 0, @@ -128,11 +129,11 @@ purge_doc(ConnPid, #index{} = Index, DocId, MatchSeq, PurgeSeq) when PurgeSeq > 0 -> ReqBody = #{match_seq => MatchSeq, seq => PurgeSeq, purge => true}, - Resp = send_direct_if_enabled( - ConnPid, doc_url(Index, DocId), [?JSON_CONTENT_TYPE], delete, jiffy:encode(ReqBody), [] + Resp = send_if_enabled( + doc_path(Index, DocId), [?JSON_CONTENT_TYPE], <<"DELETE">>, jiffy:encode(ReqBody) ), case Resp of - {ok, "204", _, _} -> + {ok, 200, _, _} -> ok; {ok, StatusCode, _, RespBody} -> {error, jaxrs_error(StatusCode, RespBody)}; @@ -140,8 +141,7 @@ purge_doc(ConnPid, #index{} = Index, DocId, MatchSeq, PurgeSeq) when send_error(Reason) end. -update_doc_async(ConnPid, #index{} = Index, DocId, MatchSeq, UpdateSeq, Partition, Fields) when - is_pid(ConnPid), +update_doc(#index{} = Index, DocId, MatchSeq, UpdateSeq, Partition, Fields) when is_binary(DocId), is_integer(MatchSeq), MatchSeq >= 0, @@ -156,25 +156,29 @@ update_doc_async(ConnPid, #index{} = Index, DocId, MatchSeq, UpdateSeq, Partitio partition => Partition, fields => Fields }, - send_direct_if_enabled( - ConnPid, - doc_url(Index, DocId), + Resp = send_if_enabled( + doc_path(Index, DocId), [?JSON_CONTENT_TYPE], - put, - jiffy:encode(ReqBody), - [ - {stream_to, self()} - ] - ). + <<"PUT">>, + jiffy:encode(ReqBody) + ), + case Resp of + {ok, 200, _, _} -> + ok; + {ok, StatusCode, _, RespBody} -> + {error, jaxrs_error(StatusCode, RespBody)}; + {error, Reason} -> + send_error(Reason) + end. search(#index{} = Index, QueryArgs) -> Resp = send_if_enabled( - search_url(Index), [?JSON_CONTENT_TYPE], post, jiffy:encode(QueryArgs) + search_path(Index), [?JSON_CONTENT_TYPE], <<"POST">>, jiffy:encode(QueryArgs) ), case Resp of - {ok, "200", _, RespBody} -> + {ok, 200, _, RespBody} -> {ok, jiffy:decode(RespBody, [return_maps])}; - {ok, "409", _, _} -> + {ok, 409, _, _} -> %% Index was not current enough. {error, stale_index}; {ok, StatusCode, _, RespBody} -> @@ -183,26 +187,26 @@ search(#index{} = Index, QueryArgs) -> send_error(Reason) end. -set_update_seq(ConnPid, #index{} = Index, MatchSeq, UpdateSeq) -> +set_update_seq(#index{} = Index, MatchSeq, UpdateSeq) -> ReqBody = #{ match_update_seq => MatchSeq, update_seq => UpdateSeq }, - set_seq(ConnPid, Index, ReqBody). + set_seq(Index, ReqBody). -set_purge_seq(ConnPid, #index{} = Index, MatchSeq, PurgeSeq) -> +set_purge_seq(#index{} = Index, MatchSeq, PurgeSeq) -> ReqBody = #{ match_purge_seq => MatchSeq, purge_seq => PurgeSeq }, - set_seq(ConnPid, Index, ReqBody). + set_seq(Index, ReqBody). -set_seq(ConnPid, #index{} = Index, ReqBody) -> - Resp = send_direct_if_enabled( - ConnPid, index_url(Index), [?JSON_CONTENT_TYPE], post, jiffy:encode(ReqBody), [] +set_seq(#index{} = Index, ReqBody) -> + Resp = send_if_enabled( + index_path(Index), [?JSON_CONTENT_TYPE], <<"POST">>, jiffy:encode(ReqBody) ), case Resp of - {ok, "204", _, _} -> + {ok, 200, _, _} -> ok; {ok, StatusCode, _, RespBody} -> {error, jaxrs_error(StatusCode, RespBody)}; @@ -210,90 +214,37 @@ set_seq(ConnPid, #index{} = Index, ReqBody) -> send_error(Reason) end. -%% wait for enough async responses to reduce the Queue to Min length. -drain_async_responses(Queue0, Min) when Min >= 0 -> - case queue:len(Queue0) > Min of - true -> - {{value, ReqId}, Queue1} = queue:out(Queue0), - wait_for_response(ReqId), - drain_async_responses(Queue1, Min); - false -> - Queue0 - end. - -wait_for_response(ReqId) -> - case drain_async_response(ReqId) of - {ok, "204", _Headers, _Body} -> - ok; - {ok, StatusCode, _Headers, RespBody} -> - exit({error, jaxrs_error(StatusCode, RespBody)}) - end. - -drain_async_response(ReqId) -> - drain_async_response(ReqId, undefined, undefined, undefined). - -drain_async_response(ReqId, Code0, Headers0, Body0) -> - receive - {ibrowse_async_headers, ReqId, Code1, Headers1} -> - drain_async_response(ReqId, Code1, Headers1, Body0); - {ibrowse_async_response, ReqId, Body1} -> - drain_async_response(ReqId, Code0, Headers0, Body1); - {ibrowse_async_response_end, ReqId} -> - {ok, Code0, Headers0, Body0} - end. - %% private functions -index_path(Path) -> - lists:flatten( - io_lib:format( - "~s/index/~s", - [ - nouveau_util:nouveau_url(), - couch_util:url_encode(Path) - ] - ) - ). +index_path(Path) when is_binary(Path) -> + [<<"/index/">>, couch_util:url_encode(Path)]; +index_path(#index{} = Index) -> + [<<"/index/">>, couch_util:url_encode(nouveau_util:index_name(Index))]. -index_url(#index{} = Index) -> - lists:flatten( - io_lib:format( - "~s/index/~s", - [ - nouveau_util:nouveau_url(), - couch_util:url_encode(nouveau_util:index_name(Index)) - ] - ) - ). +doc_path(#index{} = Index, DocId) -> + [ + <<"/index/">>, + couch_util:url_encode(nouveau_util:index_name(Index)), + <<"/doc/">>, + couch_util:url_encode(DocId) + ]. -doc_url(#index{} = Index, DocId) -> - lists:flatten( - io_lib:format( - "~s/index/~s/doc/~s", - [ - nouveau_util:nouveau_url(), - couch_util:url_encode(nouveau_util:index_name(Index)), - couch_util:url_encode(DocId) - ] - ) - ). +search_path(#index{} = Index) -> + [index_path(Index), <<"/search">>]. -search_url(IndexName) -> - index_url(IndexName) ++ "/search". - -jaxrs_error("400", Body) -> +jaxrs_error(400, Body) -> {bad_request, message(Body)}; -jaxrs_error("404", Body) -> +jaxrs_error(404, Body) -> {not_found, message(Body)}; -jaxrs_error("405", Body) -> +jaxrs_error(405, Body) -> {method_not_allowed, message(Body)}; -jaxrs_error("409", Body) -> +jaxrs_error(409, Body) -> {conflict, message(Body)}; -jaxrs_error("417", Body) -> +jaxrs_error(417, Body) -> {expectation_failed, message(Body)}; -jaxrs_error("422", Body) -> +jaxrs_error(422, Body) -> {bad_request, lists:join(" and ", errors(Body))}; -jaxrs_error("500", Body) -> +jaxrs_error(500, Body) -> {internal_server_error, message(Body)}. send_error({conn_failed, _}) -> @@ -309,71 +260,44 @@ errors(Body) -> Json = jiffy:decode(Body, [return_maps]), maps:get(<<"errors">>, Json). -send_if_enabled(Url, Header, Method) -> - send_if_enabled(Url, Header, Method, []). +send_if_enabled(Path, ReqHeaders, Method) -> + send_if_enabled(Path, ReqHeaders, Method, <<>>). -send_if_enabled(Url, Header, Method, Body) -> - send_if_enabled(Url, Header, Method, Body, []). +send_if_enabled(Path, ReqHeaders, Method, ReqBody) -> + send_if_enabled(Path, ReqHeaders, Method, ReqBody, 5). -send_if_enabled(Url, Header, Method, Body, Options0) -> +send_if_enabled(Path, ReqHeaders, Method, ReqBody, RemainingTries) -> case nouveau:enabled() of true -> - Options1 = ibrowse_options(Options0), - retry_if_connection_closes(fun() -> - ibrowse:send_req(Url, Header, Method, Body, Options1) - end); + case + gun_pool:request( + Method, + Path, + [nouveau_gun:host_header() | ReqHeaders], + ReqBody + ) + of + {async, PoolStreamRef} -> + Timeout = config:get_integer("nouveau", "request_timeout", 30000), + case gun_pool:await(PoolStreamRef, Timeout) of + {response, fin, Status, RespHeaders} -> + {ok, Status, RespHeaders, []}; + {response, nofin, Status, RespHeaders} -> + case gun_pool:await_body(PoolStreamRef, Timeout) of + {ok, RespBody} -> + {ok, Status, RespHeaders, RespBody}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end; + {error, no_connection_available, _Reason} when RemainingTries > 0 -> + timer:sleep(1000), + send_if_enabled(Path, ReqHeaders, Method, ReqBody, RemainingTries - 1); + {error, _Type, Reason} -> + {error, Reason} + end; false -> {error, nouveau_not_enabled} end. - -send_direct_if_enabled(ConnPid, Url, Header, Method, Body, Options0) -> - case nouveau:enabled() of - true -> - Options1 = ibrowse_options(Options0), - retry_if_connection_closes(fun() -> - ibrowse:send_req_direct(ConnPid, Url, Header, Method, Body, Options1) - end); - false -> - {error, nouveau_not_enabled} - end. - -retry_if_connection_closes(Fun) -> - MaxRetries = max(1, config:get_integer("nouveau", "max_retries", 5)), - retry_if_connection_closes(Fun, MaxRetries). - -retry_if_connection_closes(_Fun, 0) -> - {error, connection_closed}; -retry_if_connection_closes(Fun, N) when is_integer(N), N > 0 -> - case Fun() of - {error, connection_closed} -> - couch_stats:increment_counter([nouveau, connection_closed_errors]), - timer:sleep(1000), - retry_if_connection_closes(Fun, N - 1); - Else -> - Else - end. - -ibrowse_options(BaseOptions) when is_list(BaseOptions) -> - CACertFile = config:get("nouveau", "ssl_cacert_file"), - KeyFile = config:get("nouveau", "ssl_key_file"), - CertFile = config:get("nouveau", "ssl_cert_file"), - Password = config:get("nouveau", "ssl_password"), - if - KeyFile /= undefined andalso CertFile /= undefined -> - CertKeyConf0 = #{ - certfile => CertFile, - keyfile => KeyFile, - password => Password, - cacertfile => CACertFile - }, - CertKeyConf1 = maps:filter(fun remove_undefined/2, CertKeyConf0), - SSLOptions = [{certs_keys, [CertKeyConf1]}], - [{ssl_options, SSLOptions} | BaseOptions]; - true -> - BaseOptions - end. - -remove_undefined(_Key, undefined) -> - false; -remove_undefined(_Key, _Value) -> - true. diff --git a/src/nouveau/src/nouveau_bookmark.erl b/src/nouveau/src/nouveau_bookmark.erl index 08aeb0f735..3b0878f726 100644 --- a/src/nouveau/src/nouveau_bookmark.erl +++ b/src/nouveau/src/nouveau_bookmark.erl @@ -51,7 +51,7 @@ unpack(DbName, PackedBookmark) when is_list(PackedBookmark) -> unpack(DbName, list_to_binary(PackedBookmark)); unpack(DbName, PackedBookmark) when is_binary(PackedBookmark) -> Bookmark = jiffy:decode(b64url:decode(PackedBookmark), [return_maps]), - maps:from_list([{range_of(DbName, V), V} || V <- Bookmark]). + #{range_of(DbName, V) => V || V <- Bookmark}. pack(nil) -> null; diff --git a/src/nouveau/src/nouveau_fabric_cleanup.erl b/src/nouveau/src/nouveau_fabric_cleanup.erl index cd4128fb1d..75c2190b8d 100644 --- a/src/nouveau/src/nouveau_fabric_cleanup.erl +++ b/src/nouveau/src/nouveau_fabric_cleanup.erl @@ -14,30 +14,52 @@ -module(nouveau_fabric_cleanup). --include_lib("couch/include/couch_db.hrl"). - --include("nouveau.hrl"). --include_lib("mem3/include/mem3.hrl"). - --export([go/1]). +-export([go/1, go_local/3]). go(DbName) -> - {ok, DesignDocs} = fabric:design_docs(DbName), - ActiveSigs = - lists:usort( - lists:flatmap( - fun(Doc) -> active_sigs(DbName, Doc) end, - [couch_doc:from_json_obj(DD) || DD <- DesignDocs] - ) - ), - Shards = mem3:shards(DbName), - lists:foreach( - fun(Shard) -> - rexi:cast(Shard#shard.node, {nouveau_rpc, cleanup, [Shard#shard.name, ActiveSigs]}) - end, - Shards - ). + case fabric_util:get_design_doc_records(DbName) of + {ok, DDocs} when is_list(DDocs) -> + Sigs = nouveau_util:get_signatures_from_ddocs(DbName, DDocs), + Shards = mem3:shards(DbName), + ByNode = maps:groups_from_list(fun mem3:node/1, fun mem3:name/1, Shards), + Fun = fun(Node, Dbs, Acc) -> + erpc:send_request(Node, ?MODULE, go_local, [DbName, Dbs, Sigs], Node, Acc) + end, + Reqs = maps:fold(Fun, erpc:reqids_new(), ByNode), + recv(DbName, Reqs, fabric_util:abs_request_timeout()); + Error -> + couch_log:error("~p : error fetching ddocs db:~p ~p", [?MODULE, DbName, Error]), + Error + end. + +% erpc endpoint for go/1 and fabric_index_cleanup:cleanup_indexes/2 +% +go_local(DbName, Dbs, Sigs) -> + try + lists:foreach( + fun(Db) -> + Sz = byte_size(DbName), + <<"shards/", Range:17/binary, "/", DbName:Sz/binary, ".", _/binary>> = Db, + Checkpoints = nouveau_util:get_purge_checkpoints(Db), + ok = couch_index_util:cleanup_purges(Db, Sigs, Checkpoints), + Path = <<"shards/", Range/binary, "/", DbName/binary, ".*/*">>, + nouveau_api:delete_path(nouveau_util:index_name(Path), maps:keys(Sigs)) + end, + Dbs + ) + catch + error:database_does_not_exist -> + ok + end. -active_sigs(DbName, #doc{} = Doc) -> - Indexes = nouveau_util:design_doc_to_indexes(DbName, Doc), - lists:map(fun(Index) -> Index#index.sig end, Indexes). +recv(DbName, Reqs, Timeout) -> + case erpc:receive_response(Reqs, Timeout, true) of + {ok, _Label, Reqs1} -> + recv(DbName, Reqs1, Timeout); + {Error, Label, Reqs1} -> + ErrMsg = "~p : error cleaning nouveau indexes db:~p node: ~p error:~p", + couch_log:error(ErrMsg, [?MODULE, DbName, Label, Error]), + recv(DbName, Reqs1, Timeout); + no_request -> + ok + end. diff --git a/src/nouveau/src/nouveau_gun.erl b/src/nouveau/src/nouveau_gun.erl new file mode 100644 index 0000000000..75792465b4 --- /dev/null +++ b/src/nouveau/src/nouveau_gun.erl @@ -0,0 +1,162 @@ +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- + +%% index manager ensures only one process is updating a nouveau index at a time. +%% calling update_index will block until at least one attempt has been made to +%% make the index as current as the database at the time update_index was called. + +-module(nouveau_gun). +-behaviour(gen_server). +-behaviour(config_listener). + +-export([start_link/0]). +-export([host_header/0]). + +%%% gen_server callbacks +-export([init/1]). +-export([handle_call/3]). +-export([handle_cast/2]). +-export([handle_info/2]). +-export([handle_continue/2]). + +% config_listener callbacks +-export([handle_config_change/5]). +-export([handle_config_terminate/3]). + +-define(NOUVEAU_HOST_HEADER, nouveau_host_header). + +-record(state, { + enabled, + url +}). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +host_header() -> + persistent_term:get(?NOUVEAU_HOST_HEADER). + +init(_) -> + ok = config:listen_for_changes(?MODULE, nil), + State = #state{enabled = false, url = nouveau_util:nouveau_url()}, + {ok, State, {continue, reconfigure}}. + +handle_call(_Msg, _From, State) -> + {reply, unexpected_msg, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(restart_config_listener, State) -> + ok = config:listen_for_changes(?MODULE, nil), + {noreply, State}; +handle_info(reconfigure, State) -> + reconfigure(new_state(), State); +handle_info(Msg, State) -> + couch_log:warning("~p received unexpected message: ~p", [?MODULE, Msg]), + {noreply, State}. + +handle_continue(reconfigure, State) -> + reconfigure(new_state(), State). + +handle_config_change("nouveau", "enable", _Value, _Persist, nil) -> + whereis(?MODULE) ! reconfigure, + {ok, nil}; +handle_config_change("nouveau", "url", _Value, _Persist, nil) -> + whereis(?MODULE) ! reconfigure, + {ok, nil}; +handle_config_change(_Section, _Key, _Value, _Persist, nil) -> + {ok, nil}. + +handle_config_terminate(_Server, stop, nil) -> + ok; +handle_config_terminate(_Server, _Reason, nil) -> + erlang:send_after( + 500, + whereis(?MODULE), + restart_config_listener + ). + +%% private functions + +new_state() -> + #state{enabled = nouveau:enabled(), url = nouveau_util:nouveau_url()}. + +reconfigure(#state{} = State, #state{} = State) -> + %% no change + {noreply, State}; +reconfigure(#state{enabled = false} = NewState, #state{enabled = true} = CurrState) -> + %% turning off + stop_gun(CurrState#state.url), + {noreply, NewState}; +reconfigure(#state{enabled = true} = NewState, #state{enabled = false}) -> + %% turning on + case start_gun(NewState#state.url) of + {ok, _PoolPid} -> + {noreply, NewState}; + {error, Reason} -> + {stop, Reason} + end; +reconfigure(#state{enabled = true} = NewState, #state{enabled = true} = CurrState) when + NewState#state.url /= CurrState#state.url +-> + %% changing url while on + stop_gun(CurrState#state.url), + reconfigure(NewState, CurrState#state{enabled = false}); +reconfigure(#state{enabled = false} = NewState, #state{enabled = false} = CurrState) when + NewState#state.url /= CurrState#state.url +-> + %% changing url while off + {noreply, NewState}. + +start_gun(URL) -> + #{host := Host, port := Port, scheme := Scheme} = uri_string:parse(URL), + persistent_term:put(?NOUVEAU_HOST_HEADER, {<<"host">>, [Host, $:, integer_to_binary(Port)]}), + PoolSize = config:get_integer("nouveau", "pool_size", 10), + CACertFile = config:get("nouveau", "ssl_cacert_file"), + KeyFile = config:get("nouveau", "ssl_key_file"), + CertFile = config:get("nouveau", "ssl_cert_file"), + Password = config:get("nouveau", "ssl_password"), + Transport = scheme_to_transport(Scheme), + BaseConnOptions = #{transport => Transport, protocols => [http2]}, + ConnOptions = + if + Transport == tls andalso KeyFile /= undefined andalso CertFile /= undefined -> + CertKeyConf0 = #{ + certfile => CertFile, + keyfile => KeyFile, + password => Password, + cacertfile => CACertFile + }, + CertKeyConf1 = maps:filter(fun remove_undefined/2, CertKeyConf0), + BaseConnOptions#{ + tls_opts => [{certs_keys, [CertKeyConf1]}] + }; + true -> + BaseConnOptions + end, + gun_pool:start_pool(Host, Port, #{size => PoolSize, conn_opts => ConnOptions}). + +stop_gun(URL) -> + #{host := Host, port := Port, scheme := Scheme} = uri_string:parse(URL), + gun_pool:stop_pool(Host, Port, #{transport => scheme_to_transport(Scheme)}). + +remove_undefined(_Key, Value) -> + Value /= undefined. + +scheme_to_transport("http") -> + tcp; +scheme_to_transport("https") -> + tls. diff --git a/src/nouveau/src/nouveau_index_manager.erl b/src/nouveau/src/nouveau_index_manager.erl index eb18bc6f7f..45f7a1e8d7 100644 --- a/src/nouveau/src/nouveau_index_manager.erl +++ b/src/nouveau/src/nouveau_index_manager.erl @@ -36,9 +36,6 @@ handle_info/2 ]). -% config_listener api --export([handle_config_change/5, handle_config_terminate/3]). - -export([handle_db_event/3]). -define(BY_DBSIG, nouveau_by_dbsig). @@ -60,8 +57,6 @@ init(_) -> ets:new(?BY_DBSIG, [set, named_table]), ets:new(?BY_REF, [set, named_table]), couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]), - configure_ibrowse(nouveau_util:nouveau_url()), - ok = config:listen_for_changes(?MODULE, nil), {ok, nil}. handle_call({update, #index{} = Index0}, From, State) -> @@ -131,31 +126,3 @@ handle_db_event(DbName, deleted, State) -> {ok, State}; handle_db_event(_DbName, _Event, State) -> {ok, State}. - -handle_config_change("nouveau", "url", URL, _Persist, State) -> - configure_ibrowse(URL), - {ok, State}; -handle_config_change(_Section, _Key, _Value, _Persist, State) -> - {ok, State}. - -handle_config_terminate(_Server, stop, _State) -> - ok; -handle_config_terminate(_Server, _Reason, _State) -> - erlang:send_after( - 5000, - whereis(?MODULE), - restart_config_listener - ). - -configure_ibrowse(URL) -> - #{host := Host, port := Port} = uri_string:parse(URL), - ibrowse:set_max_sessions( - Host, - Port, - nouveau_util:max_sessions() - ), - ibrowse:set_max_pipeline_size( - Host, - Port, - nouveau_util:max_pipeline_size() - ). diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl index efed245db4..3952a893f2 100644 --- a/src/nouveau/src/nouveau_index_updater.erl +++ b/src/nouveau/src/nouveau_index_updater.erl @@ -33,10 +33,7 @@ changes_done, total_changes, exclude_idrevs, - reqids, - conn_pid, - update_seq, - max_pipeline_size + update_seq }). -record(purge_acc, { @@ -79,12 +76,11 @@ update(#index{} = Index) -> %% update status every half second couch_task_status:set_update_frequency(500), - {ok, ConnPid} = ibrowse:spawn_link_worker_process(nouveau_util:nouveau_url()), PurgeAcc0 = #purge_acc{ index_update_seq = IndexUpdateSeq, index_purge_seq = IndexPurgeSeq }, - {ok, PurgeAcc1} = purge_index(ConnPid, Db, Index, PurgeAcc0), + {ok, PurgeAcc1} = purge_index(Db, Index, PurgeAcc0), NewCurSeq = couch_db:get_update_seq(Db), Proc = get_os_process(Index#index.def_lang), @@ -98,18 +94,13 @@ update(#index{} = Index) -> changes_done = 0, total_changes = TotalChanges, exclude_idrevs = PurgeAcc1#purge_acc.exclude_list, - reqids = queue:new(), - conn_pid = ConnPid, - update_seq = PurgeAcc1#purge_acc.index_update_seq, - max_pipeline_size = nouveau_util:max_pipeline_size() + update_seq = PurgeAcc1#purge_acc.index_update_seq }, {ok, Acc1} = couch_db:fold_changes( Db, Acc0#acc.update_seq, fun load_docs/2, Acc0, [] ), - nouveau_api:drain_async_responses(Acc1#acc.reqids, 0), - exit(nouveau_api:set_update_seq(ConnPid, Index, Acc1#acc.update_seq, NewCurSeq)) + exit(nouveau_api:set_update_seq(Index, Acc1#acc.update_seq, NewCurSeq)) after - ibrowse:stop_worker_process(ConnPid), ret_os_process(Proc) end end @@ -119,11 +110,7 @@ update(#index{} = Index) -> load_docs(#full_doc_info{id = <<"_design/", _/binary>>}, #acc{} = Acc) -> {ok, Acc}; -load_docs(FDI, #acc{} = Acc0) -> - %% block for responses so we stay under the max pipeline size - ReqIds1 = nouveau_api:drain_async_responses(Acc0#acc.reqids, Acc0#acc.max_pipeline_size), - Acc1 = Acc0#acc{reqids = ReqIds1}, - +load_docs(FDI, #acc{} = Acc1) -> couch_task_status:update([ {changes_done, Acc1#acc.changes_done}, {progress, (Acc1#acc.changes_done * 100) div Acc1#acc.total_changes} @@ -138,7 +125,6 @@ load_docs(FDI, #acc{} = Acc0) -> false -> case update_or_delete_index( - Acc1#acc.conn_pid, Acc1#acc.db, Acc1#acc.index, Acc1#acc.update_seq, @@ -146,10 +132,9 @@ load_docs(FDI, #acc{} = Acc0) -> Acc1#acc.proc ) of - {ibrowse_req_id, ReqId} -> + ok -> Acc1#acc{ - update_seq = DI#doc_info.high_seq, - reqids = queue:in(ReqId, Acc1#acc.reqids) + update_seq = DI#doc_info.high_seq }; {error, Reason} -> exit({error, Reason}) @@ -157,11 +142,11 @@ load_docs(FDI, #acc{} = Acc0) -> end, {ok, Acc2#acc{changes_done = Acc2#acc.changes_done + 1}}. -update_or_delete_index(ConnPid, Db, #index{} = Index, MatchSeq, #doc_info{} = DI, Proc) -> +update_or_delete_index(Db, #index{} = Index, MatchSeq, #doc_info{} = DI, Proc) -> #doc_info{id = Id, high_seq = Seq, revs = [#rev_info{deleted = Del} | _]} = DI, case Del of true -> - nouveau_api:delete_doc_async(ConnPid, Index, Id, MatchSeq, Seq); + nouveau_api:delete_doc(Index, Id, MatchSeq, Seq); false -> {ok, Doc} = couch_db:open_doc(Db, DI, []), Json = couch_doc:to_json_obj(Doc, []), @@ -175,10 +160,10 @@ update_or_delete_index(ConnPid, Db, #index{} = Index, MatchSeq, #doc_info{} = DI end, case Fields of [] -> - nouveau_api:delete_doc_async(ConnPid, Index, Id, MatchSeq, Seq); + nouveau_api:delete_doc(Index, Id, MatchSeq, Seq); _ -> - nouveau_api:update_doc_async( - ConnPid, Index, Id, MatchSeq, Seq, Partition, Fields + nouveau_api:update_doc( + Index, Id, MatchSeq, Seq, Partition, Fields ) end end. @@ -223,7 +208,7 @@ index_definition(#index{} = Index) -> <<"field_analyzers">> => Index#index.field_analyzers }. -purge_index(ConnPid, Db, Index, #purge_acc{} = PurgeAcc0) -> +purge_index(Db, Index, #purge_acc{} = PurgeAcc0) -> Proc = get_os_process(Index#index.def_lang), try true = proc_prompt(Proc, [<<"add_fun">>, Index#index.def, <<"nouveau">>]), @@ -232,7 +217,7 @@ purge_index(ConnPid, Db, Index, #purge_acc{} = PurgeAcc0) -> case couch_db:get_full_doc_info(Db, Id) of not_found -> ok = nouveau_api:purge_doc( - ConnPid, Index, Id, PurgeAcc1#purge_acc.index_purge_seq, PurgeSeq + Index, Id, PurgeAcc1#purge_acc.index_purge_seq, PurgeSeq ), PurgeAcc1#purge_acc{index_purge_seq = PurgeSeq}; FDI -> @@ -243,7 +228,6 @@ purge_index(ConnPid, Db, Index, #purge_acc{} = PurgeAcc0) -> PurgeAcc1; false -> update_or_delete_index( - ConnPid, Db, Index, PurgeAcc1#purge_acc.index_update_seq, @@ -265,7 +249,7 @@ purge_index(ConnPid, Db, Index, #purge_acc{} = PurgeAcc0) -> ), DbPurgeSeq = couch_db:get_purge_seq(Db), ok = nouveau_api:set_purge_seq( - ConnPid, Index, PurgeAcc3#purge_acc.index_purge_seq, DbPurgeSeq + Index, PurgeAcc3#purge_acc.index_purge_seq, DbPurgeSeq ), update_local_doc(Db, Index, DbPurgeSeq), {ok, PurgeAcc3} diff --git a/src/nouveau/src/nouveau_rpc.erl b/src/nouveau/src/nouveau_rpc.erl index 5d954b5f3e..2037c7e7ef 100644 --- a/src/nouveau/src/nouveau_rpc.erl +++ b/src/nouveau/src/nouveau_rpc.erl @@ -17,52 +17,62 @@ -export([ search/3, - info/2, - cleanup/2 + info/2 ]). -include("nouveau.hrl"). -import(nouveau_util, [index_path/1]). -search(DbName, #index{} = Index0, QueryArgs0) -> +search(DbName, #index{} = Index, #{} = QueryArgs) -> + search(DbName, #index{} = Index, QueryArgs, 0). + +search(DbName, #index{} = Index0, QueryArgs0, UpdateLatency) -> %% Incorporate the shard name into the record. Index1 = Index0#index{dbname = DbName}, - %% get minimum seqs for search - {MinUpdateSeq, MinPurgeSeq} = nouveau_index_updater:get_db_info(Index1), + Update = maps:get(update, QueryArgs0, true), %% Incorporate min seqs into the query args. - QueryArgs1 = QueryArgs0#{ - min_update_seq => MinUpdateSeq, - min_purge_seq => MinPurgeSeq - }, - Update = maps:get(update, QueryArgs1, true), - - %% check if index is up to date - T0 = erlang:monotonic_time(), - case Update andalso nouveau_index_updater:outdated(Index1) of - true -> - case nouveau_index_manager:update_index(Index1) of - ok -> - ok; - {error, Reason} -> - rexi:reply({error, Reason}) - end; - false -> - ok; - {error, Reason} -> - rexi:reply({error, Reason}) - end, - T1 = erlang:monotonic_time(), - UpdateLatency = erlang:convert_time_unit(T1 - T0, native, millisecond), + QueryArgs1 = + case Update of + true -> + %% get minimum seqs for search + {MinUpdateSeq, MinPurgeSeq} = nouveau_index_updater:get_db_info(Index1), + QueryArgs0#{ + min_update_seq => MinUpdateSeq, + min_purge_seq => MinPurgeSeq + }; + false -> + QueryArgs0#{ + min_update_seq => 0, + min_purge_seq => 0 + } + end, %% Run the search case nouveau_api:search(Index1, QueryArgs1) of {ok, Response} -> rexi:reply({ok, Response#{update_latency => UpdateLatency}}); - {error, stale_index} -> - %% try again. - search(DbName, Index0, QueryArgs0); + {error, stale_index} when Update -> + update_and_retry(DbName, Index0, QueryArgs0, UpdateLatency); + {error, {not_found, _}} when Update -> + update_and_retry(DbName, Index0, QueryArgs0, UpdateLatency); + Else -> + rexi:reply(Else) + end. + +update_and_retry(DbName, Index, QueryArgs, UpdateLatency) -> + T0 = erlang:monotonic_time(), + case nouveau_index_manager:update_index(Index#index{dbname = DbName}) of + ok -> + T1 = erlang:monotonic_time(), + search( + DbName, + Index, + QueryArgs, + UpdateLatency + + erlang:convert_time_unit(T1 - T0, native, millisecond) + ); Else -> rexi:reply(Else) end. @@ -77,7 +87,3 @@ info(DbName, #index{} = Index0) -> {error, Reason} -> rexi:reply({error, Reason}) end. - -cleanup(DbName, Exclusions) -> - nouveau_api:delete_path(nouveau_util:index_name(DbName), Exclusions), - rexi:reply(ok). diff --git a/src/nouveau/src/nouveau_sup.erl b/src/nouveau/src/nouveau_sup.erl index 3547b43fa1..65afe744a3 100644 --- a/src/nouveau/src/nouveau_sup.erl +++ b/src/nouveau/src/nouveau_sup.erl @@ -23,6 +23,7 @@ start_link() -> init(_Args) -> Children = [ + child(nouveau_gun), child(nouveau_index_manager) ], {ok, {{one_for_one, 10, 1}, couch_epi:register_service(nouveau_epi, Children)}}. diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl index b6dd0fcbd5..3df43f2ffc 100644 --- a/src/nouveau/src/nouveau_util.erl +++ b/src/nouveau/src/nouveau_util.erl @@ -27,9 +27,9 @@ maybe_create_local_purge_doc/2, get_local_purge_doc_id/1, get_local_purge_doc_body/3, - nouveau_url/0, - max_sessions/0, - max_pipeline_size/0 + get_purge_checkpoints/1, + get_signatures_from_ddocs/2, + nouveau_url/0 ]). index_name(Path) when is_binary(Path) -> @@ -75,16 +75,15 @@ design_doc_to_index(DbName, #doc{id = Id, body = {Fields}}, IndexName) -> undefined -> {error, InvalidDDocError}; Def -> - Sig = ?l2b( - couch_util:to_hex( + Sig = + couch_util:to_hex_bin( crypto:hash( sha256, ?term_to_bin( {DefaultAnalyzer, FieldAnalyzers, Def} ) ) - ) - ), + ), {ok, #index{ dbname = DbName, default_analyzer = DefaultAnalyzer, @@ -177,6 +176,9 @@ maybe_create_local_purge_doc(Db, Index) -> get_local_purge_doc_id(Sig) -> iolist_to_binary([?LOCAL_DOC_PREFIX, "purge-", "nouveau-", Sig]). +get_purge_checkpoints(Db) -> + couch_index_util:get_purge_checkpoints(Db, <<"nouveau">>). + get_local_purge_doc_body(LocalDocId, PurgeSeq, Index) -> #index{ name = IdxName, @@ -196,11 +198,13 @@ get_local_purge_doc_body(LocalDocId, PurgeSeq, Index) -> ]}, couch_doc:from_json_obj(JsonList). -nouveau_url() -> - config:get("nouveau", "url", "http://127.0.0.1:5987"). +get_signatures_from_ddocs(DbName, DesignDocs) -> + SigList = lists:flatmap(fun(Doc) -> active_sigs(DbName, Doc) end, DesignDocs), + #{Sig => true || Sig <- SigList}. -max_sessions() -> - config:get_integer("nouveau", "max_sessions", 100). +active_sigs(DbName, #doc{} = Doc) -> + Indexes = nouveau_util:design_doc_to_indexes(DbName, Doc), + lists:map(fun(Index) -> Index#index.sig end, Indexes). -max_pipeline_size() -> - config:get_integer("nouveau", "max_pipeline_size", 1000). +nouveau_url() -> + config:get("nouveau", "url", "http://127.0.0.1:5987"). diff --git a/src/rexi/src/rexi.erl b/src/rexi/src/rexi.erl index 02d3a9e555..99333c02ff 100644 --- a/src/rexi/src/rexi.erl +++ b/src/rexi/src/rexi.erl @@ -275,7 +275,7 @@ wait_for_ack(Count, Timeout) -> end. drain_acks(Count) when Count < 0 -> - erlang:error(mismatched_rexi_ack); + error(mismatched_rexi_ack); drain_acks(Count) -> receive {rexi_ack, N} -> drain_acks(Count - N) diff --git a/src/rexi/src/rexi_monitor.erl b/src/rexi/src/rexi_monitor.erl index 7fe66db71d..8208b7691b 100644 --- a/src/rexi/src/rexi_monitor.erl +++ b/src/rexi/src/rexi_monitor.erl @@ -27,7 +27,7 @@ start(Procs) -> ), spawn_link(fun() -> [notify_parent(Parent, P, noconnect) || P <- Skip], - [erlang:monitor(process, P) || P <- Mon], + [monitor(process, P) || P <- Mon], wait_monitors(Parent) end). diff --git a/src/rexi/src/rexi_server.erl b/src/rexi/src/rexi_server.erl index b2df65c719..9028616bd4 100644 --- a/src/rexi/src/rexi_server.erl +++ b/src/rexi/src/rexi_server.erl @@ -206,7 +206,7 @@ notify_caller({Caller, Ref}, Reason) -> kill_worker(FromRef, #st{clients = Clients} = St) -> case find_worker(FromRef, Clients) of #job{worker = KeyRef, worker_pid = Pid} = Job -> - erlang:demonitor(KeyRef), + demonitor(KeyRef), exit(Pid, kill), remove_job(Job, St), ok; diff --git a/src/rexi/src/rexi_server_mon.erl b/src/rexi/src/rexi_server_mon.erl index 1677bd3108..7049cfa216 100644 --- a/src/rexi/src/rexi_server_mon.erl +++ b/src/rexi/src/rexi_server_mon.erl @@ -57,7 +57,6 @@ aggregate_queue_len(ChildMod) -> % Mem3 cluster callbacks cluster_unstable(Server) -> - couch_log:notice("~s : cluster unstable", [?MODULE]), gen_server:cast(Server, cluster_unstable), Server. @@ -75,7 +74,7 @@ init(ChildMod) -> ?CLUSTER_STABILITY_PERIOD_SEC ), start_servers(ChildMod), - couch_log:notice("~s : started servers", [ChildMod]), + couch_log:info("~s : started servers", [ChildMod]), {ok, ChildMod}. handle_call(status, _From, ChildMod) -> @@ -93,13 +92,13 @@ handle_call(Msg, _From, St) -> % can be started, but do not immediately stop nodes, defer that till cluster % stabilized. handle_cast(cluster_unstable, ChildMod) -> - couch_log:notice("~s : cluster unstable", [ChildMod]), + couch_log:info("~s : cluster unstable", [ChildMod]), start_servers(ChildMod), {noreply, ChildMod}; % When cluster is stable, start any servers for new nodes and stop servers for % the ones that disconnected. handle_cast(cluster_stable, ChildMod) -> - couch_log:notice("~s : cluster stable", [ChildMod]), + couch_log:info("~s : cluster stable", [ChildMod]), start_servers(ChildMod), stop_servers(ChildMod), {noreply, ChildMod}; @@ -153,7 +152,7 @@ start_server(ChildMod, ChildId) -> {ok, Pid} -> {ok, Pid}; Else -> - erlang:error(Else) + error(Else) end. stop_server(ChildMod, ChildId) -> diff --git a/src/rexi/src/rexi_utils.erl b/src/rexi/src/rexi_utils.erl index 146d0238ac..598ef8e7e0 100644 --- a/src/rexi/src/rexi_utils.erl +++ b/src/rexi/src/rexi_utils.erl @@ -38,7 +38,7 @@ send(Dest, Msg) -> recv(Refs, Keypos, Fun, Acc0, infinity, PerMsgTO) -> process_mailbox(Refs, Keypos, Fun, Acc0, nil, PerMsgTO); recv(Refs, Keypos, Fun, Acc0, GlobalTimeout, PerMsgTO) -> - TimeoutRef = erlang:make_ref(), + TimeoutRef = make_ref(), TRef = erlang:send_after(GlobalTimeout, self(), {timeout, TimeoutRef}), try process_mailbox(Refs, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) diff --git a/src/rexi/test/rexi_tests.erl b/src/rexi/test/rexi_tests.erl index 18b05b545c..cc2c0cc1a5 100644 --- a/src/rexi/test/rexi_tests.erl +++ b/src/rexi/test/rexi_tests.erl @@ -143,7 +143,7 @@ t_stream2(_) -> t_stream2_acks(_) -> Ref = rexi:cast(node(), {?MODULE, rpc_test_fun, [stream2_acks]}), {WPid, _Tag} = From = stream_init(Ref), - Mon = erlang:monitor(process, WPid), + Mon = monitor(process, WPid), rexi:stream_start(From), ?assertEqual(a, recv(Ref)), ?assertEqual(b, recv(Ref)), @@ -168,7 +168,7 @@ t_stream2_acks(_) -> t_stream2_cancel(_) -> Ref = rexi:cast(node(), {?MODULE, rpc_test_fun, [stream2_init]}), {WPid, _Tag} = From = stream_init(Ref), - Mon = erlang:monitor(process, WPid), + Mon = monitor(process, WPid), rexi:stream_cancel(From), Res = receive diff --git a/src/setup/src/setup.erl b/src/setup/src/setup.erl index 7a38e69cbe..31bc1f58ef 100644 --- a/src/setup/src/setup.erl +++ b/src/setup/src/setup.erl @@ -376,7 +376,7 @@ add_node_int(Options, true) -> end. get_port(Port) when is_integer(Port) -> - list_to_binary(integer_to_list(Port)); + integer_to_binary(Port); get_port(Port) when is_list(Port) -> list_to_binary(Port); get_port(Port) when is_binary(Port) -> diff --git a/src/setup/test/t-frontend-setup.sh b/src/setup/test/t-frontend-setup.sh index e025cfba2d..106312dece 100755 --- a/src/setup/test/t-frontend-setup.sh +++ b/src/setup/test/t-frontend-setup.sh @@ -64,8 +64,8 @@ curl a:b@127.0.0.1:25984/_node/node2@127.0.0.1/_config/cluster/n curl a:b@127.0.0.1:15984/_node/node1@127.0.0.1/_config/couchdb/uuid curl a:b@127.0.0.1:15984/_node/node2@127.0.0.1/_config/couchdb/uuid -curl a:b@127.0.0.1:15984/_node/node1@127.0.0.1/_config/couch_httpd_auth/secret -curl a:b@127.0.0.1:15984/_node/node2@127.0.0.1/_config/couch_httpd_auth/secret +curl a:b@127.0.0.1:15984/_node/node1@127.0.0.1/_config/chttpd_auth/secret +curl a:b@127.0.0.1:15984/_node/node2@127.0.0.1/_config/chttpd_auth/secret echo "YAY ALL GOOD" diff --git a/src/smoosh/src/smoosh.erl b/src/smoosh/src/smoosh.erl index 68e8d1828e..00dc186ca5 100644 --- a/src/smoosh/src/smoosh.erl +++ b/src/smoosh/src/smoosh.erl @@ -64,7 +64,14 @@ fold_local_shards(Fun, Acc0) -> enqueue_views(ShardName) -> DbName = mem3:dbname(ShardName), - {ok, DDocs} = fabric:design_docs(DbName), + DDocs = + case fabric:design_docs(DbName) of + {ok, Resp} when is_list(Resp) -> + Resp; + Else -> + couch_log:debug("Invalid design docs: ~p~n", [Else]), + [] + end, [sync_enqueue({ShardName, id(DDoc)}) || DDoc <- DDocs]. id(#doc{id = Id}) -> diff --git a/src/smoosh/src/smoosh_channel.erl b/src/smoosh/src/smoosh_channel.erl index 3cfbcdec69..eabb751adb 100644 --- a/src/smoosh/src/smoosh_channel.erl +++ b/src/smoosh/src/smoosh_channel.erl @@ -169,8 +169,8 @@ handle_info({Ref, {ok, Pid}}, #state{} = State) when is_reference(Ref) -> LogMsg = "~s: Started compaction for ~s", LogArgs = [Name, smoosh_utils:stringify(Key)], couch_log:Level(LogMsg, LogArgs), - erlang:monitor(process, Pid), - erlang:demonitor(Ref, [flush]), + monitor(process, Pid), + demonitor(Ref, [flush]), Active1 = Active#{Key => Pid}, State1 = State#state{active = Active1, starting = Starting1}, {noreply, set_status(State1)}; @@ -350,7 +350,7 @@ start_compact(#state{} = State, {Shard, GroupId} = Key) -> case couch_index_server:get_index(couch_mrview_index, Shard, GroupId) of {ok, Pid} -> schedule_cleanup_index_files(Shard), - Ref = erlang:monitor(process, Pid), + Ref = monitor(process, Pid), Pid ! {'$gen_call', {self(), Ref}, compact}, State#state{starting = Starting#{Ref => Key}}; Error -> @@ -370,12 +370,12 @@ start_compact(#state{} = State, Db) -> case couch_db:get_compactor_pid(Db) of nil -> DbPid = couch_db:get_pid(Db), - Ref = erlang:monitor(process, DbPid), + Ref = monitor(process, DbPid), DbPid ! {'$gen_call', {self(), Ref}, start_compact}, State#state{starting = Starting#{Ref => Key}}; % Compaction is already running, so monitor existing compaction pid. CPid when is_pid(CPid) -> - erlang:monitor(process, CPid), + monitor(process, CPid), Level = smoosh_utils:log_level("compaction_log_level", "notice"), LogMsg = "~s : db ~s continuing compaction", LogArgs = [Name, smoosh_utils:stringify(Key)], @@ -398,7 +398,7 @@ maybe_remonitor_cpid(#state{} = State, DbName, Reason) when is_binary(DbName) -> re_enqueue(DbName), State; CPid when is_pid(CPid) -> - erlang:monitor(process, CPid), + monitor(process, CPid), Level = smoosh_utils:log_level("compaction_log_level", "notice"), LogMsg = "~s: ~s compaction already running. Re-monitor Pid ~p", LogArgs = [Name, smoosh_utils:stringify(DbName), CPid], @@ -451,7 +451,7 @@ re_enqueue(Obj) -> cleanup_index_files(DbName) -> case should_clean_up_indices() of - true -> fabric:cleanup_index_files(DbName); + true -> fabric:cleanup_index_files_this_node(DbName); false -> ok end. diff --git a/src/smoosh/src/smoosh_persist.erl b/src/smoosh/src/smoosh_persist.erl index c1519f65fa..f615fcbb93 100644 --- a/src/smoosh/src/smoosh_persist.erl +++ b/src/smoosh/src/smoosh_persist.erl @@ -71,8 +71,8 @@ persist(true, Waiting, Active, Starting) -> % already running. We want them to be the first ones to continue after % restart. We're relying on infinity sorting higher than float and integer % numeric values here. - AMap = maps:map(fun(_, _) -> infinity end, Active), - SMap = maps:from_list([{K, infinity} || K <- maps:values(Starting)]), + AMap = #{K => infinity || K := _Pid <- Active}, + SMap = #{K => infinity || _Ref := K <- Starting}, Path = file_path(Name), write(maps:merge(WMap, maps:merge(AMap, SMap)), Path). diff --git a/src/smoosh/test/smoosh_tests.erl b/src/smoosh/test/smoosh_tests.erl index 6861db5e18..5170248753 100644 --- a/src/smoosh/test/smoosh_tests.erl +++ b/src/smoosh/test/smoosh_tests.erl @@ -155,7 +155,7 @@ t_index_cleanup_happens_by_default(DbName) -> get_channel_pid("index_cleanup") ! unpause, {ok, _} = fabric:query_view(DbName, <<"foo">>, <<"bar">>), % View cleanup should have been invoked - meck:wait(fabric, cleanup_index_files, [DbName], 4000), + meck:wait(fabric, cleanup_index_files_this_node, [DbName], 4000), wait_view_compacted(DbName, <<"foo">>). t_index_cleanup_can_be_disabled(DbName) -> @@ -183,17 +183,17 @@ t_suspend_resume(DbName) -> ok = wait_to_enqueue(DbName), CompPid = wait_db_compactor_pid(), ok = smoosh:suspend(), - ?assertEqual({status, suspended}, erlang:process_info(CompPid, status)), + ?assertEqual({status, suspended}, process_info(CompPid, status)), ?assertEqual({1, 0, 0}, sync_status("ratio_dbs")), % Suspending twice should work too ok = smoosh:suspend(), - ?assertEqual({status, suspended}, erlang:process_info(CompPid, status)), + ?assertEqual({status, suspended}, process_info(CompPid, status)), ?assertEqual({1, 0, 0}, sync_status("ratio_dbs")), ok = smoosh:resume(), - ?assertNotEqual({status, suspended}, erlang:process_info(CompPid, status)), + ?assertNotEqual({status, suspended}, process_info(CompPid, status)), % Resuming twice should work too ok = smoosh:resume(), - ?assertNotEqual({status, suspended}, erlang:process_info(CompPid, status)), + ?assertNotEqual({status, suspended}, process_info(CompPid, status)), CompPid ! continue, wait_compacted(DbName). @@ -205,7 +205,7 @@ t_check_window_can_resume(DbName) -> ok = wait_to_enqueue(DbName), CompPid = wait_db_compactor_pid(), ok = smoosh:suspend(), - ?assertEqual({status, suspended}, erlang:process_info(CompPid, status)), + ?assertEqual({status, suspended}, process_info(CompPid, status)), get_channel_pid("ratio_dbs") ! check_window, CompPid ! continue, wait_compacted(DbName). @@ -252,7 +252,7 @@ t_checkpointing_works(DbName) -> checkpoint(), % Stop smoosh and then crash the compaction ok = application:stop(smoosh), - Ref = erlang:monitor(process, CompPid), + Ref = monitor(process, CompPid), CompPid ! {raise, exit, kapow}, receive {'DOWN', Ref, _, _, kapow} -> @@ -276,7 +276,7 @@ t_ignore_checkpoint_resume_if_compacted_already(DbName) -> checkpoint(), % Stop smoosh and then let the compaction finish ok = application:stop(smoosh), - Ref = erlang:monitor(process, CompPid), + Ref = monitor(process, CompPid), CompPid ! continue, receive {'DOWN', Ref, _, _, normal} -> ok diff --git a/src/weatherreport/src/weatherreport_check_mem3_sync.erl b/src/weatherreport/src/weatherreport_check_mem3_sync.erl index cabca5d505..7193d77d70 100644 --- a/src/weatherreport/src/weatherreport_check_mem3_sync.erl +++ b/src/weatherreport/src/weatherreport_check_mem3_sync.erl @@ -43,7 +43,7 @@ valid() -> -spec check(list()) -> [{atom(), term()}]. check(_Opts) -> - case erlang:whereis(mem3_sync) of + case whereis(mem3_sync) of undefined -> [{warning, mem3_sync_not_found}]; Pid -> diff --git a/src/weatherreport/src/weatherreport_check_node_stats.erl b/src/weatherreport/src/weatherreport_check_node_stats.erl index 6c3353dc6c..d8aa5da9f1 100644 --- a/src/weatherreport/src/weatherreport_check_node_stats.erl +++ b/src/weatherreport/src/weatherreport_check_node_stats.erl @@ -60,7 +60,7 @@ mean_to_message({Statistic, Mean}) -> -spec check(list()) -> [{atom(), term()}]. check(_Opts) -> SumOfStats = recon:node_stats(?SAMPLES, 100, fun sum_absolute_stats/2, []), - MeanStats = [{K, erlang:round(V / ?SAMPLES)} || {K, V} <- SumOfStats], + MeanStats = [{K, round(V / ?SAMPLES)} || {K, V} <- SumOfStats], lists:map(fun mean_to_message/1, MeanStats). -spec format(term()) -> {io:format(), [term()]}. diff --git a/src/weatherreport/src/weatherreport_check_nodes_connected.erl b/src/weatherreport/src/weatherreport_check_nodes_connected.erl index 3890542091..6ad369c23d 100644 --- a/src/weatherreport/src/weatherreport_check_nodes_connected.erl +++ b/src/weatherreport/src/weatherreport_check_nodes_connected.erl @@ -50,7 +50,7 @@ valid() -> -spec check(list()) -> [{atom(), term()}]. check(_Opts) -> NodeName = node(), - ConnectedNodes = [NodeName | erlang:nodes()], + ConnectedNodes = [NodeName | nodes()], Members = mem3:nodes(), [ {warning, {node_disconnected, N}} diff --git a/src/weatherreport/src/weatherreport_check_process_calls.erl b/src/weatherreport/src/weatherreport_check_process_calls.erl index b6a228aeb4..64a5585a42 100644 --- a/src/weatherreport/src/weatherreport_check_process_calls.erl +++ b/src/weatherreport/src/weatherreport_check_process_calls.erl @@ -62,7 +62,7 @@ fold_processes([{Count, {M, F, A}} | T], Acc, Lim, CallType, Opts) -> case proplists:get_value(expert, Opts) of true -> PidFun = list_to_atom("find_by_" ++ CallType ++ "_call"), - Pids = erlang:apply(recon, PidFun, [M, F]), + Pids = apply(recon, PidFun, [M, F]), Pinfos = lists:map( fun(Pid) -> Pinfo = recon:info(Pid), diff --git a/src/weatherreport/src/weatherreport_getopt.erl b/src/weatherreport/src/weatherreport_getopt.erl index 7361126300..c66f8e085f 100644 --- a/src/weatherreport/src/weatherreport_getopt.erl +++ b/src/weatherreport/src/weatherreport_getopt.erl @@ -423,7 +423,7 @@ to_type(boolean, Arg) -> true -> false; false -> - erlang:error(badarg) + error(badarg) end end; to_type(_Type, Arg) -> diff --git a/src/weatherreport/src/weatherreport_node.erl b/src/weatherreport/src/weatherreport_node.erl index d108d0f7f9..c1fad83410 100644 --- a/src/weatherreport/src/weatherreport_node.erl +++ b/src/weatherreport/src/weatherreport_node.erl @@ -75,7 +75,7 @@ local_command(Module, Function, Args, Timeout) -> "Local function call: ~p:~p(~p)", [Module, Function, Args] ), - erlang:apply(Module, Function, Args); + apply(Module, Function, Args); _ -> weatherreport_log:log( node(), diff --git a/src/weatherreport/src/weatherreport_util.erl b/src/weatherreport/src/weatherreport_util.erl index ef42505e91..c4f9fb42a6 100644 --- a/src/weatherreport/src/weatherreport_util.erl +++ b/src/weatherreport/src/weatherreport_util.erl @@ -54,7 +54,7 @@ run_command(Command) -> "Running shell command: ~s", [Command] ), - Port = erlang:open_port({spawn, Command}, [exit_status, stderr_to_stdout]), + Port = open_port({spawn, Command}, [exit_status, stderr_to_stdout]), do_read(Port, []). do_read(Port, Acc) -> diff --git a/test/elixir/lib/asserts.ex b/test/elixir/lib/asserts.ex new file mode 100644 index 0000000000..cfbd64738f --- /dev/null +++ b/test/elixir/lib/asserts.ex @@ -0,0 +1,20 @@ +defmodule Couch.Test.Asserts do + @moduledoc """ + Custom asserts. + """ + defmacro assert_on_status(resp, expected, failure_message) do + expected_list = List.wrap(expected) + + expected_msg = case expected_list do + [single] -> "Expected #{single}" + multiple -> "Expected one of #{inspect(multiple)}" + end + + quote do + status_code = unquote(resp).status_code + body = unquote(resp).body + message = "#{unquote(failure_message)} #{unquote(expected_msg)}, got: #{status_code}, body: #{inspect(body)}" + ExUnit.Assertions.assert(status_code in unquote(expected_list), "#{message}") + end + end +end diff --git a/test/elixir/test/changes_async_test.exs b/test/elixir/test/changes_async_test.exs index 4850393c72..ae8fb41b26 100644 --- a/test/elixir/test/changes_async_test.exs +++ b/test/elixir/test/changes_async_test.exs @@ -394,7 +394,7 @@ defmodule ChangesAsyncTest do end end - defp process_response(id, chunk_parser, timeout \\ 1000) do + defp process_response(id, chunk_parser, timeout \\ 3000) do receive do %HTTPotion.AsyncChunk{id: ^id} = msg -> chunk_parser.(msg) diff --git a/test/elixir/test/config/nouveau.elixir b/test/elixir/test/config/nouveau.elixir index 02157e2ca3..1c962aafac 100644 --- a/test/elixir/test/config/nouveau.elixir +++ b/test/elixir/test/config/nouveau.elixir @@ -30,6 +30,7 @@ "purge with conflicts", "index same field with different field types", "index not found", - "meta" + "meta", + "stale search" ] } diff --git a/test/elixir/test/config/suite.elixir b/test/elixir/test/config/suite.elixir index 1d1e6059a3..9b5dacd756 100644 --- a/test/elixir/test/config/suite.elixir +++ b/test/elixir/test/config/suite.elixir @@ -513,6 +513,18 @@ "serial execution is not spuriously counted as loop on test_rewrite_suite_db", "serial execution is not spuriously counted as loop on test_rewrite_suite_db%2Fwith_slashes" ], + "ValidateDocUpdateTest": [ + "JavaScript VDU accepts a valid document", + "JavaScript VDU rejects an invalid document", + "JavaScript VDU accepts a valid change", + "JavaScript VDU rejects an invalid change", + "Mango VDU accepts a valid document", + "Mango VDU rejects an invalid document", + "updating a Mango VDU updates its effects", + "converting a Mango VDU to JavaScript updates its effects", + "deleting a Mango VDU removes its effects", + "Mango VDU rejects a doc if any existing ddoc fails to match", + ], "SecurityValidationTest": [ "Author presence and user security", "Author presence and user security when replicated", diff --git a/test/elixir/test/nouveau_test.exs b/test/elixir/test/nouveau_test.exs index 5f97c0aa3c..36426d9995 100644 --- a/test/elixir/test/nouveau_test.exs +++ b/test/elixir/test/nouveau_test.exs @@ -694,6 +694,23 @@ defmodule NouveauTest do assert resp.body["update_latency"] > 0 end + @tag :with_db + test "stale search", context do + db_name = context[:db_name] + url = "/#{db_name}/_design/foo/_nouveau/bar" + create_ddoc(db_name) + + resp = Couch.get(url, query: %{q: "*:*", update: false, include_docs: true}) + assert_status_code(resp, 404) + + create_search_docs(db_name) + resp = Couch.get(url, query: %{q: "*:*", include_docs: true}) + assert_status_code(resp, 200) + ids = get_ids(resp) + # nouveau sorts by _id as tie-breaker + assert ids == ["doc1", "doc2", "doc3", "doc4"] + end + def seq(str) do String.to_integer(hd(Regex.run(~r/^[0-9]+/, str))) end diff --git a/test/elixir/test/partition_search_test.exs b/test/elixir/test/partition_search_test.exs index 1219954492..9310e701d2 100644 --- a/test/elixir/test/partition_search_test.exs +++ b/test/elixir/test/partition_search_test.exs @@ -1,5 +1,6 @@ defmodule PartitionSearchTest do use CouchTestCase + import Couch.Test.Asserts @moduletag :search @@ -22,7 +23,7 @@ defmodule PartitionSearchTest do end resp = Couch.post("/#{db_name}/_bulk_docs", headers: ["Content-Type": "application/json"], body: %{:docs => docs}, query: %{w: 3}) - assert resp.status_code in [201, 202] + assert_on_status(resp, [201, 202], "Cannot create search docs.") end def create_ddoc(db_name, opts \\ %{}) do @@ -39,7 +40,7 @@ defmodule PartitionSearchTest do ddoc = Enum.into(opts, default_ddoc) resp = Couch.put("/#{db_name}/_design/library", body: ddoc) - assert resp.status_code in [201, 202] + assert_on_status(resp, [201, 202], "Cannot create design doc.") assert Map.has_key?(resp.body, "ok") == true end @@ -56,13 +57,13 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_partition/foo/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field"}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do partitioned search.") ids = get_ids(resp) assert ids == ["foo:10", "foo:2", "foo:4", "foo:6", "foo:8"] url = "/#{db_name}/_partition/bar/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field"}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do partitioned search.") ids = get_ids(resp) assert ids == ["bar:1", "bar:3", "bar:5", "bar:7", "bar:9"] end @@ -75,7 +76,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_partition/foo/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field"}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do partitioned search.") ids = get_ids(resp) assert ids == ["foo:10", "foo:2", "foo:4", "foo:6", "foo:8"] end @@ -88,24 +89,24 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_partition/foo/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field", limit: 3}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do partitioned search.") ids = get_ids(resp) assert ids == ["foo:10", "foo:2", "foo:4"] %{:body => %{"bookmark" => bookmark}} = resp resp = Couch.get(url, query: %{q: "some:field", limit: 3, bookmark: bookmark}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do partitioned search with a bookmark.") ids = get_ids(resp) assert ids == ["foo:6", "foo:8"] resp = Couch.get(url, query: %{q: "some:field", limit: 2000, bookmark: bookmark}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do partition search with an upper bound on the limit.") ids = get_ids(resp) assert ids == ["foo:6", "foo:8"] resp = Couch.get(url, query: %{q: "some:field", limit: 2001, bookmark: bookmark}) - assert resp.status_code == 400 + assert_on_status(resp, 400, "Should fail to do partition search with over limit.") end @tag :with_db @@ -116,7 +117,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_design/library/_search/books" resp = Couch.post(url, body: %{:q => "some:field", :limit => 1}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do POST for non-partitioned db with limit.") end @tag :with_partitioned_db @@ -127,7 +128,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_partition/foo/_design/library/_search/books" resp = Couch.post(url, body: %{:q => "some:field", :limit => 1}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do POST for partitioned db with limit.") end @tag :with_partitioned_db @@ -138,7 +139,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field"}) - assert resp.status_code == 400 + assert_on_status(resp, 400, "Expected a failure to do a global query on partitioned view.") %{:body => %{"reason" => reason}} = resp assert Regex.match?(~r/mandatory for queries to this index./, reason) end @@ -151,7 +152,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_partition/foo/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field"}) - assert resp.status_code == 400 + assert_on_status(resp, 400, "Expected a failure to do a query with a global search ddoc.") %{:body => %{"reason" => reason}} = resp assert reason == "`partition` not supported on this index" end @@ -164,7 +165,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field"}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Failed to search on non-partitioned dbs.") ids = get_ids(resp) assert Enum.sort(ids) == Enum.sort(["bar:1", "bar:5", "bar:9", "foo:2", "bar:3", "foo:4", "foo:6", "bar:7", "foo:8", "foo:10"]) end @@ -177,7 +178,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field"}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Failed to search on non-partitioned dbs without the limit.") ids = get_ids(resp) assert Enum.sort(ids) == Enum.sort(["bar:1", "bar:5", "bar:9", "foo:2", "bar:3", "foo:4", "foo:6", "bar:7", "foo:8", "foo:10"]) end @@ -189,10 +190,16 @@ defmodule PartitionSearchTest do create_ddoc(db_name) url = "/#{db_name}/_design/library/_search/books" + + # score order varies by Lucene version, so captured this order first. + resp = Couch.get(url, query: %{q: "some:field"}) + assert_on_status(resp, 200, "Failed to search on non-partitioned dbs without the limit.") + expected_ids = get_ids(resp) + + # Assert that the limit:3 results are the first 3 results from the unlimited search resp = Couch.get(url, query: %{q: "some:field", limit: 3}) - assert resp.status_code == 200 - ids = get_ids(resp) - assert Enum.sort(ids) == Enum.sort(["bar:1", "bar:5", "bar:9"]) + assert_on_status(resp, 200, "Failed to search on non-partitioned dbs with the limit.") + assert List.starts_with?(expected_ids, get_ids(resp)) end @tag :with_db @@ -203,7 +210,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_design/library/_search/books" resp = Couch.get(url, query: %{q: "some:field", limit: 201}) - assert resp.status_code == 400 + assert_on_status(resp, 400, "Expected a failure on non-partitioned dbs with over limit.") end @tag :with_partitioned_db @@ -214,7 +221,7 @@ defmodule PartitionSearchTest do url = "/#{db_name}/_partition/foo/_design/library/_search/books" resp = Couch.post(url, body: %{q: "some:field", partition: "bar"}) - assert resp.status_code == 400 + assert_on_status(resp, 400, "Expected a failure on conflicting partition values.") end @tag :with_partitioned_db diff --git a/test/elixir/test/search_test.exs b/test/elixir/test/search_test.exs index edf08f30d5..ad5a13dbbb 100644 --- a/test/elixir/test/search_test.exs +++ b/test/elixir/test/search_test.exs @@ -1,5 +1,6 @@ defmodule SearchTest do use CouchTestCase + import Couch.Test.Asserts @moduletag :search @@ -17,7 +18,10 @@ defmodule SearchTest do %{"item" => "date", "place" => "lobby", "state" => "unknown", "price" => 1.25}, ]} ) - assert resp.status_code in [201, 202] + assert resp.status_code in [201, 202], + "Cannot create search docs. " <> + "Expected one of [201, 202], got: #{resp.status_code}, body: #{inspect resp.body}" + end def create_ddoc(db_name, opts \\ %{}) do @@ -40,7 +44,7 @@ defmodule SearchTest do ddoc = Enum.into(opts, default_ddoc) resp = Couch.put("/#{db_name}/_design/inventory", body: ddoc) - assert resp.status_code in [201, 202] + assert_on_status(resp, [201, 202], "Cannot create design doc.") assert Map.has_key?(resp.body, "ok") == true end @@ -54,7 +58,7 @@ defmodule SearchTest do ddoc = Enum.into(opts, invalid_ddoc) resp = Couch.put("/#{db_name}/_design/search", body: ddoc) - assert resp.status_code in [201, 202] + assert_on_status(resp, [201, 202], "Cannot create design doc.") assert Map.has_key?(resp.body, "ok") == true end @@ -71,7 +75,7 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.get(url, query: %{q: "*:*", include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") ids = get_items(resp) assert Enum.sort(ids) == Enum.sort(["apple", "banana", "carrot", "date"]) end @@ -84,7 +88,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.get(url, query: %{q: "*:*", drilldown: :jiffy.encode(["place", "kitchen"]), include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == Enum.sort(["apple", "banana", "carrot"]) end @@ -97,7 +102,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.get(url, query: %{q: "*:*", drilldown: :jiffy.encode(["state", "new", "unknown"]), include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == Enum.sort(["apple", "banana", "date"]) end @@ -110,7 +116,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.get(url, query: %{q: "*:*", drilldown: :jiffy.encode([["state", "old"], ["item", "apple"]]), include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == [] end @@ -123,7 +130,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits?q=*:*&drilldown=[\"state\",\"old\"]&drilldown=[\"item\",\"apple\"]&include_docs=true" resp = Couch.get(url) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == [] end @@ -137,7 +145,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.post(url, body: %{q: "*:*", include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == Enum.sort(["apple", "banana", "carrot", "date"]) end @@ -150,7 +159,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.post(url, body: %{query: "*:*", drilldown: ["place", "kitchen"], include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == Enum.sort(["apple", "banana", "carrot"]) end @@ -163,7 +173,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.post(url, body: %{query: "*:*", drilldown: ["state", "new", "unknown"], include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == Enum.sort(["apple", "banana", "date"]) end @@ -176,7 +187,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.post(url, body: %{q: "*:*", drilldown: [["state", "old"], ["item", "apple"]], include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == [] end @@ -189,7 +201,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.post(url, body: %{q: "*:*", drilldown: [["place", "kitchen"], ["state", "new"], ["item", "apple"]], include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == ["apple"] end @@ -202,7 +215,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.post(url, body: %{q: "*:*", drilldown: [["state", "old", "new"], ["item", "apple"]], include_docs: true}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == ["apple"] end @@ -215,7 +229,8 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" resp = Couch.post(url, body: "{\"include_docs\": true, \"q\": \"*:*\", \"drilldown\": [\"state\", \"old\"], \"drilldown\": [\"item\", \"apple\"]}") - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") + ids = get_items(resp) assert Enum.sort(ids) == ["apple"] end @@ -228,7 +243,7 @@ defmodule SearchTest do create_invalid_ddoc(db_name) resp = Couch.post("/#{db_name}/_search_cleanup") - assert resp.status_code in [201, 202] + assert_on_status(resp, [201, 202], "Fail to do a _search_cleanup.") end @tag :with_db @@ -240,7 +255,7 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" counts = ["place"] resp = Couch.get(url, query: %{q: "*:*", limit: 0, counts: :jiffy.encode(counts)}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") %{:body => %{"counts" => counts}} = resp assert counts == %{"place" => %{"kitchen" => 3, "lobby" => 1}} @@ -255,7 +270,7 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" counts = ["place"] resp = Couch.get(url, query: %{q: "item:tomato", limit: 0, counts: :jiffy.encode(counts)}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") %{:body => %{"counts" => counts}} = resp assert counts == %{"place" => %{}} @@ -270,7 +285,7 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" ranges = %{"price" => %{"cheap" => "[0 TO 0.99]", "expensive" => "[1.00 TO Infinity]"}} resp = Couch.get(url, query: %{q: "*:*", limit: 0, ranges: :jiffy.encode(ranges)}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") %{:body => %{"ranges" => ranges}} = resp assert ranges == %{"price" => %{"cheap" => 2, "expensive" => 2}} @@ -285,9 +300,37 @@ defmodule SearchTest do url = "/#{db_name}/_design/inventory/_search/fruits" ranges = %{"price" => %{}} resp = Couch.get(url, query: %{q: "*:*", limit: 0, ranges: :jiffy.encode(ranges)}) - assert resp.status_code == 200 + assert_on_status(resp, 200, "Fail to do search.") %{:body => %{"ranges" => ranges}} = resp assert ranges == %{"price" => %{}} end + + @tag :with_db + test "timeouts do not expose internal state", context do + db_name = context[:db_name] + create_search_docs(db_name) + create_ddoc(db_name) + + config = [ + %{ + :section => "fabric", + :key => "search_timeout", + :value => "0" + } + ] + + run_on_modified_server(config, fn -> + url = "/#{db_name}/_design/inventory/_search/fruits" + resp = Couch.get(url, query: %{q: "*:*", include_docs: true}) + assert resp.status_code == 500 + + %{ + :body => %{ + "error" => "timeout", + "reason" => "The request could not be processed in a reasonable amount of time." + } + } = resp + end) + end end diff --git a/test/elixir/test/validate_doc_update_test.exs b/test/elixir/test/validate_doc_update_test.exs new file mode 100644 index 0000000000..93ed8f177c --- /dev/null +++ b/test/elixir/test/validate_doc_update_test.exs @@ -0,0 +1,212 @@ +defmodule ValidateDocUpdateTest do + use CouchTestCase + + @moduledoc """ + Test validate_doc_update behaviour + """ + + @js_type_check %{ + language: "javascript", + + validate_doc_update: ~s""" + function (newDoc) { + if (!newDoc.type) { + throw {forbidden: 'Documents must have a type field'}; + } + } + """ + } + + @tag :with_db + test "JavaScript VDU accepts a valid document", context do + db = context[:db_name] + Couch.put("/#{db}/_design/js-test", body: @js_type_check) + + resp = Couch.put("/#{db}/doc", body: %{"type" => "movie"}) + assert resp.status_code == 201 + assert resp.body["ok"] == true + end + + @tag :with_db + test "JavaScript VDU rejects an invalid document", context do + db = context[:db_name] + Couch.put("/#{db}/_design/js-test", body: @js_type_check) + + resp = Couch.put("/#{db}/doc", body: %{"not" => "valid"}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end + + @js_change_check %{ + language: "javascript", + + validate_doc_update: ~s""" + function (newDoc, oldDoc) { + if (oldDoc && newDoc.type !== oldDoc.type) { + throw {forbidden: 'Documents cannot change their type field'}; + } + } + """ + } + + @tag :with_db + test "JavaScript VDU accepts a valid change", context do + db = context[:db_name] + Couch.put("/#{db}/_design/js-test", body: @js_change_check) + + Couch.put("/#{db}/doc", body: %{"type" => "movie"}) + + doc = Couch.get("/#{db}/doc").body + updated = doc |> Map.merge(%{"type" => "movie", "title" => "Duck Soup"}) + resp = Couch.put("/#{db}/doc", body: updated) + + assert resp.status_code == 201 + end + + @tag :with_db + test "JavaScript VDU rejects an invalid change", context do + db = context[:db_name] + Couch.put("/#{db}/_design/js-test", body: @js_change_check) + + Couch.put("/#{db}/doc", body: %{"type" => "movie"}) + + doc = Couch.get("/#{db}/doc").body + updated = doc |> Map.put("type", "director") + resp = Couch.put("/#{db}/doc", body: updated) + + assert resp.status_code == 403 + end + + @mango_type_check %{ + language: "query", + + validate_doc_update: %{ + "newDoc" => %{"type" => %{"$exists" => true}} + } + } + + @tag :with_db + test "Mango VDU accepts a valid document", context do + db = context[:db_name] + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc", body: %{"type" => "movie"}) + assert resp.status_code == 201 + assert resp.body["ok"] == true + end + + @tag :with_db + test "Mango VDU rejects an invalid document", context do + db = context[:db_name] + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc", body: %{"no" => "type"}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end + + @tag :with_db + test "updating a Mango VDU updates its effects", context do + db = context[:db_name] + + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + ddoc = %{ + language: "query", + + validate_doc_update: %{ + "newDoc" => %{ + "type" => %{"$type" => "string"}, + "year" => %{"$lt" => 2026} + } + } + } + resp = Couch.put("/#{db}/_design/mango-test", body: ddoc, query: %{rev: resp.body["rev"]}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc1", body: %{"type" => "movie", "year" => 1994}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc2", body: %{"type" => 42, "year" => 1994}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + + resp = Couch.put("/#{db}/doc3", body: %{"type" => "movie", "year" => 2094}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end + + @tag :with_db + test "converting a Mango VDU to JavaScript updates its effects", context do + db = context[:db_name] + + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + ddoc = %{ + language: "javascript", + + validate_doc_update: ~s""" + function (newDoc) { + if (typeof newDoc.year !== 'number') { + throw {forbidden: 'Documents must have a valid year field'}; + } + } + """ + } + resp = Couch.put("/#{db}/_design/mango-test", body: ddoc, query: %{rev: resp.body["rev"]}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc1", body: %{"year" => 1994}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc2", body: %{"year" => "1994"}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end + + @tag :with_db + test "deleting a Mango VDU removes its effects", context do + db = context[:db_name] + + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + resp = Couch.delete("/#{db}/_design/mango-test", query: %{rev: resp.body["rev"]}) + assert resp.status_code == 200 + + resp = Couch.put("/#{db}/doc", body: %{"no" => "type"}) + assert resp.status_code == 201 + end + + @tag :with_db + test "Mango VDU rejects a doc if any existing ddoc fails to match", context do + db = context[:db_name] + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + ddoc = %{ + language: "query", + + validate_doc_update: %{ + "newDoc" => %{"year" => %{"$lt" => 2026}} + } + } + resp = Couch.put("/#{db}/_design/mango-test-2", body: ddoc) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc1", body: %{"type" => "movie", "year" => 1994}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc2", body: %{"year" => 1994}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + + resp = Couch.put("/#{db}/doc3", body: %{"type" => "movie", "year" => 2094}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end +end diff --git a/version.mk b/version.mk index 2a199c9ffc..fd4879ca04 100644 --- a/version.mk +++ b/version.mk @@ -1,3 +1,3 @@ vsn_major=3 vsn_minor=5 -vsn_patch=0 +vsn_patch=1