diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4b96db04e7c..401dd14eb8d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,3 +13,4 @@ /handwritten/firestore @googleapis/firestore-team /handwritten/spanner @googleapis/spanner-team /handwritten/bigquery-storage @googleapis/bigquery-team +/core/paginator jsteam-handwritten-libraries diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 965b1bc91f6..5422bacd1c2 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,10 +1,11 @@ { + "core/paginator": "6.0.0", "handwritten/bigquery": "8.2.0", "handwritten/bigquery-storage": "5.1.0", "handwritten/cloud-profiler": "6.0.4", "handwritten/datastore": "10.1.0", - "handwritten/firestore": "8.3.0", "handwritten/error-reporting": "3.0.5", + "handwritten/firestore": "8.3.0", "handwritten/logging-bunyan": "5.1.1", "handwritten/logging-winston": "6.0.1", "handwritten/spanner": "8.6.0", diff --git a/core/paginator/.OwlBot.yaml b/core/paginator/.OwlBot.yaml new file mode 100644 index 00000000000..46bbeee9f9f --- /dev/null +++ b/core/paginator/.OwlBot.yaml @@ -0,0 +1,17 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +begin-after-commit-hash: 397c0bfd367a2427104f988d5329bc117caafd95 + diff --git a/core/paginator/.compodocrc b/core/paginator/.compodocrc new file mode 100644 index 00000000000..cd8b42152a6 --- /dev/null +++ b/core/paginator/.compodocrc @@ -0,0 +1,10 @@ +--- +tsconfig: ./tsconfig.json +output: ./docs +theme: material +hideGenerator: true +disablePrivate: true +disableProtected: true +disableInternal: true +disableCoverage: true +disableGraph: true diff --git a/core/paginator/.eslintignore b/core/paginator/.eslintignore new file mode 100644 index 00000000000..c4a0963e9bd --- /dev/null +++ b/core/paginator/.eslintignore @@ -0,0 +1,8 @@ +**/node_modules +**/coverage +test/fixtures +build/ +docs/ +protos/ +samples/generated/ +system-test/**/fixtures diff --git a/core/paginator/.eslintrc.json b/core/paginator/.eslintrc.json new file mode 100644 index 00000000000..78215349546 --- /dev/null +++ b/core/paginator/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "./node_modules/gts" +} diff --git a/core/paginator/.gitattributes b/core/paginator/.gitattributes new file mode 100644 index 00000000000..33739cb74e4 --- /dev/null +++ b/core/paginator/.gitattributes @@ -0,0 +1,4 @@ +*.ts text eol=lf +*.js text eol=lf +protos/* linguist-generated +**/api-extractor.json linguist-language=JSON-with-Comments diff --git a/core/paginator/.gitignore b/core/paginator/.gitignore new file mode 100644 index 00000000000..94d7296289d --- /dev/null +++ b/core/paginator/.gitignore @@ -0,0 +1,8 @@ +node_modules +.nyc_output +build +package-lock.json +docs/ +.coverage +__pycache__ +.DS_Store \ No newline at end of file diff --git a/core/paginator/.jsdoc.js b/core/paginator/.jsdoc.js new file mode 100644 index 00000000000..595bd3a14c4 --- /dev/null +++ b/core/paginator/.jsdoc.js @@ -0,0 +1,51 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +'use strict'; + +module.exports = { + opts: { + readme: './README.md', + package: './package.json', + template: './node_modules/jsdoc-fresh', + recurse: true, + verbose: true, + destination: './docs/' + }, + plugins: [ + 'plugins/markdown', + 'jsdoc-region-tag' + ], + source: { + excludePattern: '(^|\\/|\\\\)[._]', + include: [ + 'build/src', + ], + includePattern: '\\.js$' + }, + templates: { + copyright: 'Copyright 2019 Google, LLC.', + includeDate: false, + sourceFiles: false, + systemName: '@google-cloud/paginator', + theme: 'lumen', + default: { + outputSourceFiles: false + } + }, + markdown: { + idInHeadings: true + } +}; diff --git a/core/paginator/.kokoro/.gitattributes b/core/paginator/.kokoro/.gitattributes new file mode 100644 index 00000000000..87acd4f484e --- /dev/null +++ b/core/paginator/.kokoro/.gitattributes @@ -0,0 +1 @@ +* linguist-generated=true diff --git a/core/paginator/.kokoro/common.cfg b/core/paginator/.kokoro/common.cfg new file mode 100644 index 00000000000..5bbf287fbb7 --- /dev/null +++ b/core/paginator/.kokoro/common.cfg @@ -0,0 +1,24 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-paginator/core/paginator/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:14-user" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/test.sh" +} diff --git a/core/paginator/.kokoro/continuous/node18/common.cfg b/core/paginator/.kokoro/continuous/node18/common.cfg new file mode 100644 index 00000000000..186a3c85bd3 --- /dev/null +++ b/core/paginator/.kokoro/continuous/node18/common.cfg @@ -0,0 +1,24 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-paginator/core/paginator/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/test.sh" +} diff --git a/core/paginator/.kokoro/continuous/node18/lint.cfg b/core/paginator/.kokoro/continuous/node18/lint.cfg new file mode 100644 index 00000000000..84a0ec4715f --- /dev/null +++ b/core/paginator/.kokoro/continuous/node18/lint.cfg @@ -0,0 +1,4 @@ +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/lint.sh" +} diff --git a/core/paginator/.kokoro/continuous/node18/samples-test.cfg b/core/paginator/.kokoro/continuous/node18/samples-test.cfg new file mode 100644 index 00000000000..77cb50e1cd2 --- /dev/null +++ b/core/paginator/.kokoro/continuous/node18/samples-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/samples-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/core/paginator/.kokoro/continuous/node18/system-test.cfg b/core/paginator/.kokoro/continuous/node18/system-test.cfg new file mode 100644 index 00000000000..4175c5b4d5c --- /dev/null +++ b/core/paginator/.kokoro/continuous/node18/system-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/system-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/core/paginator/.kokoro/continuous/node18/test.cfg b/core/paginator/.kokoro/continuous/node18/test.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/paginator/.kokoro/docs.sh b/core/paginator/.kokoro/docs.sh new file mode 100755 index 00000000000..85901242b5e --- /dev/null +++ b/core/paginator/.kokoro/docs.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +cd $(dirname $0)/.. + +npm install + +npm run docs-test diff --git a/core/paginator/.kokoro/lint.sh b/core/paginator/.kokoro/lint.sh new file mode 100755 index 00000000000..aef4866e4c4 --- /dev/null +++ b/core/paginator/.kokoro/lint.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +cd $(dirname $0)/.. + +npm install + +# Install and link samples +if [ -f samples/package.json ]; then + cd samples/ + npm link ../ + npm install + cd .. +fi + +npm run lint diff --git a/core/paginator/.kokoro/populate-secrets.sh b/core/paginator/.kokoro/populate-secrets.sh new file mode 100755 index 00000000000..deb2b199eb4 --- /dev/null +++ b/core/paginator/.kokoro/populate-secrets.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright 2020 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is called in the early stage of `trampoline_v2.sh` to +# populate secrets needed for the CI builds. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + +# Populates requested secrets set in SECRET_MANAGER_KEYS + +# In Kokoro CI builds, we use the service account attached to the +# Kokoro VM. This means we need to setup auth on other CI systems. +# For local run, we just use the gcloud command for retrieving the +# secrets. + +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + GCLOUD_COMMANDS=( + "docker" + "run" + "--entrypoint=gcloud" + "--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR}" + "gcr.io/google.com/cloudsdktool/cloud-sdk" + ) + if [[ "${TRAMPOLINE_CI:-}" == "kokoro" ]]; then + SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" + else + echo "Authentication for this CI system is not implemented yet." + exit 2 + # TODO: Determine appropriate SECRET_LOCATION and the GCLOUD_COMMANDS. + fi +else + # For local run, use /dev/shm or temporary directory for + # KOKORO_GFILE_DIR. + if [[ -d "/dev/shm" ]]; then + export KOKORO_GFILE_DIR=/dev/shm + else + export KOKORO_GFILE_DIR=$(mktemp -d -t ci-XXXXXXXX) + fi + SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" + GCLOUD_COMMANDS=("gcloud") +fi + +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} + +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + "${GCLOUD_COMMANDS[@]}" \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret $key > \ + "$SECRET_LOCATION/$key" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + exit 2 + fi +done diff --git a/core/paginator/.kokoro/presubmit/node18/common.cfg b/core/paginator/.kokoro/presubmit/node18/common.cfg new file mode 100644 index 00000000000..186a3c85bd3 --- /dev/null +++ b/core/paginator/.kokoro/presubmit/node18/common.cfg @@ -0,0 +1,24 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-paginator/core/paginator/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/test.sh" +} diff --git a/core/paginator/.kokoro/presubmit/node18/samples-test.cfg b/core/paginator/.kokoro/presubmit/node18/samples-test.cfg new file mode 100644 index 00000000000..77cb50e1cd2 --- /dev/null +++ b/core/paginator/.kokoro/presubmit/node18/samples-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/samples-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/core/paginator/.kokoro/presubmit/node18/system-test.cfg b/core/paginator/.kokoro/presubmit/node18/system-test.cfg new file mode 100644 index 00000000000..4175c5b4d5c --- /dev/null +++ b/core/paginator/.kokoro/presubmit/node18/system-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/system-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/core/paginator/.kokoro/presubmit/node18/test.cfg b/core/paginator/.kokoro/presubmit/node18/test.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/paginator/.kokoro/presubmit/windows/common.cfg b/core/paginator/.kokoro/presubmit/windows/common.cfg new file mode 100644 index 00000000000..d6e25e0b1b8 --- /dev/null +++ b/core/paginator/.kokoro/presubmit/windows/common.cfg @@ -0,0 +1,2 @@ +# Format: //devtools/kokoro/config/proto/build.proto + diff --git a/core/paginator/.kokoro/presubmit/windows/test.cfg b/core/paginator/.kokoro/presubmit/windows/test.cfg new file mode 100644 index 00000000000..cb9baa82b39 --- /dev/null +++ b/core/paginator/.kokoro/presubmit/windows/test.cfg @@ -0,0 +1,2 @@ +# Use the test file directly +build_file: "nodejs-paginator/core/paginator/.kokoro/test.bat" diff --git a/core/paginator/.kokoro/publish.sh b/core/paginator/.kokoro/publish.sh new file mode 100755 index 00000000000..ca1d47af347 --- /dev/null +++ b/core/paginator/.kokoro/publish.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +# Start the releasetool reporter +python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + +cd $(dirname $0)/.. + +NPM_TOKEN=$(cat $KOKORO_KEYSTORE_DIR/73713_google-cloud-npm-token-1) +echo "//wombat-dressing-room.appspot.com/:_authToken=${NPM_TOKEN}" > ~/.npmrc + +npm install +npm pack . +# npm provides no way to specify, observe, or predict the name of the tarball +# file it generates. We have to look in the current directory for the freshest +# .tgz file. +TARBALL=$(ls -1 -t *.tgz | head -1) + +npm publish --access=public --registry=https://wombat-dressing-room.appspot.com "$TARBALL" + +# Kokoro collects *.tgz and package-lock.json files and stores them in Placer +# so we can generate SBOMs and attestations. +# However, we *don't* want Kokoro to collect package-lock.json and *.tgz files +# that happened to be installed with dependencies. +find node_modules -name package-lock.json -o -name "*.tgz" | xargs rm -f \ No newline at end of file diff --git a/core/paginator/.kokoro/release/common.cfg b/core/paginator/.kokoro/release/common.cfg new file mode 100644 index 00000000000..3ba2eb095fe --- /dev/null +++ b/core/paginator/.kokoro/release/common.cfg @@ -0,0 +1,8 @@ +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} diff --git a/core/paginator/.kokoro/release/docs-devsite.cfg b/core/paginator/.kokoro/release/docs-devsite.cfg new file mode 100644 index 00000000000..134d9b4257a --- /dev/null +++ b/core/paginator/.kokoro/release/docs-devsite.cfg @@ -0,0 +1,26 @@ +# service account used to publish up-to-date docs. +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +# doc publications use a Python image. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-paginator/core/paginator/.kokoro/trampoline_v2.sh" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/release/docs-devsite.sh" +} diff --git a/core/paginator/.kokoro/release/docs-devsite.sh b/core/paginator/.kokoro/release/docs-devsite.sh new file mode 100755 index 00000000000..81a89f6c172 --- /dev/null +++ b/core/paginator/.kokoro/release/docs-devsite.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +if [[ -z "$CREDENTIALS" ]]; then + # if CREDENTIALS are explicitly set, assume we're testing locally + # and don't set NPM_CONFIG_PREFIX. + export NPM_CONFIG_PREFIX=${HOME}/.npm-global + export PATH="$PATH:${NPM_CONFIG_PREFIX}/bin" + cd $(dirname $0)/../.. +fi + +npm install +npm install --no-save @google-cloud/cloud-rad@^0.4.0 +# publish docs to devsite +npx @google-cloud/cloud-rad . cloud-rad diff --git a/core/paginator/.kokoro/release/docs.cfg b/core/paginator/.kokoro/release/docs.cfg new file mode 100644 index 00000000000..96f5e78f753 --- /dev/null +++ b/core/paginator/.kokoro/release/docs.cfg @@ -0,0 +1,26 @@ +# service account used to publish up-to-date docs. +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +# doc publications use a Python image. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-paginator/core/paginator/.kokoro/trampoline_v2.sh" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/release/docs.sh" +} diff --git a/core/paginator/.kokoro/release/docs.sh b/core/paginator/.kokoro/release/docs.sh new file mode 100755 index 00000000000..e9079a60530 --- /dev/null +++ b/core/paginator/.kokoro/release/docs.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# build jsdocs (Python is installed on the Node 18 docker image). +if [[ -z "$CREDENTIALS" ]]; then + # if CREDENTIALS are explicitly set, assume we're testing locally + # and don't set NPM_CONFIG_PREFIX. + export NPM_CONFIG_PREFIX=${HOME}/.npm-global + export PATH="$PATH:${NPM_CONFIG_PREFIX}/bin" + cd $(dirname $0)/../.. +fi +npm install +npm run docs + +# create docs.metadata, based on package.json and .repo-metadata.json. +npm i json@9.0.6 -g +python3 -m docuploader create-metadata \ + --name=$(cat .repo-metadata.json | json name) \ + --version=$(cat package.json | json version) \ + --language=$(cat .repo-metadata.json | json language) \ + --distribution-name=$(cat .repo-metadata.json | json distribution_name) \ + --product-page=$(cat .repo-metadata.json | json product_documentation) \ + --github-repository=$(cat .repo-metadata.json | json repo) \ + --issue-tracker=$(cat .repo-metadata.json | json issue_tracker) +cp docs.metadata ./docs/docs.metadata + +# deploy the docs. +if [[ -z "$CREDENTIALS" ]]; then + CREDENTIALS=${KOKORO_KEYSTORE_DIR}/73713_docuploader_service_account +fi +if [[ -z "$BUCKET" ]]; then + BUCKET=docs-staging +fi +python3 -m docuploader upload ./docs --credentials $CREDENTIALS --staging-bucket $BUCKET diff --git a/core/paginator/.kokoro/release/publish.cfg b/core/paginator/.kokoro/release/publish.cfg new file mode 100644 index 00000000000..94af1b9c344 --- /dev/null +++ b/core/paginator/.kokoro/release/publish.cfg @@ -0,0 +1,51 @@ +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-npm-token-1" + } + } +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-paginator/core/paginator/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-paginator/core/paginator/.kokoro/publish.sh" +} + +# Store the packages we uploaded to npmjs.org and their corresponding +# package-lock.jsons in Placer. That way, we have a record of exactly +# what we published, and which version of which tools we used to publish +# it, which we can use to generate SBOMs and attestations. +action { + define_artifacts { + regex: "github/**/*.tgz" + regex: "github/**/package-lock.json" + strip_prefix: "github" + } +} diff --git a/core/paginator/.kokoro/samples-test.sh b/core/paginator/.kokoro/samples-test.sh new file mode 100755 index 00000000000..1025155a0e9 --- /dev/null +++ b/core/paginator/.kokoro/samples-test.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Ensure the npm global directory is writable, otherwise rebuild `npm` +mkdir -p $NPM_CONFIG_PREFIX +npm config -g ls || npm i -g npm@`npm --version` + +# Setup service account credentials. +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/secret_manager/long-door-651-kokoro-system-test-service-account +export GCLOUD_PROJECT=long-door-651 + +cd $(dirname $0)/.. + +# Run a pre-test hook, if a pre-samples-test.sh is in the project +if [ -f .kokoro/pre-samples-test.sh ]; then + set +x + . .kokoro/pre-samples-test.sh + set -x +fi + +if [ -f samples/package.json ]; then + npm install + + # Install and link samples + cd samples/ + npm link ../ + npm install + cd .. + # If tests are running against main branch, configure flakybot + # to open issues on failures: + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then + export MOCHA_REPORTER_OUTPUT=test_output_sponge_log.xml + export MOCHA_REPORTER=xunit + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP + fi + + npm run samples-test +fi + +# codecov combines coverage across integration and unit tests. Include +# the logic below for any environment you wish to collect coverage for: +COVERAGE_NODE=14 +if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then + NYC_BIN=./node_modules/nyc/bin/nyc.js + if [ -f "$NYC_BIN" ]; then + $NYC_BIN report || true + fi + bash $KOKORO_GFILE_DIR/codecov.sh +else + echo "coverage is only reported for Node $COVERAGE_NODE" +fi diff --git a/core/paginator/.kokoro/system-test.sh b/core/paginator/.kokoro/system-test.sh new file mode 100755 index 00000000000..0b3043d268c --- /dev/null +++ b/core/paginator/.kokoro/system-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +# Setup service account credentials. +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/secret_manager/long-door-651-kokoro-system-test-service-account +export GCLOUD_PROJECT=long-door-651 + +cd $(dirname $0)/.. + +# Run a pre-test hook, if a pre-system-test.sh is in the project +if [ -f .kokoro/pre-system-test.sh ]; then + set +x + . .kokoro/pre-system-test.sh + set -x +fi + +npm install + +# If tests are running against main branch, configure flakybot +# to open issues on failures: +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then + export MOCHA_REPORTER_OUTPUT=test_output_sponge_log.xml + export MOCHA_REPORTER=xunit + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi + +npm run system-test + +# codecov combines coverage across integration and unit tests. Include +# the logic below for any environment you wish to collect coverage for: +COVERAGE_NODE=14 +if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then + NYC_BIN=./node_modules/nyc/bin/nyc.js + if [ -f "$NYC_BIN" ]; then + $NYC_BIN report || true + fi + bash $KOKORO_GFILE_DIR/codecov.sh +else + echo "coverage is only reported for Node $COVERAGE_NODE" +fi diff --git a/core/paginator/.kokoro/test.bat b/core/paginator/.kokoro/test.bat new file mode 100644 index 00000000000..0bb12405231 --- /dev/null +++ b/core/paginator/.kokoro/test.bat @@ -0,0 +1,33 @@ +@rem Copyright 2018 Google LLC. All rights reserved. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@echo "Starting Windows build" + +cd /d %~dp0 +cd .. + +@rem npm path is not currently set in our image, we should fix this next time +@rem we upgrade Node.js in the image: +SET PATH=%PATH%;/cygdrive/c/Program Files/nodejs/npm + +call nvm use v14.17.3 +call which node + +call npm install || goto :error +call npm run test || goto :error + +goto :EOF + +:error +exit /b 1 diff --git a/core/paginator/.kokoro/test.sh b/core/paginator/.kokoro/test.sh new file mode 100755 index 00000000000..862d478d324 --- /dev/null +++ b/core/paginator/.kokoro/test.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +cd $(dirname $0)/.. + +npm install +# If tests are running against main branch, configure flakybot +# to open issues on failures: +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then + export MOCHA_REPORTER_OUTPUT=test_output_sponge_log.xml + export MOCHA_REPORTER=xunit + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi +# Unit tests exercise the entire API surface, which may include +# deprecation warnings: +export MOCHA_THROW_DEPRECATION=false +npm test + +# codecov combines coverage across integration and unit tests. Include +# the logic below for any environment you wish to collect coverage for: +COVERAGE_NODE=14 +if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then + NYC_BIN=./node_modules/nyc/bin/nyc.js + if [ -f "$NYC_BIN" ]; then + $NYC_BIN report || true + fi + bash $KOKORO_GFILE_DIR/codecov.sh +else + echo "coverage is only reported for Node $COVERAGE_NODE" +fi diff --git a/core/paginator/.kokoro/trampoline.sh b/core/paginator/.kokoro/trampoline.sh new file mode 100755 index 00000000000..f693a1ce7aa --- /dev/null +++ b/core/paginator/.kokoro/trampoline.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is not used any more, but we keep this file for making it +# easy to roll back. +# TODO: Remove this file from the template. + +set -eo pipefail + +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT + +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" diff --git a/core/paginator/.kokoro/trampoline_v2.sh b/core/paginator/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000000..1fdd89c9080 --- /dev/null +++ b/core/paginator/.kokoro/trampoline_v2.sh @@ -0,0 +1,510 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# If you want to make a change to this file, consider doing so at: +# https://github.com/googlecloudplatform/docker-ci-helper +# +# This script is for running CI builds. For Kokoro builds, we +# set this script to `build_file` field in the Kokoro configuration. + +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. +# +# Here is an example for running this script. +# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:10-user \ +# TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \ +# .kokoro/trampoline_v2.sh + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.7" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" + # Contains path to build artifacts being executed. + "KOKORO_BUILD_ARTIFACTS_SUBDIR" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For flakybot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")/core/paginator" +else + PROJECT_ROOT="$(repo_root $(pwd))/core/paginator" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# Auto-injected conditional check +# Check if the package directory has changes. If not, skip tests. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # The package path is hardcoded during migration + RELATIVE_PKG_PATH="core/paginator" + + echo "Checking for changes in ${RELATIVE_PKG_PATH}..." + + # Determine the diff range based on the CI system/event + # Safe default: HEAD~1..HEAD + DIFF_RANGE="HEAD~1..HEAD" + + if git diff --quiet "${DIFF_RANGE}" -- "${RELATIVE_PKG_PATH}"; then + echo "No changes detected in ${RELATIVE_PKG_PATH}. Skipping tests." + exit 0 + else + echo "Changes detected in ${RELATIVE_PKG_PATH}. Proceeding with tests." + fi +fi + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}" | grep "${TRAMPOLINE_IMAGE%:*}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/core/paginator/.mocharc.js b/core/paginator/.mocharc.js new file mode 100644 index 00000000000..0b600509bed --- /dev/null +++ b/core/paginator/.mocharc.js @@ -0,0 +1,29 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +const config = { + "enable-source-maps": true, + "throw-deprecation": true, + "timeout": 10000, + "recursive": true +} +if (process.env.MOCHA_THROW_DEPRECATION === 'false') { + delete config['throw-deprecation']; +} +if (process.env.MOCHA_REPORTER) { + config.reporter = process.env.MOCHA_REPORTER; +} +if (process.env.MOCHA_REPORTER_OUTPUT) { + config['reporter-option'] = `output=${process.env.MOCHA_REPORTER_OUTPUT}`; +} +module.exports = config diff --git a/core/paginator/.nycrc b/core/paginator/.nycrc new file mode 100644 index 00000000000..b18d5472b62 --- /dev/null +++ b/core/paginator/.nycrc @@ -0,0 +1,24 @@ +{ + "report-dir": "./.coverage", + "reporter": ["text", "lcov"], + "exclude": [ + "**/*-test", + "**/.coverage", + "**/apis", + "**/benchmark", + "**/conformance", + "**/docs", + "**/samples", + "**/scripts", + "**/protos", + "**/test", + "**/*.d.ts", + ".jsdoc.js", + "**/.jsdoc.js", + "karma.conf.js", + "webpack-tests.config.js", + "webpack.config.js" + ], + "exclude-after-remap": false, + "all": true +} diff --git a/core/paginator/.prettierignore b/core/paginator/.prettierignore new file mode 100644 index 00000000000..9340ad9b86d --- /dev/null +++ b/core/paginator/.prettierignore @@ -0,0 +1,6 @@ +**/node_modules +**/coverage +test/fixtures +build/ +docs/ +protos/ diff --git a/core/paginator/.prettierrc.js b/core/paginator/.prettierrc.js new file mode 100644 index 00000000000..d1b95106f4c --- /dev/null +++ b/core/paginator/.prettierrc.js @@ -0,0 +1,17 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module.exports = { + ...require('gts/.prettierrc.json') +} diff --git a/core/paginator/.repo-metadata.json b/core/paginator/.repo-metadata.json new file mode 100644 index 00000000000..8e1cd714bf8 --- /dev/null +++ b/core/paginator/.repo-metadata.json @@ -0,0 +1,11 @@ +{ + "name": "paginator", + "name_pretty": "Google Cloud Common Paginator", + "release_level": "stable", + "language": "nodejs", + "repo": "googleapis/google-cloud-node", + "distribution_name": "@google-cloud/paginator", + "client_documentation": "https://cloud.google.com/nodejs/docs/reference/paginator/latest", + "library_type": "OTHER", + "codeowner_team": "jsteam-handwritten-libraries" +} diff --git a/core/paginator/.trampolinerc b/core/paginator/.trampolinerc new file mode 100644 index 00000000000..9b4f78ac8be --- /dev/null +++ b/core/paginator/.trampolinerc @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Template for .trampolinerc + +# Add required env vars here. +required_envvars+=( +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + "AUTORELEASE_PR" + "VERSION" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE="" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi + +# Secret Manager secrets. +source ${PROJECT_ROOT}/core/paginator/.kokoro/populate-secrets.sh diff --git a/core/paginator/CHANGELOG.md b/core/paginator/CHANGELOG.md new file mode 100644 index 00000000000..3cab10d401d --- /dev/null +++ b/core/paginator/CHANGELOG.md @@ -0,0 +1,273 @@ +# Changelog + +[npm history][1] + +[1]: https://www.npmjs.com/package/nodejs-paginator?activeTab=versions + +## [6.0.0](https://github.com/googleapis/nodejs-paginator/compare/v5.0.2...v6.0.0) (2025-03-04) + + +### ⚠ BREAKING CHANGES + +* migrate to node 18 ([#386](https://github.com/googleapis/nodejs-paginator/issues/386)) + +### Miscellaneous Chores + +* Migrate to node 18 ([#386](https://github.com/googleapis/nodejs-paginator/issues/386)) ([1f4b49f](https://github.com/googleapis/nodejs-paginator/commit/1f4b49f1d0a0251a99b106adf6ce6a7e01f2fa27)) + +## [5.0.2](https://github.com/googleapis/nodejs-paginator/compare/v5.0.1...v5.0.2) (2024-05-23) + + +### Bug Fixes + +* Query should be on the list of extra args ([#365](https://github.com/googleapis/nodejs-paginator/issues/365)) ([50e40d0](https://github.com/googleapis/nodejs-paginator/commit/50e40d064aed1bd0d5f93a51ad54112343086644)) + +## [5.0.1](https://github.com/googleapis/nodejs-paginator/compare/v5.0.0...v5.0.1) (2024-05-22) + + +### Bug Fixes + +* Should pass extra callback arguments back to consumer ([#361](https://github.com/googleapis/nodejs-paginator/issues/361)) ([cc5c48b](https://github.com/googleapis/nodejs-paginator/commit/cc5c48b95b21e9c6a4e555ff98de267258657b6e)) + +## [5.0.0](https://github.com/googleapis/nodejs-paginator/compare/v4.0.1...v5.0.0) (2023-08-09) + + +### ⚠ BREAKING CHANGES + +* update to Node 14 ([#346](https://github.com/googleapis/nodejs-paginator/issues/346)) + +### Miscellaneous Chores + +* Update to Node 14 ([#346](https://github.com/googleapis/nodejs-paginator/issues/346)) ([262ad70](https://github.com/googleapis/nodejs-paginator/commit/262ad70d3cc5e1aa8a67ece54c04920b24ceea09)) + +## [4.0.1](https://github.com/googleapis/nodejs-paginator/compare/v4.0.0...v4.0.1) (2022-09-09) + + +### Bug Fixes + +* Remove pip install statements ([#1546](https://github.com/googleapis/nodejs-paginator/issues/1546)) ([#329](https://github.com/googleapis/nodejs-paginator/issues/329)) ([697567b](https://github.com/googleapis/nodejs-paginator/commit/697567bdd86226b740304734b9562a2f2241a96f)) + +## [4.0.0](https://github.com/googleapis/nodejs-paginator/compare/v3.0.7...v4.0.0) (2022-05-17) + + +### ⚠ BREAKING CHANGES + +* update library to use Node 12 (#325) + +### Build System + +* update library to use Node 12 ([#325](https://github.com/googleapis/nodejs-paginator/issues/325)) ([02887ae](https://github.com/googleapis/nodejs-paginator/commit/02887ae2b370bff18cae7fe1d434ecdf663b5748)) + +### [3.0.7](https://github.com/googleapis/nodejs-paginator/compare/v3.0.6...v3.0.7) (2022-02-14) + + +### Bug Fixes + +* update signature of end to comply with update node types definition ([#311](https://github.com/googleapis/nodejs-paginator/issues/311)) ([79e6fbd](https://github.com/googleapis/nodejs-paginator/commit/79e6fbdae5008d874613d2919a6cf723708fc919)) + +### [3.0.6](https://www.github.com/googleapis/nodejs-paginator/compare/v3.0.5...v3.0.6) (2021-09-09) + + +### Bug Fixes + +* **build:** switch primary branch to main ([#287](https://www.github.com/googleapis/nodejs-paginator/issues/287)) ([1b796f3](https://www.github.com/googleapis/nodejs-paginator/commit/1b796f3377174354a62b7475d16f52213197f650)) + +### [3.0.5](https://www.github.com/googleapis/nodejs-paginator/compare/v3.0.4...v3.0.5) (2020-09-02) + + +### Bug Fixes + +* add configs by running synthtool ([#241](https://www.github.com/googleapis/nodejs-paginator/issues/241)) ([643593a](https://www.github.com/googleapis/nodejs-paginator/commit/643593ae9ffb8febff69a7bdae19239f5bcb1266)) + +### [3.0.4](https://www.github.com/googleapis/nodejs-paginator/compare/v3.0.3...v3.0.4) (2020-08-06) + + +### Bug Fixes + +* destroy ResourceStream with pre-flight error ([#236](https://www.github.com/googleapis/nodejs-paginator/issues/236)) ([d57beb4](https://www.github.com/googleapis/nodejs-paginator/commit/d57beb424d875a7bf502d458cc208f1bbe47a42a)) + +### [3.0.3](https://www.github.com/googleapis/nodejs-paginator/compare/v3.0.2...v3.0.3) (2020-07-24) + + +### Bug Fixes + +* move gitattributes files to node templates ([#234](https://www.github.com/googleapis/nodejs-paginator/issues/234)) ([30e881c](https://www.github.com/googleapis/nodejs-paginator/commit/30e881ce7415749b93b6b7e4e71745ea3fb248b6)) + +### [3.0.2](https://www.github.com/googleapis/nodejs-paginator/compare/v3.0.1...v3.0.2) (2020-07-06) + + +### Bug Fixes + +* update node issue template ([#221](https://www.github.com/googleapis/nodejs-paginator/issues/221)) ([088153c](https://www.github.com/googleapis/nodejs-paginator/commit/088153c4fca6d53e2e5ef4bb42365ce5493b913d)) + +### [3.0.1](https://www.github.com/googleapis/nodejs-paginator/compare/v3.0.0...v3.0.1) (2020-05-20) + + +### Bug Fixes + +* apache license URL ([#468](https://www.github.com/googleapis/nodejs-paginator/issues/468)) ([#211](https://www.github.com/googleapis/nodejs-paginator/issues/211)) ([f343b7f](https://www.github.com/googleapis/nodejs-paginator/commit/f343b7f7e184fd1b453f20ac1463d17520aac7ad)) + +## [3.0.0](https://www.github.com/googleapis/nodejs-paginator/compare/v2.0.3...v3.0.0) (2020-03-25) + + +### ⚠ BREAKING CHANGES + +* **dep:** upgrade gts 2.0.0 (#194) +* **deps:** deprecated node 8 to 10; upgrade typescript + +### Miscellaneous Chores + +* **dep:** upgrade gts 2.0.0 ([#194](https://www.github.com/googleapis/nodejs-paginator/issues/194)) ([4eaf9be](https://www.github.com/googleapis/nodejs-paginator/commit/4eaf9bed1fcfd0f10e877ff15c1d0e968e3356c8)) +* **deps:** deprecated node 8 to 10; upgrade typescript ([f6434ab](https://www.github.com/googleapis/nodejs-paginator/commit/f6434ab9cacb6ab804c070f19c38b6072ca326b5)) + +### [2.0.3](https://www.github.com/googleapis/nodejs-paginator/compare/v2.0.2...v2.0.3) (2019-12-05) + + +### Bug Fixes + +* **deps:** pin TypeScript below 3.7.0 ([e06e1b0](https://www.github.com/googleapis/nodejs-paginator/commit/e06e1b0a2e2bb1cf56fc806c1703b8b5e468b954)) + +### [2.0.2](https://www.github.com/googleapis/nodejs-paginator/compare/v2.0.1...v2.0.2) (2019-11-13) + + +### Bug Fixes + +* **docs:** add jsdoc-region-tag plugin ([#155](https://www.github.com/googleapis/nodejs-paginator/issues/155)) ([b983799](https://www.github.com/googleapis/nodejs-paginator/commit/b98379905848fd179c6268aff3e1cfaf2bf76663)) + +### [2.0.1](https://www.github.com/googleapis/nodejs-paginator/compare/v2.0.0...v2.0.1) (2019-08-25) + + +### Bug Fixes + +* **deps:** use the latest extend ([#141](https://www.github.com/googleapis/nodejs-paginator/issues/141)) ([61b383e](https://www.github.com/googleapis/nodejs-paginator/commit/61b383e)) + +## [2.0.0](https://www.github.com/googleapis/nodejs-paginator/compare/v1.0.2...v2.0.0) (2019-07-12) + + +### ⚠ BREAKING CHANGES + +* rewrite streaming logic (#136) + +### Code Refactoring + +* rewrite streaming logic ([#136](https://www.github.com/googleapis/nodejs-paginator/issues/136)) ([641d82d](https://www.github.com/googleapis/nodejs-paginator/commit/641d82d)) + +### [1.0.2](https://www.github.com/googleapis/nodejs-paginator/compare/v1.0.1...v1.0.2) (2019-06-26) + + +### Bug Fixes + +* **docs:** link to reference docs section on googleapis.dev ([#132](https://www.github.com/googleapis/nodejs-paginator/issues/132)) ([be231be](https://www.github.com/googleapis/nodejs-paginator/commit/be231be)) + +### [1.0.1](https://www.github.com/googleapis/nodejs-paginator/compare/v1.0.0...v1.0.1) (2019-06-14) + + +### Bug Fixes + +* **docs:** move to new client docs URL ([#129](https://www.github.com/googleapis/nodejs-paginator/issues/129)) ([689f483](https://www.github.com/googleapis/nodejs-paginator/commit/689f483)) + +## [1.0.0](https://www.github.com/googleapis/nodejs-paginator/compare/v0.2.0...v1.0.0) (2019-05-03) + + +### Bug Fixes + +* **deps:** update dependency arrify to v2 ([#109](https://www.github.com/googleapis/nodejs-paginator/issues/109)) ([9f06c83](https://www.github.com/googleapis/nodejs-paginator/commit/9f06c83)) + + +### Build System + +* upgrade engines field to >=8.10.0 ([#115](https://www.github.com/googleapis/nodejs-paginator/issues/115)) ([0921076](https://www.github.com/googleapis/nodejs-paginator/commit/0921076)) + + +### BREAKING CHANGES + +* upgrade engines field to >=8.10.0 (#115) + +## v0.2.0 + +03-08-2019 12:15 PST + +### New Features +- feat: handle promise based functions ([#91](https://github.com/googleapis/nodejs-paginator/pull/91)) +- refactor(ts): create generic for object streams ([#101](https://github.com/googleapis/nodejs-paginator/pull/101)) + +### Dependencies +- chore(deps): update dependency through2 to v3 ([#53](https://github.com/googleapis/nodejs-paginator/pull/53)) +- chore(deps): update dependency @types/is to v0.0.21 ([#55](https://github.com/googleapis/nodejs-paginator/pull/55)) +- chore(deps): update dependency gts to ^0.9.0 ([#57](https://github.com/googleapis/nodejs-paginator/pull/57)) +- fix: Pin @types/sinon to last compatible version ([#61](https://github.com/googleapis/nodejs-paginator/pull/61)) +- refactor: trim a few dependencies ([#60](https://github.com/googleapis/nodejs-paginator/pull/60)) +- chore(deps): update dependency @types/sinon to v5.0.7 ([#62](https://github.com/googleapis/nodejs-paginator/pull/62)) +- chore(deps): update dependency @types/sinon to v7 ([#81](https://github.com/googleapis/nodejs-paginator/pull/81)) +- chore(deps): update dependency mocha to v6 + +### Documentation +- docs: add lint/fix example to contributing guide ([#85](https://github.com/googleapis/nodejs-paginator/pull/85)) +- chore: move CONTRIBUTING.md to root ([#87](https://github.com/googleapis/nodejs-paginator/pull/87)) +- docs: update links in contrib guide ([#94](https://github.com/googleapis/nodejs-paginator/pull/94)) +- docs: update contributing path in README ([#88](https://github.com/googleapis/nodejs-paginator/pull/88)) + +### Internal / Testing Changes +- chore: include build in eslintignore ([#49](https://github.com/googleapis/nodejs-paginator/pull/49)) +- chore: update CircleCI config ([#52](https://github.com/googleapis/nodejs-paginator/pull/52)) +- chore: use latest npm on Windows ([#54](https://github.com/googleapis/nodejs-paginator/pull/54)) +- chore: update eslintignore config ([#56](https://github.com/googleapis/nodejs-paginator/pull/56)) +- chore: add synth.metadata +- fix(build): fix system key decryption ([#64](https://github.com/googleapis/nodejs-paginator/pull/64)) +- chore: update license file ([#68](https://github.com/googleapis/nodejs-paginator/pull/68)) +- chore(build): update prettier config ([#69](https://github.com/googleapis/nodejs-paginator/pull/69)) +- chore: nyc ignore build/test by default ([#71](https://github.com/googleapis/nodejs-paginator/pull/71)) +- chore: always nyc report before calling codecov ([#72](https://github.com/googleapis/nodejs-paginator/pull/72)) +- build: add Kokoro configs for autorelease ([#75](https://github.com/googleapis/nodejs-paginator/pull/75)) +- fix(build): fix Kokoro release script ([#76](https://github.com/googleapis/nodejs-paginator/pull/76)) +- chore: fix publish.sh permission +x ([#77](https://github.com/googleapis/nodejs-paginator/pull/77)) +- chore: update nyc and eslint configs ([#79](https://github.com/googleapis/nodejs-paginator/pull/79)) +- chore(build): inject yoshi automation key ([#80](https://github.com/googleapis/nodejs-paginator/pull/80)) +- build: check broken links in generated docs ([#82](https://github.com/googleapis/nodejs-paginator/pull/82)) +- build: ignore googleapis.com in doc link check ([#84](https://github.com/googleapis/nodejs-paginator/pull/84)) +- build: test using @grpc/grpc-js in CI ([#89](https://github.com/googleapis/nodejs-paginator/pull/89)) +- build: create docs test npm scripts ([#90](https://github.com/googleapis/nodejs-paginator/pull/90)) +- build: use linkinator for docs test ([#93](https://github.com/googleapis/nodejs-paginator/pull/93)) +- build: update release configuration +- build: fix types for sinon ([#98](https://github.com/googleapis/nodejs-paginator/pull/98)) +- build: use node10 to run samples-test, system-test etc ([#97](https://github.com/googleapis/nodejs-paginator/pull/97)) +- build: Add docuploader credentials to node publish jobs ([#99](https://github.com/googleapis/nodejs-paginator/pull/99)) + +## v0.1.2 + +### Bug fixes +- fix: call limiter.makeRequest() instead of original method ([#43](https://github.com/googleapis/nodejs-paginator/pull/43)) + +### Internal / Testing Changes +- chore: update issue templates ([#42](https://github.com/googleapis/nodejs-paginator/pull/42)) +- chore: remove old issue template ([#40](https://github.com/googleapis/nodejs-paginator/pull/40)) +- build: run tests on node11 ([#39](https://github.com/googleapis/nodejs-paginator/pull/39)) +- chores(build): run codecov on continuous builds ([#36](https://github.com/googleapis/nodejs-paginator/pull/36)) +- chores(build): do not collect sponge.xml from windows builds ([#37](https://github.com/googleapis/nodejs-paginator/pull/37)) +- chore: update new issue template ([#35](https://github.com/googleapis/nodejs-paginator/pull/35)) +- chore(deps): update dependency sinon to v7 ([#31](https://github.com/googleapis/nodejs-paginator/pull/31)) +- build: fix codecov uploading on Kokoro ([#32](https://github.com/googleapis/nodejs-paginator/pull/32)) +- Update kokoro config ([#29](https://github.com/googleapis/nodejs-paginator/pull/29)) +- Update CI config ([#27](https://github.com/googleapis/nodejs-paginator/pull/27)) +- Don't publish sourcemaps ([#25](https://github.com/googleapis/nodejs-paginator/pull/25)) +- build: prevent system/sample-test from leaking credentials +- Update kokoro config ([#23](https://github.com/googleapis/nodejs-paginator/pull/23)) +- test: remove appveyor config ([#22](https://github.com/googleapis/nodejs-paginator/pull/22)) +- Update CI config ([#21](https://github.com/googleapis/nodejs-paginator/pull/21)) +- Enable prefer-const in the eslint config ([#20](https://github.com/googleapis/nodejs-paginator/pull/20)) +- Enable no-var in eslint ([#19](https://github.com/googleapis/nodejs-paginator/pull/19)) +- Update CI config ([#18](https://github.com/googleapis/nodejs-paginator/pull/18)) + +## v0.1.1 + +### Internal / Testing Changes +- Add synth script and update CI config (#14) +- chore(deps): update dependency nyc to v13 (#12) +- chore: ignore package-lock.json (#11) +- chore(deps): lock file maintenance (#10) +- chore: update renovate config (#9) +- remove that whitespace (#8) +- chore(deps): lock file maintenance (#7) +- chore(deps): update dependency typescript to v3 (#6) +- chore: assert.deelEqual => assert.deepStrictEqual (#5) +- chore: move mocha options to mocha.opts (#4) diff --git a/core/paginator/CODE_OF_CONDUCT.md b/core/paginator/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..2add2547a81 --- /dev/null +++ b/core/paginator/CODE_OF_CONDUCT.md @@ -0,0 +1,94 @@ + +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + +Reports should be directed to *googleapis-stewards@google.com*, the +Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out to the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/core/paginator/CONTRIBUTING.md b/core/paginator/CONTRIBUTING.md new file mode 100644 index 00000000000..72c44cada5e --- /dev/null +++ b/core/paginator/CONTRIBUTING.md @@ -0,0 +1,74 @@ +# How to become a contributor and submit your own code + +**Table of contents** + +* [Contributor License Agreements](#contributor-license-agreements) +* [Contributing a patch](#contributing-a-patch) +* [Running the tests](#running-the-tests) +* [Releasing the library](#releasing-the-library) + +## Contributor License Agreements + +We'd love to accept your sample apps and patches! Before we can take them, we +have to jump a couple of legal hurdles. + +Please fill out either the individual or corporate Contributor License Agreement +(CLA). + + * If you are an individual writing original source code and you're sure you + own the intellectual property, then you'll need to sign an [individual CLA](https://developers.google.com/open-source/cla/individual). + * If you work for a company that wants to allow you to contribute your work, + then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). + +Follow either of the two links above to access the appropriate CLA and +instructions for how to sign and return it. Once we receive it, we'll be able to +accept your pull requests. + +## Contributing A Patch + +1. Submit an issue describing your proposed change to the repo in question. +1. The repo owner will respond to your issue promptly. +1. If your proposed change is accepted, and you haven't already done so, sign a + Contributor License Agreement (see details above). +1. Fork the desired repo, develop and test your code changes. +1. Ensure that your code adheres to the existing style in the code to which + you are contributing. +1. Ensure that your code has an appropriate set of tests which all pass. +1. Title your pull request following [Conventional Commits](https://www.conventionalcommits.org/) styling. +1. Submit a pull request. + +### Before you begin + +1. [Select or create a Cloud Platform project][projects]. +1. [Set up authentication with a service account][auth] so you can access the + API from your local workstation. + + +## Running the tests + +1. [Prepare your environment for Node.js setup][setup]. + +1. Install dependencies: + + npm install + +1. Run the tests: + + # Run unit tests. + npm test + + # Run sample integration tests. + npm run samples-test + + # Run all system tests. + npm run system-test + +1. Lint (and maybe fix) any changes: + + npm run fix + +[setup]: https://cloud.google.com/nodejs/docs/setup +[projects]: https://console.cloud.google.com/project +[billing]: https://support.google.com/cloud/answer/6293499#enable-billing + +[auth]: https://cloud.google.com/docs/authentication/getting-started \ No newline at end of file diff --git a/core/paginator/LICENSE b/core/paginator/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/core/paginator/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/core/paginator/README.md b/core/paginator/README.md new file mode 100644 index 00000000000..d3ddff1fb7e --- /dev/null +++ b/core/paginator/README.md @@ -0,0 +1,136 @@ +[//]: # "This README.md file is auto-generated, all changes to this file will be lost." +[//]: # "To regenerate it, use `python -m synthtool`." +Google Cloud Platform logo + +# [Google Cloud Common Paginator: Node.js Client](https://github.com/googleapis/nodejs-paginator) + +[![release level](https://img.shields.io/badge/release%20level-stable-brightgreen.svg?style=flat)](https://cloud.google.com/terms/launch-stages) +[![npm version](https://img.shields.io/npm/v/@google-cloud/paginator.svg)](https://www.npmjs.org/package/@google-cloud/paginator) + + + + +A result paging utility used by Google node.js modules + + +A comprehensive list of changes in each version may be found in +[the CHANGELOG](https://github.com/googleapis/nodejs-paginator/blob/main/CHANGELOG.md). + +* [Google Cloud Common Paginator Node.js Client API Reference][client-docs] + +* [github.com/googleapis/nodejs-paginator](https://github.com/googleapis/nodejs-paginator) + +Read more about the client libraries for Cloud APIs, including the older +Google APIs Client Libraries, in [Client Libraries Explained][explained]. + +[explained]: https://cloud.google.com/apis/docs/client-libraries-explained + +**Table of contents:** + + +* [Quickstart](#quickstart) + + * [Installing the client library](#installing-the-client-library) + * [Using the client library](#using-the-client-library) +* [Samples](#samples) +* [Versioning](#versioning) +* [Contributing](#contributing) +* [License](#license) + +## Quickstart + +### Installing the client library + +```bash +npm install @google-cloud/paginator +``` + + +### Using the client library + +```javascript +const {paginator} = require('@google-cloud/paginator'); +console.log(paginator); + +``` + + + +## Samples + +Samples are in the [`samples/`](https://github.com/googleapis/nodejs-paginator/tree/main/samples) directory. Each sample's `README.md` has instructions for running its sample. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +| Quickstart | [source code](https://github.com/googleapis/nodejs-paginator/blob/main/samples/quickstart.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-paginator&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) | +| Streamify | [source code](https://github.com/googleapis/nodejs-paginator/blob/main/samples/streamify.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-paginator&page=editor&open_in_editor=samples/streamify.js,samples/README.md) | + + + +The [Google Cloud Common Paginator Node.js Client API Reference][client-docs] documentation +also contains samples. + +## Supported Node.js Versions + +Our client libraries follow the [Node.js release schedule](https://github.com/nodejs/release#release-schedule). +Libraries are compatible with all current _active_ and _maintenance_ versions of +Node.js. +If you are using an end-of-life version of Node.js, we recommend that you update +as soon as possible to an actively supported LTS version. + +Google's client libraries support legacy versions of Node.js runtimes on a +best-efforts basis with the following warnings: + +* Legacy versions are not tested in continuous integration. +* Some security patches and features cannot be backported. +* Dependencies cannot be kept up-to-date. + +Client libraries targeting some end-of-life versions of Node.js are available, and +can be installed through npm [dist-tags](https://docs.npmjs.com/cli/dist-tag). +The dist-tags follow the naming convention `legacy-(version)`. +For example, `npm install @google-cloud/paginator@legacy-8` installs client libraries +for versions compatible with Node.js 8. + +## Versioning + +This library follows [Semantic Versioning](http://semver.org/). + + + +This library is considered to be **stable**. The code surface will not change in backwards-incompatible ways +unless absolutely necessary (e.g. because of critical security issues) or with +an extensive deprecation period. Issues and requests against **stable** libraries +are addressed with the highest priority. + + + + + + +More Information: [Google Cloud Platform Launch Stages][launch_stages] + +[launch_stages]: https://cloud.google.com/terms/launch-stages + +## Contributing + +Contributions welcome! See the [Contributing Guide](https://github.com/googleapis/nodejs-paginator/blob/main/CONTRIBUTING.md). + +Please note that this `README.md`, the `samples/README.md`, +and a variety of configuration files in this repository (including `.nycrc` and `tsconfig.json`) +are generated from a central template. To edit one of these files, make an edit +to its templates in +[directory](https://github.com/googleapis/synthtool). + +## License + +Apache Version 2.0 + +See [LICENSE](https://github.com/googleapis/nodejs-paginator/blob/main/LICENSE) + +[client-docs]: https://cloud.google.com/nodejs/docs/reference/paginator/latest + +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png +[projects]: https://console.cloud.google.com/project +[billing]: https://support.google.com/cloud/answer/6293499#enable-billing + +[auth]: https://cloud.google.com/docs/authentication/external/set-up-adc-local diff --git a/core/paginator/linkinator.config.json b/core/paginator/linkinator.config.json new file mode 100644 index 00000000000..29a223b6db6 --- /dev/null +++ b/core/paginator/linkinator.config.json @@ -0,0 +1,10 @@ +{ + "recurse": true, + "skip": [ + "https://codecov.io/gh/googleapis/", + "www.googleapis.com", + "img.shields.io" + ], + "silent": true, + "concurrency": 10 +} diff --git a/core/paginator/owlbot.py b/core/paginator/owlbot.py new file mode 100644 index 00000000000..f074a33e342 --- /dev/null +++ b/core/paginator/owlbot.py @@ -0,0 +1,19 @@ +# Copyright 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import synthtool as s +import synthtool.gcp as gcp + +common_templates = gcp.CommonTemplates() +templates = common_templates.node_library() +s.copy(sources=templates, excludes=["LICENSE", "README.md", ".github/ISSUE_TEMPLATE", ".github/scripts", ".kokoro", ".github/workflows/issues-no-repro.yaml", ".jsdoc.js"]) diff --git a/core/paginator/package.json b/core/paginator/package.json new file mode 100644 index 00000000000..de0cecfcd05 --- /dev/null +++ b/core/paginator/package.json @@ -0,0 +1,64 @@ +{ + "name": "@google-cloud/paginator", + "version": "6.0.0", + "description": "A result paging utility used by Google node.js modules", + "main": "build/src/index.js", + "types": "build/src/index.d.ts", + "repository": { + "type": "git", + "directory": "core/paginator", + "url": "https://github.com/googleapis/google-cloud-node.git" + }, + "scripts": { + "test": "c8 mocha build/test", + "compile": "tsc -p .", + "fix": "gts fix", + "prelint": "cd samples; npm link ../; npm install", + "lint": "gts check", + "prepare": "npm run compile", + "pretest": "npm run compile", + "docs": "jsdoc -c .jsdoc.js", + "presystem-test": "npm run compile", + "samples-test": "cd samples/ && npm link ../ && npm test && cd ../", + "system-test": "mocha build/system-test", + "docs-test": "linkinator docs", + "predocs-test": "npm run docs", + "clean": "gts clean", + "precompile": "gts clean" + }, + "keywords": [], + "files": [ + "build/src", + "!build/src/**/*.map" + ], + "author": "Google Inc.", + "license": "Apache-2.0", + "devDependencies": { + "@types/extend": "^3.0.4", + "@types/mocha": "^10.0.10", + "@types/node": "^22.13.8", + "@types/proxyquire": "^1.3.31", + "@types/sinon": "^17.0.4", + "@types/uuid": "^10.0.0", + "c8": "^10.1.3", + "codecov": "^3.8.3", + "gts": "^6.0.2", + "jsdoc": "^4.0.4", + "jsdoc-fresh": "^3.0.0", + "jsdoc-region-tag": "^3.0.0", + "linkinator": "^6.1.2", + "mocha": "^11.1.0", + "path-to-regexp": "^8.2.0", + "proxyquire": "^2.1.3", + "sinon": "^19.0.2", + "typescript": "^5.8.2", + "uuid": "^11.1.0" + }, + "dependencies": { + "extend": "^3.0.2" + }, + "engines": { + "node": ">=18" + }, + "homepage": "https://github.com/googleapis/google-cloud-node/tree/main/core/paginator" +} diff --git a/core/paginator/src/index.ts b/core/paginator/src/index.ts new file mode 100644 index 00000000000..dd5f569840c --- /dev/null +++ b/core/paginator/src/index.ts @@ -0,0 +1,275 @@ +/*! + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module common/paginator + */ + +import * as extend from 'extend'; +import {TransformOptions} from 'stream'; +import {ResourceStream} from './resource-stream'; + +export interface ParsedArguments extends TransformOptions { + /** + * Query object. This is most commonly an object, but to make the API more + * simple, it can also be a string in some places. + */ + query?: ParsedArguments; + + /** + * Callback function. + */ + callback?: Function; + + /** + * Auto-pagination enabled. + */ + autoPaginate?: boolean; + + /** + * Maximum API calls to make. + */ + maxApiCalls?: number; + + /** + * Maximum results to return. + */ + maxResults?: number; + + pageSize?: number; + + streamOptions?: ParsedArguments; +} + +/*! Developer Documentation + * + * paginator is used to auto-paginate `nextQuery` methods as well as + * streamifying them. + * + * Before: + * + * search.query('done=true', function(err, results, nextQuery) { + * search.query(nextQuery, function(err, results, nextQuery) {}); + * }); + * + * After: + * + * search.query('done=true', function(err, results) {}); + * + * Methods to extend should be written to accept callbacks and return a + * `nextQuery`. + */ + +export class Paginator { + /** + * Cache the original method, then overwrite it on the Class's prototype. + * + * @param {function} Class - The parent class of the methods to extend. + * @param {string|string[]} methodNames - Name(s) of the methods to extend. + */ + // tslint:disable-next-line:variable-name + extend(Class: Function, methodNames: string | string[]) { + if (typeof methodNames === 'string') { + methodNames = [methodNames]; + } + methodNames.forEach(methodName => { + const originalMethod = Class.prototype[methodName]; + + // map the original method to a private member + Class.prototype[methodName + '_'] = originalMethod; + + // overwrite the original to auto-paginate + /* eslint-disable @typescript-eslint/no-explicit-any */ + Class.prototype[methodName] = function (...args: any[]) { + const parsedArguments = paginator.parseArguments_(args); + return paginator.run_(parsedArguments, originalMethod.bind(this)); + }; + }); + } + + /** + * Wraps paginated API calls in a readable object stream. + * + * This method simply calls the nextQuery recursively, emitting results to a + * stream. The stream ends when `nextQuery` is null. + * + * `maxResults` will act as a cap for how many results are fetched and emitted + * to the stream. + * + * @param {string} methodName - Name of the method to streamify. + * @return {function} - Wrapped function. + */ + /* eslint-disable @typescript-eslint/no-explicit-any */ + streamify(methodName: string) { + return function ( + // tslint:disable-next-line:no-any + this: {[index: string]: Function}, + /* eslint-disable @typescript-eslint/no-explicit-any */ + ...args: any[] + ): ResourceStream { + const parsedArguments = paginator.parseArguments_(args); + const originalMethod = this[methodName + '_'] || this[methodName]; + return paginator.runAsStream_( + parsedArguments, + originalMethod.bind(this), + ); + }; + } + + /** + * Parse a pseudo-array `arguments` for a query and callback. + * + * @param {array} args - The original `arguments` pseduo-array that the original + * method received. + */ + /* eslint-disable @typescript-eslint/no-explicit-any */ + parseArguments_(args: any[]) { + let query: string | ParsedArguments | undefined; + let autoPaginate = true; + let maxApiCalls = -1; + let maxResults = -1; + let callback: Function | undefined; + + const firstArgument = args[0]; + const lastArgument = args[args.length - 1]; + + if (typeof firstArgument === 'function') { + callback = firstArgument; + } else { + query = firstArgument; + } + + if (typeof lastArgument === 'function') { + callback = lastArgument; + } + + if (typeof query === 'object') { + query = extend<{}, ParsedArguments>(true, {}, query) as ParsedArguments; + + // Check if the user only asked for a certain amount of results. + if (query.maxResults && typeof query.maxResults === 'number') { + // `maxResults` is used API-wide. + maxResults = query.maxResults; + } else if (typeof query.pageSize === 'number') { + // `pageSize` is Pub/Sub's `maxResults`. + maxResults = query.pageSize!; + } + + if (query.maxApiCalls && typeof query.maxApiCalls === 'number') { + maxApiCalls = query.maxApiCalls; + delete query.maxApiCalls; + } + + // maxResults is the user specified limit. + if (maxResults !== -1 || query.autoPaginate === false) { + autoPaginate = false; + } + } + + const parsedArguments = { + query: query || {}, + autoPaginate, + maxApiCalls, + maxResults, + callback, + } as ParsedArguments; + + parsedArguments.streamOptions = extend<{}, ParsedArguments>( + true, + {}, + parsedArguments.query as ParsedArguments, + ); + delete parsedArguments.streamOptions.autoPaginate; + delete parsedArguments.streamOptions.maxResults; + delete parsedArguments.streamOptions.pageSize; + + return parsedArguments; + } + + /** + * This simply checks to see if `autoPaginate` is set or not, if it's true + * then we buffer all results, otherwise simply call the original method. + * + * @param {array} parsedArguments - Parsed arguments from the original method + * call. + * @param {object=|string=} parsedArguments.query - Query object. This is most + * commonly an object, but to make the API more simple, it can also be a + * string in some places. + * @param {function=} parsedArguments.callback - Callback function. + * @param {boolean} parsedArguments.autoPaginate - Auto-pagination enabled. + * @param {boolean} parsedArguments.maxApiCalls - Maximum API calls to make. + * @param {number} parsedArguments.maxResults - Maximum results to return. + * @param {function} originalMethod - The cached method that accepts a callback + * and returns `nextQuery` to receive more results. + */ + run_(parsedArguments: ParsedArguments, originalMethod: Function) { + const query = parsedArguments.query; + const callback = parsedArguments.callback!; + if (!parsedArguments.autoPaginate) { + return originalMethod(query, callback); + } + const results = new Array<{}>(); + let otherArgs: unknown[] = []; + const promise = new Promise((resolve, reject) => { + const stream = paginator.runAsStream_(parsedArguments, originalMethod); + stream + .on('error', reject) + .on('data', (data: {}) => results.push(data)) + .on('end', () => { + otherArgs = stream._otherArgs || []; + resolve(results); + }); + }); + if (!callback) { + return promise.then(results => [results, query, ...otherArgs]); + } + promise.then( + results => callback(null, results, query, ...otherArgs), + (err: Error) => callback(err), + ); + } + + /** + * This method simply calls the nextQuery recursively, emitting results to a + * stream. The stream ends when `nextQuery` is null. + * + * `maxResults` will act as a cap for how many results are fetched and emitted + * to the stream. + * + * @param {object=|string=} parsedArguments.query - Query object. This is most + * commonly an object, but to make the API more simple, it can also be a + * string in some places. + * @param {function=} parsedArguments.callback - Callback function. + * @param {boolean} parsedArguments.autoPaginate - Auto-pagination enabled. + * @param {boolean} parsedArguments.maxApiCalls - Maximum API calls to make. + * @param {number} parsedArguments.maxResults - Maximum results to return. + * @param {function} originalMethod - The cached method that accepts a callback + * and returns `nextQuery` to receive more results. + * @return {stream} - Readable object stream. + */ + /* eslint-disable @typescript-eslint/no-explicit-any */ + runAsStream_( + parsedArguments: ParsedArguments, + originalMethod: Function, + ): ResourceStream { + return new ResourceStream(parsedArguments, originalMethod); + } +} + +const paginator = new Paginator(); +export {paginator}; + +export {ResourceStream}; diff --git a/core/paginator/src/resource-stream.ts b/core/paginator/src/resource-stream.ts new file mode 100644 index 00000000000..fdb1522e71b --- /dev/null +++ b/core/paginator/src/resource-stream.ts @@ -0,0 +1,118 @@ +/*! + * Copyright 2019 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Transform, Writable} from 'stream'; +import {ParsedArguments} from './'; + +interface ResourceEvents { + addListener(event: 'data', listener: (data: T) => void): this; + emit(event: 'data', data: T): boolean; + on(event: 'data', listener: (data: T) => void): this; + once(event: 'data', listener: (data: T) => void): this; + prependListener(event: 'data', listener: (data: T) => void): this; + prependOnceListener(event: 'data', listener: (data: T) => void): this; + removeListener(event: 'data', listener: (data: T) => void): this; +} + +export class ResourceStream extends Transform implements ResourceEvents { + _ended: boolean; + _maxApiCalls: number; + _nextQuery: {} | null; + _otherArgs: unknown[]; + _reading: boolean; + _requestFn: Function; + _requestsMade: number; + _resultsToSend: number; + constructor(args: ParsedArguments, requestFn: Function) { + const options = Object.assign({objectMode: true}, args.streamOptions); + super(options); + + this._ended = false; + this._maxApiCalls = args.maxApiCalls === -1 ? Infinity : args.maxApiCalls!; + this._nextQuery = args.query!; + this._reading = false; + this._requestFn = requestFn; + this._requestsMade = 0; + this._resultsToSend = args.maxResults === -1 ? Infinity : args.maxResults!; + this._otherArgs = []; + } + /* eslint-disable @typescript-eslint/no-explicit-any */ + end( + ...args: any[] + ): ReturnType extends Writable ? this : void { + this._ended = true; + return super.end(...args); + } + _read() { + if (this._reading) { + return; + } + + this._reading = true; + + // Wrap in a try/catch to catch input linting errors, e.g. + // an invalid BigQuery query. These errors are thrown in an + // async fashion, which makes them un-catchable by the user. + try { + this._requestFn( + this._nextQuery, + ( + err: Error | null, + results: T[], + nextQuery: {} | null, + ...otherArgs: [] + ) => { + if (err) { + this.destroy(err); + return; + } + + this._otherArgs = otherArgs; + this._nextQuery = nextQuery; + + if (this._resultsToSend !== Infinity) { + results = results.splice(0, this._resultsToSend); + this._resultsToSend -= results.length; + } + + let more = true; + + for (const result of results) { + if (this._ended) { + break; + } + more = this.push(result); + } + + const isFinished = !this._nextQuery || this._resultsToSend < 1; + const madeMaxCalls = ++this._requestsMade >= this._maxApiCalls; + + if (isFinished || madeMaxCalls) { + this.end(); + } + + if (more && !this._ended) { + setImmediate(() => this._read()); + } + + this._reading = false; + }, + ); + } catch (e) { + this.destroy(e as Error); + } + } +} diff --git a/core/paginator/system-test/system.ts b/core/paginator/system-test/system.ts new file mode 100644 index 00000000000..05638eda311 --- /dev/null +++ b/core/paginator/system-test/system.ts @@ -0,0 +1,15 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +console.warn('no system tests available 👻'); diff --git a/core/paginator/test/index.ts b/core/paginator/test/index.ts new file mode 100644 index 00000000000..8c790f293bc --- /dev/null +++ b/core/paginator/test/index.ts @@ -0,0 +1,551 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as assert from 'assert'; +import {describe, it, beforeEach, afterEach} from 'mocha'; +import * as proxyquire from 'proxyquire'; +import * as sinon from 'sinon'; +import {PassThrough, Transform} from 'stream'; +import * as uuid from 'uuid'; +import * as P from '../src'; +import {paginator, ParsedArguments} from '../src'; + +const util = { + noop: () => { + // do nothing + }, +}; + +class FakeResourceStream extends Transform { + calledWith: IArguments; + constructor() { + super({objectMode: true}); + /* eslint-disable-next-line prefer-rest-params */ + this.calledWith = arguments; + } +} + +const p = proxyquire('../src', { + './resource-stream': {ResourceStream: FakeResourceStream}, +}) as typeof P; + +const sandbox = sinon.createSandbox(); + +// eslint-disable-next-line no-undef +afterEach(() => { + sandbox.restore(); +}); + +/* eslint-disable @typescript-eslint/no-explicit-any */ +function createFakeStream() { + return new PassThrough({objectMode: true}) as P.ResourceStream; +} + +describe('paginator', () => { + const UUID = uuid.v1(); + function FakeClass() { + // do nothing + } + + beforeEach(() => { + FakeClass.prototype.methodToExtend = () => { + return UUID; + }; + delete FakeClass.prototype.methodToExtend_; + }); + + describe('extend', () => { + it('should overwrite a method on a class', () => { + const originalMethod = FakeClass.prototype.methodToExtend; + paginator.extend(FakeClass, 'methodToExtend'); + const overwrittenMethod = FakeClass.prototype.methodToExtend; + assert.notStrictEqual(originalMethod, overwrittenMethod); + }); + + it('should store the original method as a private member', () => { + const originalMethod = FakeClass.prototype.methodToExtend; + paginator.extend(FakeClass, 'methodToExtend'); + assert.strictEqual(originalMethod, FakeClass.prototype.methodToExtend_); + }); + + it('should accept an array or string method names', () => { + const originalMethod = FakeClass.prototype.methodToExtend; + const anotherMethod = FakeClass.prototype.anotherMethodToExtend; + const methodsToExtend = ['methodToExtend', 'anotherMethodToExtend']; + paginator.extend(FakeClass, methodsToExtend); + assert.notStrictEqual(originalMethod, FakeClass.prototype.methodToExtend); + assert.notStrictEqual( + anotherMethod, + FakeClass.prototype.anotherMethodToExtend, + ); + }); + + it('should parse the arguments', done => { + sandbox.stub(paginator, 'parseArguments_').callsFake(args => { + assert.deepStrictEqual([].slice.call(args), [1, 2, 3]); + done(); + return args as ParsedArguments; + }); + sandbox.stub(paginator, 'run_').callsFake(util.noop); + paginator.extend(FakeClass, 'methodToExtend'); + FakeClass.prototype.methodToExtend(1, 2, 3); + }); + + it('should call router when the original method is called', done => { + const expectedReturnValue = FakeClass.prototype.methodToExtend(); + const parsedArguments = {a: 'b', c: 'd'} as ParsedArguments; + + sandbox.stub(paginator, 'parseArguments_').returns(parsedArguments); + sandbox.stub(paginator, 'run_').callsFake((args, originalMethod) => { + assert.strictEqual(args, parsedArguments); + assert.strictEqual(originalMethod(), expectedReturnValue); + done(); + }); + + paginator.extend(FakeClass, 'methodToExtend'); + FakeClass.prototype.methodToExtend(); + }); + + it('should maintain `this` context', done => { + FakeClass.prototype.methodToExtend = function () { + return this.uuid; + }; + + /* eslint-disable @typescript-eslint/no-explicit-any */ + const cls = new (FakeClass as any)(); + cls.uuid = uuid.v1(); + + sandbox.stub(paginator, 'run_').callsFake((_, originalMethod) => { + assert.strictEqual(originalMethod(), cls.uuid); + done(); + }); + + paginator.extend(FakeClass, 'methodToExtend'); + cls.methodToExtend(); + }); + + it('should return what the router returns', () => { + const uniqueValue = 234; + sandbox.stub(paginator, 'run_').callsFake(() => { + return uniqueValue; + }); + paginator.extend(FakeClass, 'methodToExtend'); + assert.strictEqual(FakeClass.prototype.methodToExtend(), uniqueValue); + }); + }); + + describe('streamify', () => { + beforeEach(() => { + FakeClass.prototype.streamMethod = paginator.streamify('methodToExtend'); + }); + + it('should return a function', () => { + const fakeStreamMethod = FakeClass.prototype.streamMethod; + assert.strictEqual(typeof fakeStreamMethod, 'function'); + }); + + it('should parse the arguments', done => { + const fakeArgs = [1, 2, 3]; + + sandbox.stub(paginator, 'parseArguments_').callsFake(args => { + assert.deepStrictEqual(fakeArgs, [].slice.call(args)); + done(); + return args as ParsedArguments; + }); + sandbox.stub(paginator, 'runAsStream_').callsFake(createFakeStream); + FakeClass.prototype.streamMethod(...fakeArgs); + }); + + it('should run the method as a stream', done => { + const parsedArguments = {a: 'b', c: 'd'} as ParsedArguments; + sandbox.stub(paginator, 'parseArguments_').callsFake(() => { + return parsedArguments; + }); + sandbox.stub(paginator, 'runAsStream_').callsFake((args, callback) => { + assert.strictEqual(args, parsedArguments); + assert.strictEqual(callback(), UUID); + setImmediate(done); + return createFakeStream(); + }); + + FakeClass.prototype.streamMethod(); + }); + + it('should apply the proper context', done => { + const parsedArguments = {a: 'b', c: 'd'} as ParsedArguments; + FakeClass.prototype.methodToExtend = function () { + return this; + }; + sandbox.stub(paginator, 'parseArguments_').callsFake(() => { + return parsedArguments; + }); + sandbox.stub(paginator, 'runAsStream_').callsFake((_, callback) => { + assert.strictEqual(callback(), FakeClass.prototype); + setImmediate(done); + return createFakeStream(); + }); + FakeClass.prototype.streamMethod(); + }); + + it('should check for a private member', done => { + const parsedArguments = {a: 'b', c: 'd'} as ParsedArguments; + const fakeValue = 123; + + FakeClass.prototype.methodToExtend_ = () => { + return fakeValue; + }; + sandbox.stub(paginator, 'parseArguments_').callsFake(() => { + return parsedArguments; + }); + sandbox.stub(paginator, 'runAsStream_').callsFake((_, callback) => { + assert.strictEqual(callback(), fakeValue); + setImmediate(done); + return createFakeStream(); + }); + FakeClass.prototype.streamMethod(); + }); + + it('should return a stream', () => { + const fakeStream = createFakeStream(); + sandbox.stub(paginator, 'parseArguments_').returns({}); + sandbox.stub(paginator, 'runAsStream_').returns(fakeStream); + const stream = FakeClass.prototype.streamMethod(); + assert.strictEqual(fakeStream, stream); + }); + }); + + describe('parseArguments_', () => { + it('should set defaults', () => { + const parsedArguments = paginator.parseArguments_([]); + + assert.strictEqual(Object.keys(parsedArguments.query!).length, 0); + assert.strictEqual(parsedArguments.autoPaginate, true); + assert.strictEqual(parsedArguments.maxApiCalls, -1); + assert.strictEqual(parsedArguments.maxResults, -1); + assert.strictEqual(parsedArguments.callback, undefined); + }); + + it('should detect a callback if first argument is a function', () => { + const args = [util.noop]; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.callback, args[0]); + }); + + it('should use any other first argument as query', () => { + const args = ['string']; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.query, args[0]); + }); + + it('should not make an undefined value the query', () => { + const args = [undefined, util.noop]; + const parsedArguments = paginator.parseArguments_(args); + + assert.deepStrictEqual(parsedArguments.query, {}); + }); + + it('should detect a callback if last argument is a function', () => { + const args = ['string', util.noop]; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.callback, args[1]); + }); + + it('should not assign a callback if a fn is not provided', () => { + const args = ['string']; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.callback, undefined); + }); + + it('should set maxApiCalls from query.maxApiCalls', () => { + const args = [{maxApiCalls: 10}]; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.maxApiCalls, args[0].maxApiCalls); + assert.strictEqual(parsedArguments.query!.maxApiCalls, undefined); + }); + + it('should set maxResults from query.maxResults', () => { + const args = [{maxResults: 10}]; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.maxResults, args[0].maxResults); + }); + + it('should set maxResults from query.pageSize', () => { + const args = [{pageSize: 10}]; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.maxResults, args[0].pageSize); + }); + + it('should set autoPaginate: false if there is a maxResults', () => { + const args = [{maxResults: 10}, util.noop]; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.autoPaginate, false); + }); + + it('should set autoPaginate: false query.autoPaginate', () => { + const args = [{autoPaginate: false}, util.noop]; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.autoPaginate, false); + }); + + it('should parse streamOptions', () => { + const args = [{maxResults: 10, highWaterMark: 8}]; + const parsedArguments = paginator.parseArguments_(args); + + assert.strictEqual(parsedArguments.maxResults, 10); + assert.deepStrictEqual(parsedArguments.streamOptions, { + highWaterMark: 8, + }); + }); + }); + + describe('run_', () => { + describe('autoPaginate', () => { + describe('originalmethod is callback based', () => { + it('should call runAsStream_ when autoPaginate:true', done => { + const parsedArguments = { + autoPaginate: true, + callback: util.noop, + }; + + sandbox + .stub(paginator, 'runAsStream_') + .callsFake((args, originalMethod) => { + assert.strictEqual(args, parsedArguments); + originalMethod(); + return createFakeStream(); + }); + + paginator.run_(parsedArguments, done); + }); + + it('should execute callback on error', done => { + const error = new Error('Error.'); + + const parsedArguments = { + autoPaginate: true, + callback(err: Error) { + assert.strictEqual(err, error); + done(); + }, + }; + + sandbox.stub(paginator, 'runAsStream_').callsFake(() => { + const stream = createFakeStream(); + setImmediate(() => { + stream.emit('error', error); + }); + return stream; + }); + + paginator.run_(parsedArguments, util.noop); + }); + + it('should return all results on end', done => { + const results = [{a: 1}, {b: 2}, {c: 3}]; + + const parsedArguments = { + autoPaginate: true, + callback(err: Error, results_: {}) { + assert.deepStrictEqual(results_, results); + done(); + }, + }; + + sandbox.stub(paginator, 'runAsStream_').callsFake(() => { + const stream = createFakeStream(); + setImmediate(() => { + results.forEach(result => stream.push(result)); + stream.push(null); + }); + return stream; + }); + + paginator.run_(parsedArguments, util.noop); + }); + + it('should return all results and extra args', done => { + const results = [{a: 1}, {b: 2}, {c: 3}]; + const args: any[] = [{msg: 'OK'}, 10]; + + const parsedArguments = { + autoPaginate: true, + callback( + err: Error, + results_: {}, + query: {}, + fakeRes: {}, + anotherArg: number, + ) { + assert.deepStrictEqual(results_, results); + assert.deepStrictEqual(query, undefined); + assert.deepStrictEqual(fakeRes, {msg: 'OK'}); + assert.deepStrictEqual(anotherArg, 10); + done(); + }, + }; + + sandbox.stub(paginator, 'runAsStream_').callsFake(() => { + const stream = createFakeStream(); + setImmediate(() => { + results.forEach(result => stream.push(result)); + stream.push(null); + stream._otherArgs = args; + }); + return stream; + }); + + paginator.run_(parsedArguments, util.noop); + }); + }); + + describe('original method is promise based', () => { + const parsedArguments = { + autoPaginate: true, + }; + it('should call runAsStream_ when autoPaginate:true', done => { + sandbox + .stub(paginator, 'runAsStream_') + .callsFake((args, originalMethod) => { + assert.strictEqual(args, parsedArguments); + originalMethod(); + return createFakeStream(); + }); + + paginator.run_(parsedArguments, done); + }); + + it('should reject a promise on error', () => { + const error = new Error('Error.'); + + sandbox.stub(paginator, 'runAsStream_').callsFake(() => { + const stream = createFakeStream(); + setImmediate(() => { + stream.emit('error', error); + }); + return stream; + }); + + paginator + .run_(parsedArguments, util.noop) + .then(util.noop, (err: Error) => assert.strictEqual(err, error)); + }); + + it('should resolve with all results on end', () => { + const results = [{a: 1}, {b: 2}, {c: 3}]; + + sandbox.stub(paginator, 'runAsStream_').callsFake(() => { + const stream = createFakeStream(); + setImmediate(() => { + results.forEach(result => stream.push(result)); + stream.push(null); + }); + return stream; + }); + + paginator + .run_(parsedArguments, util.noop) + .then(([results_]: [1]) => + assert.deepStrictEqual(results_, results), + ); + }); + + it('should resolve with all results and extra args', done => { + const results = [{a: 1}, {b: 2}, {c: 3}]; + const args: any[] = [{msg: 'OK'}, 10]; + + sandbox.stub(paginator, 'runAsStream_').callsFake(() => { + const stream = createFakeStream(); + setImmediate(() => { + results.forEach(result => stream.push(result)); + stream.push(null); + stream._otherArgs = args; + }); + return stream; + }); + + paginator + .run_(parsedArguments, util.noop) + .then(([results_, query_, fakeRes, anotherArg]: unknown[]) => { + assert.deepStrictEqual(results_, results); + assert.deepStrictEqual(query_, undefined); + assert.deepEqual(fakeRes, {msg: 'OK'}); + assert.deepEqual(anotherArg, 10); + done(); + }); + }); + }); + }); + + describe('manual pagination', () => { + describe('originalmethod is callback based', () => { + it('should recognize autoPaginate: false', done => { + const parsedArguments = { + autoPaginate: false, + query: { + a: 'b', + c: 'd', + }, + callback: done, + } as ParsedArguments; + sandbox.stub(paginator, 'runAsStream_').callsFake(createFakeStream); + paginator.run_(parsedArguments, (query: {}, callback: () => void) => { + assert.deepStrictEqual(query, parsedArguments.query); + callback(); + }); + }); + }); + + describe('original method is promise based', () => { + it('should recognize autoPaginate: false', () => { + const parsedArguments = { + autoPaginate: false, + query: { + a: 'b', + c: 'd', + }, + } as ParsedArguments; + sandbox.stub(paginator, 'runAsStream_').callsFake(createFakeStream); + paginator.run_(parsedArguments, (query: {}) => { + assert.deepStrictEqual(query, parsedArguments.query); + }); + }); + }); + }); + + describe('runAsStream_', () => { + it('should create a resource stream', () => { + const fakeArgs = {}; + const fakeFn = sandbox.spy(); + const stream = p.paginator.runAsStream_( + fakeArgs, + fakeFn, + ) as unknown as FakeResourceStream; + + assert(stream instanceof FakeResourceStream); + const [args, requestFn] = stream.calledWith; + assert.strictEqual(args, fakeArgs); + assert.strictEqual(requestFn, fakeFn); + }); + }); + }); +}); diff --git a/core/paginator/test/resource-stream.ts b/core/paginator/test/resource-stream.ts new file mode 100644 index 00000000000..7e9eb4b5b8a --- /dev/null +++ b/core/paginator/test/resource-stream.ts @@ -0,0 +1,321 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as assert from 'assert'; +import {describe, it, beforeEach, afterEach} from 'mocha'; +import * as sinon from 'sinon'; +import {Transform} from 'stream'; + +import {ResourceStream} from '../src/resource-stream'; + +describe('ResourceStream', () => { + const sandbox = sinon.createSandbox(); + + const config = { + maxApiCalls: -1, + maxResults: -1, + query: {}, + }; + + let requestSpy: sinon.SinonSpy; + let stream: ResourceStream<{}>; + + beforeEach(() => { + requestSpy = sandbox.spy(); + stream = new ResourceStream(config, requestSpy); + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('instantiation', () => { + it('should pass the streamingOptions to the constructor', () => { + const highWaterMark = 1; + const options = { + streamOptions: {highWaterMark}, + }, + stream = new ResourceStream(options, requestSpy); + assert.strictEqual(stream.readableHighWaterMark, highWaterMark); + }); + + it('should set ended to false', () => { + assert.strictEqual(stream._ended, false); + }); + + it('should set reading to false', () => { + assert.strictEqual(stream._reading, false); + }); + + it('should set requestsMade to 0', () => { + assert.strictEqual(stream._requestsMade, 0); + }); + + it('should localize the first query', () => { + assert.strictEqual(stream._nextQuery, config.query); + }); + + it('should localize the request function', () => { + assert.strictEqual(stream._requestFn, requestSpy); + }); + + describe('maxApiCalls', () => { + it('should localize maxApiCalls', () => { + const maxApiCalls = 100; + stream = new ResourceStream({maxApiCalls}, requestSpy); + assert.strictEqual(stream._maxApiCalls, maxApiCalls); + }); + + it('should set it to Infinity if not specified', () => { + assert.strictEqual(stream._maxApiCalls, Infinity); + }); + }); + + describe('resultsToSend', () => { + it('should localize maxResults as resultsToSend', () => { + const maxResults = 100; + stream = new ResourceStream({maxResults}, requestSpy); + assert.strictEqual(stream._resultsToSend, maxResults); + }); + + it('should set it to Infinity if not specified', () => { + assert.strictEqual(stream._resultsToSend, Infinity); + }); + }); + }); + + describe('end', () => { + it('should set ended to true', () => { + stream.end(); + assert.strictEqual(stream._ended, true); + }); + + it('should call through to super.end', () => { + const stub = sandbox.stub(Transform.prototype, 'end'); + + stream.end(); + assert.strictEqual(stub.callCount, 1); + }); + }); + + describe('_read', () => { + it('should set reading to true', () => { + stream._read(); + assert.strictEqual(stream._reading, true); + }); + + it('should noop if already reading', () => { + stream._read(); + stream._read(); + + assert.strictEqual(requestSpy.callCount, 1); + }); + + it('should pass in the query options', () => { + stream._read(); + + const query = requestSpy.lastCall.args[0]; + assert.strictEqual(query, config.query); + }); + + it('should destroy the stream if an error occurs', () => { + const fakeError = new Error('err'); + const stub = sandbox.stub(stream, 'destroy').withArgs(fakeError); + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(fakeError); + + assert.strictEqual(stub.callCount, 1); + }); + + it('should cache the next query', () => { + const fakeQuery = {}; + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, [], fakeQuery); + + assert.strictEqual(stream._nextQuery, fakeQuery); + }); + + it('should cache the rest of the callback arguments', () => { + const fakeRes = {status: 'OK'}; + const anotherArg = 10; + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, [], {}, fakeRes, anotherArg); + + assert.deepStrictEqual(stream._otherArgs, [fakeRes, anotherArg]); + }); + + it('should adjust the results to send counter', () => { + const maxResults = 100; + const results = [{}, {}]; + const expected = maxResults - results.length; + + stream = new ResourceStream({maxResults}, requestSpy); + stream._read(); + + const callback = requestSpy.lastCall.args[1]; + callback(null, results); + + assert.strictEqual(stream._resultsToSend, expected); + }); + + it('should push in all the results', () => { + // tslint:disable-next-line ban + const results = Array(20).fill({}); + const stub = sandbox.stub(stream, 'push'); + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, results, {}); + + assert.strictEqual(stub.callCount, results.length); + + results.forEach((result, i) => { + const pushed = stub.getCall(i).args[0]; + assert.strictEqual(pushed, result); + }); + }); + + it('should stop pushing results if the stream is ended', () => { + // tslint:disable-next-line ban + const results = Array(20).fill({}); + + stream.on('data', () => stream.end()); + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, results, {}); + + assert.strictEqual(requestSpy.callCount, 1); + }); + + it('should end the stream if there is no next query', () => { + const stub = sandbox.stub(stream, 'end'); + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, []); + + assert.strictEqual(stub.callCount, 1); + }); + + it('should end the stream if max results is hit', () => { + const maxResults = 10; + // tslint:disable-next-line ban + const results = Array(maxResults).fill({}); + stream = new ResourceStream({maxResults}, requestSpy); + const stub = sandbox.stub(stream, 'end'); + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, results, {}); + + assert.strictEqual(stub.callCount, 1); + }); + + it('should end the stream if max api calls is hit', () => { + const maxApiCalls = 1; + stream = new ResourceStream({maxApiCalls}, requestSpy); + const stub = sandbox.stub(stream, 'end'); + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, [], {}); + + assert.strictEqual(stub.callCount, 1); + }); + + it('should stop reading if the buffer is full', () => { + const clock = sandbox.useFakeTimers(); + + // tslint:disable-next-line ban + const results = Array(stream.readableHighWaterMark).fill({}); + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, results, {}); + + const stub = sandbox.stub(stream, '_read'); + clock.runAll(); + + assert.strictEqual(stub.callCount, 0); + }); + + it('should stop reading if the stream ended', () => { + const clock = sandbox.useFakeTimers(); + + stream.on('data', () => stream.end()); + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, [{}], {}); + + const stub = sandbox.stub(stream, '_read'); + clock.runAll(); + + assert.strictEqual(stub.callCount, 0); + }); + + it('should keep reading if not full/ended', () => { + // sinon adds a timer to `nextTick` by default beginning in v19 + // manually specifying the timers like this replicates the behavior pre v19 + const clock = sandbox.useFakeTimers({ + toFake: [ + 'setTimeout', + 'clearTimeout', + 'setInterval', + 'clearInterval', + 'Date', + 'setImmediate', + 'clearImmediate', + 'hrtime', + 'performance', + ], + }); + + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, [{}], {}); + + const stub = sandbox.stub(stream, '_read'); + clock.runAll(); + + assert.strictEqual(stub.callCount, 1); + }); + + it('should set reading to false inbetween reads', () => { + stream._read(); + const callback = requestSpy.lastCall.args[1]; + callback(null, [{}], {}); + + assert.strictEqual(stream._reading, false); + }); + + it('should destroy the stream if the request method throws', done => { + const error = new Error('Error.'); + stream._requestFn = () => { + throw error; + }; + stream.on('error', err => { + assert.strictEqual(err, error); + done(); + }); + stream._read(); + }); + }); +}); diff --git a/core/paginator/tsconfig.json b/core/paginator/tsconfig.json new file mode 100644 index 00000000000..8b14ad98550 --- /dev/null +++ b/core/paginator/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "./node_modules/gts/tsconfig-google.json", + "compilerOptions": { + "lib": ["es2018", "dom"], + "rootDir": ".", + "outDir": "build" + }, + "include": [ + "src/*.ts", + "test/*.ts", + "system-test/*.ts" + ] +} diff --git a/release-please-submodules.json b/release-please-submodules.json index b132d34c7d6..b7cf32f47e4 100644 --- a/release-please-submodules.json +++ b/release-please-submodules.json @@ -2,6 +2,7 @@ "commit-batch-size": 1, "include-component-in-tag": true, "packages": { + "core/paginator": {}, "handwritten/bigquery": { "component": "bigquery" }, @@ -14,12 +15,12 @@ "handwritten/datastore": { "component": "datastore" }, - "handwritten/firestore": { - "component": "firestore" - }, "handwritten/error-reporting": { "component": "error-reporting" }, + "handwritten/firestore": { + "component": "firestore" + }, "handwritten/logging-bunyan": { "component": "logging-bunyan" },