From a3e5035bf55f9fb94c624e3969e8dc79404cdc11 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 9 Feb 2026 13:21:36 +0000 Subject: [PATCH 1/7] chore: Migrate gsutil usage to gcloud storage --- .../generate_sitemap/generate_and_upload.sh | 17 +++++++++++++++++ vulnfeeds/cmd/combine-to-osv/README.md | 4 ++-- .../bulk-converter/run-cvelist-converter.sh | 8 ++++---- .../nvd-cve-osv/run_cve_to_osv_generation.sh | 8 ++++---- .../mirrors/cpe-repo-gen/cpe-repo-gen_map.sh | 4 ++-- .../debian-copyright-mirror.sh | 2 +- vulnfeeds/pypi/cloudbuild.yaml | 4 ++-- vulnfeeds/tools/debian/run_convert_debian.sh | 6 +++--- .../tools/debian/run_first_package_finder.sh | 2 +- 9 files changed, 36 insertions(+), 19 deletions(-) create mode 100755 gcp/workers/cron/generate_sitemap/generate_and_upload.sh diff --git a/gcp/workers/cron/generate_sitemap/generate_and_upload.sh b/gcp/workers/cron/generate_sitemap/generate_and_upload.sh new file mode 100755 index 00000000000..4c7a18eaddf --- /dev/null +++ b/gcp/workers/cron/generate_sitemap/generate_and_upload.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") + +SITEMAP_OUTPUT="sitemap_output/" +OUTPUT_BUCKET="${OUTPUT_GCS_BUCKET:=test-osv-dev-sitemap}" +BASE_URL_PATH="${BASE_URL:=https://test.osv.dev}" + +echo "Begin sitemap generation for $BASE_URL_PATH" + +"$SCRIPT_PATH/generate_sitemap.py" --base_url $BASE_URL_PATH + +echo "Begin Syncing with cloud to $OUTPUT_BUCKET" + +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects $SITEMAP_OUTPUT "gs://$OUTPUT_BUCKET/" diff --git a/vulnfeeds/cmd/combine-to-osv/README.md b/vulnfeeds/cmd/combine-to-osv/README.md index c5d252186f9..7dd4eb3c90c 100644 --- a/vulnfeeds/cmd/combine-to-osv/README.md +++ b/vulnfeeds/cmd/combine-to-osv/README.md @@ -63,6 +63,6 @@ This statically overrides the record generated, meaning any and all of the input #### Procedure -1. `gsutil cp gs://cve-osv-conversion/osv-output/CVE-YYYY-NNNN.json` +1. `gcloud storage cp gs://cve-osv-conversion/osv-output/CVE-YYYY-NNNN.json` 2. manually edit the file -3. `gsutil cp gs://cve-osv-conversion/osv-output-overrides/CVE-YYYY-NNNN.json` +3. `gcloud storage cp gs://cve-osv-conversion/osv-output-overrides/CVE-YYYY-NNNN.json` diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh index 41e182d191a..4450f834452 100755 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh @@ -64,19 +64,19 @@ find "${LOCAL_OUT_DIR}/${OSV_OUTPUT_PATH}" -type f -name \*.json \ # Copy (and remove any missing) results to GCS bucket, with some sanity # checking. -objs_present=$(gsutil ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) -objs_deleted=$(gsutil -m rsync -c -n -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) +objs_present=$(gcloud storage ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) +objs_deleted=$(gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) threshold=$(echo "scale=2; ${objs_present} * (${SAFETY_THRESHOLD_PCT:-2} / 100)" | bc) # # Bash can't deal with floats if (( $(echo "${objs_deleted} > ${threshold}" | bc -l) )); then echo "Aborting. Unexpectedly high (${objs_deleted}) number of CVE records would be deleted!" >> /dev/stderr - gsutil -m rsync -c -n -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr + gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr exit 1 fi echo "Copying CVEList records successfully converted to GCS bucket" -gsutil -q -m rsync -c -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" echo "Conversion run complete" diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh index d4d89806adf..7fc792b41fc 100755 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh @@ -65,19 +65,19 @@ done # Copy (and remove any missing) results to GCS bucket, with some sanity # checking. -objs_present=$(gsutil ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) -objs_deleted=$(gsutil -m rsync -c -n -d "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) +objs_present=$(gcloud storage ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) +objs_deleted=$(gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) threshold=$(echo "scale=2; ${objs_present} * (${SAFETY_THRESHOLD_PCT:-2} / 100)" | bc) # Bash can't deal with floats if (( $(echo "${objs_deleted} > ${threshold}" | bc -l) )); then echo "Warning. Unexpectedly high (${objs_deleted}) number of CVE records would be deleted!" >> /dev/stderr - gsutil -m rsync -c -n -d "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr + gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr # TODO: add back in once nvd-mirror issue fixed: exit 1 fi echo "Copying NVD CVE records successfully converted to GCS bucket" -gsutil -q -m rsync -c "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" +gcloud -q storage rsync --checksums-only "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" echo "Conversion run complete" diff --git a/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh index 0779c28930a..4d4427e0965 100755 --- a/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh +++ b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh @@ -38,7 +38,7 @@ unzip -o "${WORK_DIR}/cpe.zip" -d "${CPE_JSON_DIR}" MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="" if [[ -n "${DEBIAN_COPYRIGHT_GCS_PATH}" ]]; then - gsutil ${BE_VERBOSE="-q"} cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}" + gcloud ${BE_VERBOSE="-q"} storage cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}" tar -C "${WORK_DIR}" -xf "${WORK_DIR}/$(basename ${DEBIAN_COPYRIGHT_GCS_PATH})" MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="--debian-metadata-path ${WORK_DIR}/metadata.ftp-master.debian.org" fi @@ -49,4 +49,4 @@ fi --output-dir="${WORK_DIR}" -gsutil ${BE_VERBOSE="-q"} cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}" +gcloud ${BE_VERBOSE="-q"} storage cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}" diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh index bbc6d0c8fc3..d98db88a422 100755 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh @@ -33,4 +33,4 @@ python debian-copyright-mirror.py "${WORK_DIR}/metadata.ftp-master.debian.org/ch tar -C "${WORK_DIR}" -cf "${WORK_DIR}/$(basename ${GCS_PATH})" . -gsutil ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" +gcloud storage ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" diff --git a/vulnfeeds/pypi/cloudbuild.yaml b/vulnfeeds/pypi/cloudbuild.yaml index e07547058d2..7b1ff81d8af 100644 --- a/vulnfeeds/pypi/cloudbuild.yaml +++ b/vulnfeeds/pypi/cloudbuild.yaml @@ -25,5 +25,5 @@ steps: args: - -c - bq query --max_rows=10000000 --format=json --nouse_legacy_sql < vulnfeeds/pypi/pypi_versions.sql > pypi_versions.json -- name: gcr.io/cloud-builders/gsutil - args: ['cp', 'pypi_links.json', 'pypi_versions.json', 'gs://pypa-advisory-db/triage/'] +- name: gcr.io/cloud-builders/gcloud + args: ['storage', 'cp', 'pypi_links.json', 'pypi_versions.json', 'gs://pypa-advisory-db/triage/'] diff --git a/vulnfeeds/tools/debian/run_convert_debian.sh b/vulnfeeds/tools/debian/run_convert_debian.sh index 8a44235a31d..909da3cfed3 100755 --- a/vulnfeeds/tools/debian/run_convert_debian.sh +++ b/vulnfeeds/tools/debian/run_convert_debian.sh @@ -44,7 +44,7 @@ poetry run python3 convert_debian.py --adv_type=DTSA -o $OSV_DTSA_OUT $WEBWML_PA popd echo "Begin Syncing with cloud" -gsutil -m rsync -c -d $OSV_DSA_OUT gs://$OUTPUT_BUCKET/dsa-osv -gsutil -m rsync -c -d "$OSV_DLA_OUT" "gs://${OUTPUT_BUCKET}/dla-osv" -gsutil -m rsync -c -d "$OSV_DTSA_OUT" "gs://${OUTPUT_BUCKET}/dtsa-osv" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects $OSV_DSA_OUT gs://$OUTPUT_BUCKET/dsa-osv +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects "$OSV_DLA_OUT" "gs://${OUTPUT_BUCKET}/dla-osv" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects "$OSV_DTSA_OUT" "gs://${OUTPUT_BUCKET}/dtsa-osv" echo "Successfully synced with cloud" \ No newline at end of file diff --git a/vulnfeeds/tools/debian/run_first_package_finder.sh b/vulnfeeds/tools/debian/run_first_package_finder.sh index 0dfebc5747d..b5062ecda69 100755 --- a/vulnfeeds/tools/debian/run_first_package_finder.sh +++ b/vulnfeeds/tools/debian/run_first_package_finder.sh @@ -25,7 +25,7 @@ echo "Finding first packages" poetry run python3 first_package_finder.py echo "Syncing with cloud first_package_output ${OUTPUT_BUCKET}" -gsutil -q -m rsync -c -d 'first_package_output' "gs://${OUTPUT_BUCKET}/first_package_output" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects 'first_package_output' "gs://${OUTPUT_BUCKET}/first_package_output" echo "Successfully synced with cloud" popd \ No newline at end of file From 20c68d2e15f5450b6857e406feb8a94ad7f6eeac Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 19 Mar 2026 09:52:04 +0530 Subject: [PATCH 2/7] Update vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh Co-authored-by: Jess Lowe <86962800+jess-lowe@users.noreply.github.com> --- .../cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh index 7fc792b41fc..9a77e11b49c 100755 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh @@ -78,6 +78,6 @@ if (( $(echo "${objs_deleted} > ${threshold}" | bc -l) )); then fi echo "Copying NVD CVE records successfully converted to GCS bucket" -gcloud -q storage rsync --checksums-only "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" +gcloud storage rsync --quiet --checksums-only "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" echo "Conversion run complete" From 8481e0dd3a8d9f6de99fac8870bc620755ecc393 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 19 Mar 2026 09:52:17 +0530 Subject: [PATCH 3/7] Update vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh Co-authored-by: Jess Lowe <86962800+jess-lowe@users.noreply.github.com> --- vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh index 4d4427e0965..b93b86c2531 100755 --- a/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh +++ b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh @@ -38,7 +38,7 @@ unzip -o "${WORK_DIR}/cpe.zip" -d "${CPE_JSON_DIR}" MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="" if [[ -n "${DEBIAN_COPYRIGHT_GCS_PATH}" ]]; then - gcloud ${BE_VERBOSE="-q"} storage cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}" + gcloud storage cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}" ${BE_VERBOSE="--quiet"} tar -C "${WORK_DIR}" -xf "${WORK_DIR}/$(basename ${DEBIAN_COPYRIGHT_GCS_PATH})" MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="--debian-metadata-path ${WORK_DIR}/metadata.ftp-master.debian.org" fi From 7ed1aa4078269805e2bd23e0fee31ab81177f820 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 19 Mar 2026 09:52:33 +0530 Subject: [PATCH 4/7] Update vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh Co-authored-by: Jess Lowe <86962800+jess-lowe@users.noreply.github.com> --- vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh index b93b86c2531..835010c49f3 100755 --- a/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh +++ b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh @@ -49,4 +49,4 @@ fi --output-dir="${WORK_DIR}" -gcloud ${BE_VERBOSE="-q"} storage cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}" +gcloud storage cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}" ${BE_VERBOSE="--quiet"} From 32a163b40513c1dd04ba81647c9969b3eeaeb0d8 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 19 Mar 2026 09:52:51 +0530 Subject: [PATCH 5/7] Update vulnfeeds/pypi/cloudbuild.yaml Co-authored-by: Jess Lowe <86962800+jess-lowe@users.noreply.github.com> --- vulnfeeds/pypi/cloudbuild.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vulnfeeds/pypi/cloudbuild.yaml b/vulnfeeds/pypi/cloudbuild.yaml index 7b1ff81d8af..ef4eb12fdac 100644 --- a/vulnfeeds/pypi/cloudbuild.yaml +++ b/vulnfeeds/pypi/cloudbuild.yaml @@ -25,5 +25,6 @@ steps: args: - -c - bq query --max_rows=10000000 --format=json --nouse_legacy_sql < vulnfeeds/pypi/pypi_versions.sql > pypi_versions.json -- name: gcr.io/cloud-builders/gcloud +- name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' + entrypoint: 'gcloud' args: ['storage', 'cp', 'pypi_links.json', 'pypi_versions.json', 'gs://pypa-advisory-db/triage/'] From 224421f6506aa94f47782ce555df2441016589ba Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 19 Mar 2026 09:53:02 +0530 Subject: [PATCH 6/7] Update vulnfeeds/tools/debian/run_first_package_finder.sh Co-authored-by: Jess Lowe <86962800+jess-lowe@users.noreply.github.com> --- vulnfeeds/tools/debian/run_first_package_finder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnfeeds/tools/debian/run_first_package_finder.sh b/vulnfeeds/tools/debian/run_first_package_finder.sh index b5062ecda69..b0e4d94996f 100755 --- a/vulnfeeds/tools/debian/run_first_package_finder.sh +++ b/vulnfeeds/tools/debian/run_first_package_finder.sh @@ -25,7 +25,7 @@ echo "Finding first packages" poetry run python3 first_package_finder.py echo "Syncing with cloud first_package_output ${OUTPUT_BUCKET}" -gcloud storage rsync --checksums-only --delete-unmatched-destination-objects 'first_package_output' "gs://${OUTPUT_BUCKET}/first_package_output" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects --quiet 'first_package_output' "gs://${OUTPUT_BUCKET}/first_package_output" echo "Successfully synced with cloud" popd \ No newline at end of file From 484392a2c0279af5d3b8e097a622cccd1c65d0d8 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Thu, 19 Mar 2026 05:11:43 +0000 Subject: [PATCH 7/7] del unused script --- .../generate_sitemap/generate_and_upload.sh | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100755 gcp/workers/cron/generate_sitemap/generate_and_upload.sh diff --git a/gcp/workers/cron/generate_sitemap/generate_and_upload.sh b/gcp/workers/cron/generate_sitemap/generate_and_upload.sh deleted file mode 100755 index 4c7a18eaddf..00000000000 --- a/gcp/workers/cron/generate_sitemap/generate_and_upload.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e - -SCRIPT_PATH=$(dirname "$(readlink -f "$0")") - -SITEMAP_OUTPUT="sitemap_output/" -OUTPUT_BUCKET="${OUTPUT_GCS_BUCKET:=test-osv-dev-sitemap}" -BASE_URL_PATH="${BASE_URL:=https://test.osv.dev}" - -echo "Begin sitemap generation for $BASE_URL_PATH" - -"$SCRIPT_PATH/generate_sitemap.py" --base_url $BASE_URL_PATH - -echo "Begin Syncing with cloud to $OUTPUT_BUCKET" - -gcloud storage rsync --checksums-only --delete-unmatched-destination-objects $SITEMAP_OUTPUT "gs://$OUTPUT_BUCKET/"