diff --git a/vulnfeeds/cmd/combine-to-osv/README.md b/vulnfeeds/cmd/combine-to-osv/README.md index c5d252186f9..7dd4eb3c90c 100644 --- a/vulnfeeds/cmd/combine-to-osv/README.md +++ b/vulnfeeds/cmd/combine-to-osv/README.md @@ -63,6 +63,6 @@ This statically overrides the record generated, meaning any and all of the input #### Procedure -1. `gsutil cp gs://cve-osv-conversion/osv-output/CVE-YYYY-NNNN.json` +1. `gcloud storage cp gs://cve-osv-conversion/osv-output/CVE-YYYY-NNNN.json` 2. manually edit the file -3. `gsutil cp gs://cve-osv-conversion/osv-output-overrides/CVE-YYYY-NNNN.json` +3. `gcloud storage cp gs://cve-osv-conversion/osv-output-overrides/CVE-YYYY-NNNN.json` diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh index 41e182d191a..4450f834452 100755 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh @@ -64,19 +64,19 @@ find "${LOCAL_OUT_DIR}/${OSV_OUTPUT_PATH}" -type f -name \*.json \ # Copy (and remove any missing) results to GCS bucket, with some sanity # checking. -objs_present=$(gsutil ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) -objs_deleted=$(gsutil -m rsync -c -n -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) +objs_present=$(gcloud storage ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) +objs_deleted=$(gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) threshold=$(echo "scale=2; ${objs_present} * (${SAFETY_THRESHOLD_PCT:-2} / 100)" | bc) # # Bash can't deal with floats if (( $(echo "${objs_deleted} > ${threshold}" | bc -l) )); then echo "Aborting. Unexpectedly high (${objs_deleted}) number of CVE records would be deleted!" >> /dev/stderr - gsutil -m rsync -c -n -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr + gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr exit 1 fi echo "Copying CVEList records successfully converted to GCS bucket" -gsutil -q -m rsync -c -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" echo "Conversion run complete" diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh index d4d89806adf..9a77e11b49c 100755 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh @@ -65,19 +65,19 @@ done # Copy (and remove any missing) results to GCS bucket, with some sanity # checking. -objs_present=$(gsutil ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) -objs_deleted=$(gsutil -m rsync -c -n -d "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) +objs_present=$(gcloud storage ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) +objs_deleted=$(gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) threshold=$(echo "scale=2; ${objs_present} * (${SAFETY_THRESHOLD_PCT:-2} / 100)" | bc) # Bash can't deal with floats if (( $(echo "${objs_deleted} > ${threshold}" | bc -l) )); then echo "Warning. Unexpectedly high (${objs_deleted}) number of CVE records would be deleted!" >> /dev/stderr - gsutil -m rsync -c -n -d "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr + gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr # TODO: add back in once nvd-mirror issue fixed: exit 1 fi echo "Copying NVD CVE records successfully converted to GCS bucket" -gsutil -q -m rsync -c "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" +gcloud storage rsync --quiet --checksums-only "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" echo "Conversion run complete" diff --git a/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh index 0779c28930a..835010c49f3 100755 --- a/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh +++ b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh @@ -38,7 +38,7 @@ unzip -o "${WORK_DIR}/cpe.zip" -d "${CPE_JSON_DIR}" MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="" if [[ -n "${DEBIAN_COPYRIGHT_GCS_PATH}" ]]; then - gsutil ${BE_VERBOSE="-q"} cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}" + gcloud storage cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}" ${BE_VERBOSE="--quiet"} tar -C "${WORK_DIR}" -xf "${WORK_DIR}/$(basename ${DEBIAN_COPYRIGHT_GCS_PATH})" MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="--debian-metadata-path ${WORK_DIR}/metadata.ftp-master.debian.org" fi @@ -49,4 +49,4 @@ fi --output-dir="${WORK_DIR}" -gsutil ${BE_VERBOSE="-q"} cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}" +gcloud storage cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}" ${BE_VERBOSE="--quiet"} diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh index bbc6d0c8fc3..d98db88a422 100755 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh @@ -33,4 +33,4 @@ python debian-copyright-mirror.py "${WORK_DIR}/metadata.ftp-master.debian.org/ch tar -C "${WORK_DIR}" -cf "${WORK_DIR}/$(basename ${GCS_PATH})" . -gsutil ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" +gcloud storage ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" diff --git a/vulnfeeds/pypi/cloudbuild.yaml b/vulnfeeds/pypi/cloudbuild.yaml index e07547058d2..ef4eb12fdac 100644 --- a/vulnfeeds/pypi/cloudbuild.yaml +++ b/vulnfeeds/pypi/cloudbuild.yaml @@ -25,5 +25,6 @@ steps: args: - -c - bq query --max_rows=10000000 --format=json --nouse_legacy_sql < vulnfeeds/pypi/pypi_versions.sql > pypi_versions.json -- name: gcr.io/cloud-builders/gsutil - args: ['cp', 'pypi_links.json', 'pypi_versions.json', 'gs://pypa-advisory-db/triage/'] +- name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' + entrypoint: 'gcloud' + args: ['storage', 'cp', 'pypi_links.json', 'pypi_versions.json', 'gs://pypa-advisory-db/triage/'] diff --git a/vulnfeeds/tools/debian/run_convert_debian.sh b/vulnfeeds/tools/debian/run_convert_debian.sh index 8a44235a31d..909da3cfed3 100755 --- a/vulnfeeds/tools/debian/run_convert_debian.sh +++ b/vulnfeeds/tools/debian/run_convert_debian.sh @@ -44,7 +44,7 @@ poetry run python3 convert_debian.py --adv_type=DTSA -o $OSV_DTSA_OUT $WEBWML_PA popd echo "Begin Syncing with cloud" -gsutil -m rsync -c -d $OSV_DSA_OUT gs://$OUTPUT_BUCKET/dsa-osv -gsutil -m rsync -c -d "$OSV_DLA_OUT" "gs://${OUTPUT_BUCKET}/dla-osv" -gsutil -m rsync -c -d "$OSV_DTSA_OUT" "gs://${OUTPUT_BUCKET}/dtsa-osv" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects $OSV_DSA_OUT gs://$OUTPUT_BUCKET/dsa-osv +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects "$OSV_DLA_OUT" "gs://${OUTPUT_BUCKET}/dla-osv" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects "$OSV_DTSA_OUT" "gs://${OUTPUT_BUCKET}/dtsa-osv" echo "Successfully synced with cloud" \ No newline at end of file diff --git a/vulnfeeds/tools/debian/run_first_package_finder.sh b/vulnfeeds/tools/debian/run_first_package_finder.sh index 0dfebc5747d..b0e4d94996f 100755 --- a/vulnfeeds/tools/debian/run_first_package_finder.sh +++ b/vulnfeeds/tools/debian/run_first_package_finder.sh @@ -25,7 +25,7 @@ echo "Finding first packages" poetry run python3 first_package_finder.py echo "Syncing with cloud first_package_output ${OUTPUT_BUCKET}" -gsutil -q -m rsync -c -d 'first_package_output' "gs://${OUTPUT_BUCKET}/first_package_output" +gcloud storage rsync --checksums-only --delete-unmatched-destination-objects --quiet 'first_package_output' "gs://${OUTPUT_BUCKET}/first_package_output" echo "Successfully synced with cloud" popd \ No newline at end of file