Skip to content

chore: Migrate gsutil usage to gcloud storage #6325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gurusai-voleti wants to merge 1 commit into from
Closed

chore: Migrate gsutil usage to gcloud storage #6325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions perfkitbenchmarker/linux_benchmarks/cloudharmony_network_benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,13 +303,13 @@ def _PrepareBucket(benchmark_spec):
bucket_uri = f'gs://{bucket}'
service.MakeBucket(bucket)
# set default permissions to allow cloudharmony test file access
perm_cmd = ['gsutil', 'defacl', 'set', 'public-read', bucket_uri]
perm_cmd = ['gcloud', 'storage', 'buckets', 'update', bucket_uri, '--predefined-default-object-acl=public-read']
vm_util.IssueCommand(perm_cmd)
# set bucket lifecyle to ensure bucket deletion after 30 days
lifecyle_config_file = data.ResourcePath(
'cloudharmony_network_gcp_lifecycle.json'
)
lc_cmd = ['gsutil', 'lifecycle', 'set', lifecyle_config_file, bucket_uri]
lc_cmd = ['gcloud', 'storage', 'buckets', 'update', bucket_uri, '--lifecycle-file', lifecyle_config_file]
vm_util.IssueCommand(lc_cmd)
# prepare preprovisioned test data
tmp_dir = vm_util.GetTempDir()
Expand All @@ -332,7 +332,7 @@ def _PrepareBucket(benchmark_spec):
# copy preprovisioned test data to test bucket
src_path = posixpath.join(remote_probe_dir, '*')
dst_url = f'{bucket_uri}/probe'
cp_cmd = ['gsutil', 'cp', '-r', src_path, dst_url]
cp_cmd = ['gcloud', 'storage', 'cp', '--recursive', src_path, dst_url]
vm_util.IssueCommand(cp_cmd, raise_on_timeout=False)
# save the service and the bucket name
benchmark_spec.service = service
Expand Down
2 changes: 1 addition & 1 deletion perfkitbenchmarker/linux_packages/ycsb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -694,7 +694,7 @@ def CheckPrerequisites():
def _DownloadAndInstallTarball(vm, install_dir, url):
"""Downloads and installs a tarball to the VM."""
if url.startswith('gs://'):
download_cmd = 'gsutil cat'
download_cmd = 'gcloud storage cat'
else:
download_cmd = 'curl -L'
install_cmd = (
Expand Down
4 changes: 2 additions & 2 deletions perfkitbenchmarker/providers/gcp/bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -611,7 +611,7 @@ def ExtractDataset(

# Make sure the bucket is empty.
vm_util.IssueCommand(
['gsutil', '-m', 'rm', gcs_uri + '/**'], raise_on_failure=False
['gcloud', 'storage', 'rm', gcs_uri + '/**'], raise_on_failure=False
)

project_dataset = self.FormatProjectAndDatasetForCommand(dataset)
Expand Down Expand Up @@ -701,7 +701,7 @@ def LoadDataset(
for table in tables:
schema_path = schema_dir + table + '.json'
local_schema = './%s.json' % table
vm_util.IssueCommand(['gsutil', 'cp', schema_path, local_schema])
vm_util.IssueCommand(['gcloud', 'storage', 'cp', schema_path, local_schema])
cmd = [
'bq',
'load',
Expand Down
3 changes: 1 addition & 2 deletions tests/data/sysbench_stderr_output_sample.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -680,8 +680,7 @@ Warning: Permanently added '' (ECDSA) to the list of known hosts.
2017-07-07 11:35:32,601 41b2ddaa MainThread mysql_service(1/0) INFO percentile_response_time is 17.730000
2017-07-07 11:35:32,623 41b2ddaa MainThread INFO Publishing 16 samples to /tmp/perfkitbenchmarker/runs/.json
2017-07-07 11:35:32,624 41b2ddaa MainThread INFO Publishing 16 samples to gs://perfkit
2017-07-07 11:35:32,624 41b2ddaa MainThread INFO Running: gsutil cp /tmp/perfkitbenchmarker/runs/
2017-07-07 11:35:34,238 41b2ddaa MainThread INFO
2017-07-07 11:35:32,624 41b2ddaa MainThread INFO Running: gcloud storage cp /tmp/perfkitbenchmarker/runs/2017-07-07 11:35:34,238 41b2ddaa MainThread INFO
-------------------------PerfKitBenchmarker Complete Results-------------------------
{'metadata': {'cloud': 'GCP',
'dedicated_host': False,
Expand Down