diff --git a/.kokoro/presubmit.cfg b/.kokoro/presubmit.cfg index 2faccd73ff7..26871e5f368 100644 --- a/.kokoro/presubmit.cfg +++ b/.kokoro/presubmit.cfg @@ -2,9 +2,9 @@ build_file: "android-cuttlefish/.kokoro/presubmit.sh" action { define_artifacts { - regex: "*/*sponge_log.xml" # Actually a glob, not a regex - regex: "*/*sponge_log.log" - regex: "*/device_logs/**" # match all files regardless of directory depth regex: "github/android-cuttlefish/*.deb" + + # Matches all files regardless of directory depth: + regex: "kokoro_test_results/**" } } diff --git a/.kokoro/presubmit_gpu.cfg b/.kokoro/presubmit_gpu.cfg index 3d4eabcdc2a..877fdf60889 100644 --- a/.kokoro/presubmit_gpu.cfg +++ b/.kokoro/presubmit_gpu.cfg @@ -6,9 +6,9 @@ env_vars { action { define_artifacts { - regex: "*/*sponge_log.xml" # Actually a glob, not a regex - regex: "*/*sponge_log.log" - regex: "*/device_logs/**" # match all files regardless of directory depth regex: "github/android-cuttlefish/*.deb" + + # Matches all files regardless of directory depth: + regex: "kokoro_test_results/**" } } diff --git a/e2etests/MODULE.bazel b/e2etests/MODULE.bazel index 978f124ee66..9e7fcdaed69 100644 --- a/e2etests/MODULE.bazel +++ b/e2etests/MODULE.bazel @@ -4,7 +4,7 @@ bazel_dep(name = "rules_python", version = "1.1.0") bazel_dep(name = "rules_shell", version = "0.6.1") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") -go_sdk.download(version = "1.23.1") +go_sdk.download(version = "1.24.0") go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") diff --git a/e2etests/cvd/BUILD.bazel b/e2etests/cvd/BUILD.bazel index db8dd777a39..ad17cc6a9fc 100644 --- a/e2etests/cvd/BUILD.bazel +++ b/e2etests/cvd/BUILD.bazel @@ -1,185 +1,15 @@ -load("@rules_python//python:defs.bzl", "py_library", "py_test") -load("boot_tests.bzl", "cvd_command_boot_test", "cvd_cts_test", "cvd_load_boot_test", "launch_cvd_boot_test", "metrics_test") - -py_library( - name = "convert_xts_xml_to_junit_xml", - srcs = ["convert_xts_xml_to_junit_xml.py"], - srcs_version = "PY3", -) - -py_test( - name = "convert_xts_xml_to_junit_xml_test", - srcs = ["convert_xts_xml_to_junit_xml_test.py"], - srcs_version = "PY3", - deps = [":convert_xts_xml_to_junit_xml"], -) - -py_binary( - name = "convert_xts_xml_to_junit_xml_bin", - srcs = ["convert_xts_xml_to_junit_xml.py"], - srcs_version = "PY3", - main = "convert_xts_xml_to_junit_xml.py", -) - -# cvd fetch + launch_cvd tests -launch_cvd_boot_test( - name = "fetch_and_launch_git_main_x64_phone", - branch = "git_main", - target = "aosp_cf_x86_64_only_phone-trunk_staging-userdebug", -) - -launch_cvd_boot_test( - name = "fetch_and_launch_aosp_main_x64_phone", - branch = "aosp-android-latest-release", - target = "aosp_cf_x86_64_only_phone-userdebug", -) - -launch_cvd_boot_test( - name = "fetch_and_launch_aosp_android14_phone", - branch = "aosp-android14-gsi", - target = "aosp_cf_x86_64_phone-userdebug", -) - -launch_cvd_boot_test( - name = "fetch_and_launch_aosp_android13_phone", - branch = "aosp-android13-gsi", - target = "aosp_cf_x86_64_phone-userdebug", -) - -launch_cvd_boot_test( - name = "fetch_and_launch_aosp_android12_phone", - branch = "aosp-android12-gsi", - target = "aosp_cf_x86_64_phone-userdebug", -) - -# cvd load tests -cvd_load_boot_test( - name = "load_git_main_x64_phone", - env_file = "environment_specs/git_main_x64_phone.json", -) - -cvd_load_boot_test( - name = "load_aosp_main_x64_phone", - env_file = "environment_specs/aosp_main_x64_phone.json", -) - -cvd_load_boot_test( - name = "load_aosp_main_x64_phone_x2", - size = "enormous", - env_file = "environment_specs/aosp_main_x64_phone_x2.json", -) - -cvd_command_boot_test( - name = "list_env_services", - branch = "aosp-android-latest-release", - cvd_command = [ - "env", - "ls", - ], - target = "aosp_cf_x86_64_only_phone-userdebug", -) - -cvd_command_boot_test( - name = "take_bugreport", - branch = "aosp-android-latest-release", - cvd_command = ["host_bugreport"], - target = "aosp_cf_x86_64_only_phone-userdebug", -) - -cvd_command_boot_test( - name = "add_displays", - branch = "aosp-android-latest-release", - cvd_command = [ - "display", - "add", - "--display=width=500,height=500", - ], - target = "aosp_cf_x86_64_only_phone-userdebug", -) - -cvd_command_boot_test( - name = "list_displays", - branch = "aosp-android-latest-release", - cvd_command = [ - "display", - "list", - ], - target = "aosp_cf_x86_64_only_phone-userdebug", -) - -metrics_test( - name = "verify_metrics_transmission", - branch = "aosp-android-latest-release", - target = "aosp_cf_x86_64_only_phone-userdebug", -) - -cvd_cts_test( - name = "aosp_android_latest_auto_enables_gfxstream_test", - cuttlefish_branch = "aosp-android-latest-release", - cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug", - cuttlefish_create_args = [ - "--gpu_mode=auto", - ], - cts_branch = "aosp-android15-tests-release", - cts_target = "test_suites_x86_64", - cts_args = [ - "--include-filter=CuttlefishGraphicsConfigurationTest", - "--module-arg=CuttlefishGraphicsConfigurationTest:set-option:expected-egl:emulation", - ], - tags = ["requires_gpu"], -) - -cvd_cts_test( - name = "aosp_android_latest_gfxstream_deqp_vk_smoke_tests", - cuttlefish_branch = "aosp-android-latest-release", - cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug", - cuttlefish_create_args = [ - "--gpu_mode=gfxstream", - ], - cts_branch = "aosp-android15-tests-release", - cts_target = "test_suites_x86_64", - cts_args = [ - "--include-filter=CtsDeqpTestCases", - "--module-arg=CtsDeqpTestCases:include-filter:dEQP-VK.api.smoke*", - ], - tags = ["requires_gpu"], -) - -cvd_cts_test( - name = "aosp_android_latest_gfxstream_guest_angle_host_swiftshader_deqp_vk_smoke_tests", - cuttlefish_branch = "aosp-android-latest-release", - cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug", - cuttlefish_create_args = [ - "--gpu_mode=gfxstream_guest_angle_host_swiftshader", - ], - cts_branch = "aosp-android15-tests-release", - cts_target = "test_suites_x86_64", - cts_args = [ - "--include-filter=CtsDeqpTestCases", - "--module-arg=CtsDeqpTestCases:include-filter:dEQP-VK.api.smoke*", - ], - # This does not actually require a GPU but currently only the GPU image - # has the XTS requirements preinstalled. - tags = ["requires_gpu"], -) - -cvd_cts_test( - name = "aosp_android_latest_gfxstream_guest_angle_host_swiftshader_graphics_cts_tests", - cuttlefish_branch = "aosp-android-latest-release", - cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug", - cuttlefish_create_args = [ - "--gpu_mode=gfxstream_guest_angle_host_swiftshader", - ], - cts_branch = "aosp-android15-tests-release", - cts_target = "test_suites_x86_64", - cts_args = [ - "--include-filter=CtsGraphicsTestCases", - "--include-filter=CtsNativeHardwareTestCases", - "--include-filter=CtsUiRenderingTestCases", - ], - # This does not actually require a GPU but currently only the GPU image - # has the XTS requirements preinstalled. - tags = ["requires_gpu"], -) +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # TODO(b/329100411) test loading older branches diff --git a/e2etests/cvd/boot_tests.bzl b/e2etests/cvd/boot_tests.bzl deleted file mode 100644 index 3d389160290..00000000000 --- a/e2etests/cvd/boot_tests.bzl +++ /dev/null @@ -1,164 +0,0 @@ -def launch_cvd_boot_test(name, branch, target, credential_source = ""): - args = ["-b", branch, "-t", target] - if credential_source: - args += ["-c", credential_source] - native.sh_test( - name = name, - size = "medium", - srcs = ["launch_cvd_boot_test.sh"], - args = args, - tags = [ - "exclusive", - "external", - "no-sandbox", - ], - ) - -def cvd_load_boot_test(name, env_file, size = "medium", credential_source = ""): - args = ["-e", "cvd/" + env_file] - if credential_source: - args += ["-c", credential_source] - native.sh_test( - name = name, - size = size, - srcs = ["cvd_load_boot_test.sh"], - args = args, - data = [env_file], - tags = [ - "exclusive", - "external", - "no-sandbox", - ], - ) - native.sh_test( - name = name + "_additional_substitution", - size = size, - srcs = ["cvd_load_boot_test.sh"], - args = args, - data = [ - env_file, - "//:debian_substitution_marker", - ], - env = {"LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE": "$(execpath //:debian_substitution_marker)"}, - tags = [ - "exclusive", - "external", - "no-sandbox", - ], - ) - -def cvd_command_boot_test(name, branch, target, cvd_command = [], credential_source = "", tags = []): - args = ["-b", branch, "-t", target] - if credential_source: - args += ["-c", credential_source] - args += cvd_command - native.sh_test( - name = name, - size = "medium", - srcs = ["cvd_command_boot_test.sh"], - args = args, - tags = [ - "exclusive", - "external", - "no-sandbox", - ] + tags, - ) - native.sh_test( - name = name + "_additional_substitution", - size = "medium", - srcs = ["cvd_command_boot_test.sh"], - args = args, - data = ["//:debian_substitution_marker"], - env = {"LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE": "$(execpath //:debian_substitution_marker)"}, - tags = [ - "exclusive", - "external", - "no-sandbox", - ] + tags, - ) - -def metrics_test(name, branch, target, credential_source = ""): - args = ["-b", branch, "-t", target] - if credential_source: - args += ["-c", credential_source] - native.sh_test( - name = name, - size = "medium", - srcs = ["metrics_test.sh"], - args = args, - tags = [ - "exclusive", - "external", - "no-sandbox", - ], - ) - native.sh_test( - name = name + "_additional_substitution", - size = "medium", - srcs = ["metrics_test.sh"], - args = args, - data = ["//:debian_substitution_marker"], - env = {"LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE": "$(execpath //:debian_substitution_marker)"}, - tags = [ - "exclusive", - "external", - "no-sandbox", - ], - ) - -def cvd_cts_test( - name, - cuttlefish_branch, - cuttlefish_target, - cts_branch, - cts_target, - cts_args, - cuttlefish_create_args = [], - credential_source = "", - tags = []): - """Runs a given set of CTS tests on a given Cuttlefish build.""" - args = [ - "--cuttlefish-create-args=\"" + " ".join(cuttlefish_create_args) + "\"", - "--cuttlefish-fetch-branch=" + cuttlefish_branch, - "--cuttlefish-fetch-target=" + cuttlefish_target, - "--xml-test-result-converter-path=$(location //cvd:convert_xts_xml_to_junit_xml_bin)", - "--xts-args=\"" + " ".join(cts_args) + "\"", - "--xts-fetch-branch=" + cts_branch, - "--xts-fetch-target=" + cts_target, - "--xts-type=cts", - ] - - if credential_source: - args.append("--credential-source=" + credential_source) - - native.sh_test( - name = name, - size = "enormous", - srcs = ["cvd_xts_test.sh"], - args = args, - data = [ - "//cvd:convert_xts_xml_to_junit_xml_bin", - ], - tags = [ - "exclusive", - "external", - "no-sandbox", - ] + tags, - ) - native.sh_test( - name = name + "_additional_substitution", - size = "enormous", - srcs = ["cvd_xts_test.sh"], - args = args, - data = [ - "//:debian_substitution_marker", - "//cvd:convert_xts_xml_to_junit_xml_bin", - ], - env = {"LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE": "$(execpath //:debian_substitution_marker)"}, - tags = [ - "exclusive", - "external", - "no-sandbox", - ] + tags, - ) - diff --git a/e2etests/cvd/bugreport_tests/BUILD.bazel b/e2etests/cvd/bugreport_tests/BUILD.bazel new file mode 100644 index 00000000000..3b3c8c80f03 --- /dev/null +++ b/e2etests/cvd/bugreport_tests/BUILD.bazel @@ -0,0 +1,45 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "bugreport_tests", + srcs = ["main_test.go"], + deps = [ + "//cvd/common:common", + ], + size = "enormous", + tags = [ + "exclusive", + "external", + "no-sandbox", + ], +) + +go_test( + name = "bugreport_tests_with_substitution", + srcs = ["main_test.go"], + deps = [ + "//cvd/common:common", + ], + data = ["//:debian_substitution_marker"], + env = {"LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE": "$(rlocationpath //:debian_substitution_marker)"}, + size = "enormous", + tags = [ + "exclusive", + "external", + "no-sandbox", + ], +) diff --git a/e2etests/cvd/bugreport_tests/main_test.go b/e2etests/cvd/bugreport_tests/main_test.go new file mode 100644 index 00000000000..5f09297f40e --- /dev/null +++ b/e2etests/cvd/bugreport_tests/main_test.go @@ -0,0 +1,42 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/google/android-cuttlefish/e2etests/cvd/common" +) + +func TestTakeBugreport(t *testing.T) { + c := e2etests.TestContext{} + c.SetUp(t) + + if err := c.CVDFetch(e2etests.FetchArgs{ + DefaultBuildBranch: "aosp-android-latest-release", + DefaultBuildTarget: "aosp_cf_x86_64_only_phone-userdebug", + }); err != nil { + t.Fatal(err) + } + if err := c.CVDCreate(e2etests.CreateArgs{}); err != nil { + t.Fatal(err) + } + + if _, err := c.RunCmd("cvd", "host_bugreport"); err != nil { + t.Fatal(err) + } + + c.TearDown() +} diff --git a/e2etests/cvd/common/BUILD.bazel b/e2etests/cvd/common/BUILD.bazel new file mode 100644 index 00000000000..ee437f46f01 --- /dev/null +++ b/e2etests/cvd/common/BUILD.bazel @@ -0,0 +1,25 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "common", + srcs = ["common.go"], + importpath = "github.com/google/android-cuttlefish/e2etests/cvd/common", + visibility = ["//visibility:public"], + deps = [ + "@rules_go//go/runfiles", + ], +) \ No newline at end of file diff --git a/e2etests/cvd/common/common.go b/e2etests/cvd/common/common.go new file mode 100644 index 00000000000..ecc43dc3ab9 --- /dev/null +++ b/e2etests/cvd/common/common.go @@ -0,0 +1,530 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2etests + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "io/fs" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/bazelbuild/rules_go/go/runfiles" +) + +func FileExists(f string) bool { + info, err := os.Stat(f) + if errors.Is(err, fs.ErrNotExist) { + return false + } + if info.IsDir() { + return false + } + return true +} + +func DirectoryExists(d string) bool { + info, err := os.Stat(d) + if errors.Is(err, fs.ErrNotExist) { + return false + } + if !info.IsDir() { + return false + } + return true +} + +type TestContext struct { + t *testing.T + tempdir string + + // This is used instead of `T.Cleanup()` to support running commands during + // cleanup. The context from `T.Context()` is cancelled before `T.Cleanup()` + // registered cleanup functions are invoked. + cleanups []func() + teardownCalled bool +} + +func (tc *TestContext) RunCmdWithEnv(command []string, envvars map[string]string) (string, error) { + cmd := exec.CommandContext(tc.t.Context(), command[0], command[1:]...) + + cmdOutputBuf := bytes.Buffer{} + cmdWriter := io.MultiWriter(&cmdOutputBuf, log.Writer()) + cmd.Stdout = cmdWriter + cmd.Stderr = cmdWriter + + envvarPairs := []string{}; + for k, v := range envvars { + envvarPairs = append(envvarPairs, fmt.Sprintf("%s=%s", k, v)) + } + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, envvarPairs...) + + log.Printf("Running `%s %s`\n", strings.Join(envvarPairs, " "), strings.Join(command, " ")) + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("`%s` failed: %w", strings.Join(command, " "), err) + } + + return cmdOutputBuf.String(), nil +} + +func (tc *TestContext) RunAdbWaitForDevice() error { + adbCommand := []string{ + "timeout", + "--kill-after=30s", + "29s", + "adb", + "wait-for-device", + } + if _, err := tc.RunCmd(adbCommand...); err != nil { + return fmt.Errorf("timed out waiting for Cuttlefish device to connect to adb: %w", err) + } + return nil +} + +func (tc *TestContext) RunCmd(args... string) (string, error) { + command := []string{} + command = append(command, args...) + return tc.RunCmdWithEnv(command, map[string]string{}) +} + +type FetchArgs struct { + DefaultBuildBranch string + DefaultBuildTarget string + TestSuiteBuildBranch string + TestSuiteBuildTarget string +} + +type CreateRunner int + +const ( + CvdCreate CreateRunner = iota + LaunchCvd +) + +type StringSliceFlag []string + +func (s *StringSliceFlag) String() string { + return fmt.Sprintf("%s", *s) +} + +func (s *StringSliceFlag) Set(value string) error { + *s = append(*s, value) + return nil +} + +type CreateArgs struct { + Args StringSliceFlag +} + +type FetchAndCreateArgs struct { + Fetch FetchArgs + Create CreateArgs +} + +func (tc *TestContext) CVDFetch(args FetchArgs) error { + log.Printf("Fetching...") + fetchCmd := []string{ + "cvd", + "fetch", + fmt.Sprintf("--default_build=%s/%s", args.DefaultBuildBranch, args.DefaultBuildTarget), + fmt.Sprintf("--target_directory=%s", tc.tempdir), + } + if args.TestSuiteBuildBranch != "" && args.TestSuiteBuildTarget != "" { + fetchCmd = append(fetchCmd, fmt.Sprintf("--test_suites_build=%s/%s", args.TestSuiteBuildBranch, args.TestSuiteBuildTarget)) + } + + credentialArg := os.Getenv("CREDENTIAL_SOURCE") + if credentialArg != "" { + fetchCmd = append(fetchCmd, fmt.Sprintf("--credential_source=%s", credentialArg)) + } + if _, err := tc.RunCmd(fetchCmd...); err != nil { + log.Printf("Failed to fetch: %w", err) + return err + } + + // Android CTS includes some files with a `kernel` suffix which confuses the + // Cuttlefish launcher prior to + // https://github.com/google/android-cuttlefish/commit/881728ed85329afaeb16e3b849d60c7a32fedcb7. + log.Printf("Adjusting fetcher_config.json to work around old launcher limitation...") + tc.RunCmd("sed", "-i", `s|_kernel\"|_kernel_zzz\"|g`, path.Join(tc.tempdir, "/fetcher_config.json")) + + log.Printf("Fetch completed!") + + return nil +} + +func (tc *TestContext) CVDCreate(args CreateArgs) error { + tempdirEnv := map[string]string{ + "HOME": tc.tempdir, + } + + createCmd := []string{"cvd", "create"}; + createCmd = append(createCmd, "--report_anonymous_usage_stats=y") + createCmd = append(createCmd, "--undefok=report_anonymous_usage_stats") + if len(args.Args) > 0 { + createCmd = append(createCmd, args.Args...) + } + if _, err := tc.RunCmdWithEnv(createCmd, tempdirEnv); err != nil { + log.Printf("Failed to create instance(s): %w", err) + return err + } + + tc.Cleanup(func() { tc.CVDStop() }) + return nil +} + +func (tc *TestContext) CVDStop() error { + tempdirEnv := map[string]string{ + "HOME": tc.tempdir, + } + + stopCmd := []string{"cvd", "stop"}; + if _, err := tc.RunCmdWithEnv(stopCmd, tempdirEnv); err != nil { + log.Printf("Failed to stop instance(s): %w", err) + return err + } + + return nil +} + +func (tc *TestContext) LaunchCVD(args CreateArgs) error { + tempdirEnv := map[string]string{ + "HOME": tc.tempdir, + } + + createCmd := []string{"bin/launch_cvd", "--daemon"} + createCmd = append(createCmd, "--report_anonymous_usage_stats=y") + createCmd = append(createCmd, "--undefok=report_anonymous_usage_stats") + if len(args.Args) > 0 { + createCmd = append(createCmd, args.Args...) + } + if _, err := tc.RunCmdWithEnv(createCmd, tempdirEnv); err != nil { + log.Printf("Failed to create instance(s): %w", err) + return err + } + + + tc.Cleanup(func() { tc.StopCVD() }) + return nil +} + +func (tc *TestContext) StopCVD() error { + tempdirEnv := map[string]string{ + "HOME": tc.tempdir, + } + + stopCmd := []string{"bin/stop_cvd"}; + if _, err := tc.RunCmdWithEnv(stopCmd, tempdirEnv); err != nil { + log.Printf("Failed to stop instance(s): %w", err) + return err + } + + return nil +} + +type LoadArgs struct { + LoadConfig string +} + +func (tc *TestContext) CVDLoad(load LoadArgs) error { + configpath := path.Join(tc.tempdir, "cvd_load_config.json") + + log.Printf("Writing config to %s", configpath) + + err := os.WriteFile(configpath, []byte(load.LoadConfig), os.ModePerm) + if err != nil { + log.Printf("Failed to write load config to file: %w", err) + return err + } + + log.Printf("Creating instance(s) via `cvd load`...") + loadCmd := []string{ + "cvd", + "load", + configpath, + }; + credentialArg := os.Getenv("CREDENTIAL_SOURCE") + if credentialArg != "" { + loadCmd = append(loadCmd, fmt.Sprintf("--credential_source=%s", credentialArg)) + } + if _, err := tc.RunCmd(loadCmd...); err != nil { + log.Printf("Failed to perform `cvd load`: %w", err) + return err + } + log.Printf("Created instance(s) via `cvd load`!") + + tc.Cleanup(func() { tc.CVDStop() }) + return nil +} + +func (tc *TestContext) SetUp(t *testing.T) { + tc.t = t + tc.teardownCalled = false + + log.Printf("Initializing %s test...", tc.t.Name()) + + log.Printf("Cleaning up any pre-existing instances...") + if _, err := tc.RunCmd("cvd", "reset", "-y"); err != nil { + log.Printf("Failed to cleanup any pre-existing instances: %w", err) + } + log.Printf("Finished cleaning up any pre-existing instances!") + + tc.tempdir = tc.t.TempDir() + + log.Printf("Chdir to %s", tc.tempdir) + tc.t.Chdir(tc.tempdir) + + marker := os.Getenv("LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE") + if marker != "" { + marker, err := runfiles.Rlocation(marker) + if err != nil { + tc.t.Fatalf("failed to find substituion runfile from %s: %w", marker, err) + } + + full, err := filepath.EvalSymlinks(marker) + if err != nil { + tc.t.Fatalf("failed to read substituion marker full path from %s: %w", marker, err) + } + + tc.t.Setenv("LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE", full) + } + + log.Printf("Initialized %s test!", tc.t.Name()) + + tc.t.Cleanup(func() { + if !tc.teardownCalled { + tc.t.Fatalf("Test %s forgot to call TearDown().", tc.t.Name()) + } + }) +} + +func (tc *TestContext) Cleanup(f func()) { + tc.cleanups = append(tc.cleanups, f) +} + +func (tc *TestContext) TearDown() { + log.Printf("Cleaning up after test...") + tc.teardownCalled = true + + log.Printf("Running registered cleanup callbacks...") + for i := len(tc.cleanups) - 1; i >= 0; i-- { + tc.cleanups[i]() + } + log.Printf("Finished running registered cleanup callbacks!") + + outdir := os.Getenv("TEST_UNDECLARED_OUTPUTS_DIR") + if outdir != "" { + testoutdir := path.Join(outdir, tc.t.Name()) + + err := os.MkdirAll(testoutdir, os.ModePerm) + if err == nil { + log.Printf("Copying logs to test output directory...") + + patterns := [...]string{ + "cuttlefish_runtime/cuttlefish_config.json", + "cuttlefish_runtime/logcat", + "cuttlefish_runtime/*.log", + } + for _, pattern := range patterns { + matches, err := filepath.Glob(path.Join(tc.tempdir, pattern)) + if err == nil { + for _, file := range matches { + _, err := tc.RunCmd("cp", "--dereference", file, testoutdir) + if err != nil { + log.Printf("failed to copy %s to %s: %w", file, testoutdir, err) + } + } + } else { + log.Printf("failed to glob files matching %s: %w", pattern, err) + } + } + + matches, err := filepath.Glob(path.Join(tc.tempdir, "cuttlefish/instances/*")) + if err == nil { + for _, instancedir := range matches { + instance := filepath.Base(instancedir) + + outinstancedir := path.Join(testoutdir, fmt.Sprintf("instance_%s", instance)) + err := os.MkdirAll(outinstancedir, os.ModePerm) + if err == nil { + logdir := path.Join(instancedir, "logs") + _, err := tc.RunCmd("cp", "-r", "--dereference", logdir, outinstancedir) + if err != nil { + log.Printf("failed to copy %s to %s: %w", logdir, outinstancedir, err) + } + } + } + } + + log.Printf("Finished copying logs to test output directory!") + } else { + log.Printf("failed to mkdir output directory %s: %w", outdir, err) + } + } + + tc.RunCmd("cvd", "reset", "-y") + + log.Printf("Finished cleaning up after test!") +} + +func xtsTradefedPath(args XtsArgs) string { + return fmt.Sprintf("android-%s/tools/%s-tradefed", args.XtsType, args.XtsType) +} + +func findLocalXTS(cuttlefishArgs FetchAndCreateArgs, xtsArgs XtsArgs) string { + // TODO: explore adding a non-user-specific cache option to cvd. + + homedir, err := os.UserHomeDir() + if err != nil { + user := os.Getenv("USER") + if user == "" { + return "" + } + homedir = path.Join("/home", user) + } + + possibleDir := path.Join(homedir, cuttlefishArgs.Fetch.TestSuiteBuildBranch, cuttlefishArgs.Fetch.TestSuiteBuildTarget) + log.Printf("Checking %s", possibleDir) + if !DirectoryExists(possibleDir) { + return "" + } + + possibleTradefed := path.Join(possibleDir, xtsTradefedPath(xtsArgs)) + if !FileExists(possibleTradefed) { + return "" + } + + return possibleDir +} + +type XtsTest struct { + Name string `xml:"name,attr"` + Result string `xml:"result,attr"` +} + +type XtsTestCase struct { + Name string `xml:"name,attr"` + Tests []XtsTest `xml:"Test"` +} + +type XtsModule struct { + Name string `xml:"name,attr"` + TestCases []XtsTestCase `xml:"TestCase"` +} + +type XtsSummary struct { + Pass int `xml:"pass,attr"` + Failed int `xml:"failed,attr"` + ModulesDone int `xml:"modules_done,attr"` + ModulesTotal int `xml:"modules_total,attr"` +} + +type XtsResult struct { + Summary XtsSummary `xml:"Summary"` + Modules []XtsModule `xml:"Module"` +} + +type XtsArgs struct { + XtsArgs StringSliceFlag + XtsType string + XtsXmlConverterBinary string +} + +func RunXts(t *testing.T, cuttlefishArgs FetchAndCreateArgs, xtsArgs XtsArgs) { + tc := TestContext{} + tc.SetUp(t) + + localXtsDir := findLocalXTS(cuttlefishArgs, xtsArgs) + if localXtsDir != "" { + log.Printf("Re-using existing XTS at %s", localXtsDir) + cuttlefishArgs.Fetch.TestSuiteBuildBranch = "" + cuttlefishArgs.Fetch.TestSuiteBuildTarget = "" + } else { + log.Printf("Failed to find existing XTS, will fetch.") + } + + if err := tc.CVDFetch(cuttlefishArgs.Fetch); err != nil { + t.Fatal(err) + } + + if err := tc.CVDCreate(cuttlefishArgs.Create); err != nil { + t.Fatal(err) + } + + if err := tc.RunAdbWaitForDevice(); err != nil { + t.Fatalf("failed to wait for Cuttlefish device to connect to adb: %w", err) + } + + xtsDir := "test_suites" + if localXtsDir != "" { + xtsDir = localXtsDir + } + + log.Printf("Chdir to %s", xtsDir) + tc.t.Chdir(xtsDir) + + log.Printf("Running XTS...") + xtsCommand := []string{ + xtsTradefedPath(xtsArgs), + "run", + "commandAndExit", + xtsArgs.XtsType, + "--log-level-display=INFO", + }; + xtsCommand = append(xtsCommand, xtsArgs.XtsArgs...) + if _, err := tc.RunCmd(xtsCommand...); err != nil { + t.Fatalf("failed to fully run XTS: %w", err) + } + log.Printf("Finished running XTS!") + + log.Printf("Parsing XTS results...") + xtsResultsPath := path.Join(fmt.Sprintf("android-%s", xtsArgs.XtsType), "results", "latest", "test_result.xml") + xtsResultsBytes, err := os.ReadFile(xtsResultsPath) + if err != nil { + t.Fatalf("failed to read XTS XML results from %s: %w", xtsResultsPath, err) + } + + var xtsResult XtsResult + if err := xml.Unmarshal([]byte(xtsResultsBytes), &xtsResult); err != nil { + t.Fatalf("failed to parse XTS XML results from %s: %w", xtsResultsPath, err) + } + + for _, xtsModule := range(xtsResult.Modules) { + for _, xtsTestCase := range(xtsModule.TestCases) { + for _, xtsTest := range(xtsTestCase.Tests) { + testname := fmt.Sprintf("%s#%s", xtsTestCase.Name, xtsTest.Name) + t.Run(testname, func(t *testing.T) { + log.Printf("%s result: %s", testname, xtsTest.Result) + if xtsTest.Result == "failed" { + t.Error("XTS test failed, see logs") + } + }) + } + } + } + log.Printf("Finished parsing XTS results!", err) + + tc.TearDown() +} diff --git a/e2etests/cvd/convert_xts_xml_to_junit_xml.py b/e2etests/cvd/convert_xts_xml_to_junit_xml.py deleted file mode 100644 index 5497dd41efd..00000000000 --- a/e2etests/cvd/convert_xts_xml_to_junit_xml.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (C) 2025 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0(the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Converts Android CTS test_result.xml files to Bazel result xml files. - -JUnit XML structure: https://github.com/testmoapp/junitxml - -""" - -import argparse -import os -import re -import sys - -from xml.etree import ElementTree -import xml.dom.minidom - -def to_pretty_xml(root, indent=" "): - return xml.dom.minidom.parseString(ElementTree.tostring(root, encoding='UTF-8', xml_declaration=True)).toprettyxml(indent) - -def convert_xts_xml_to_junit_xml(input_xml_string): - input_xml_root = ElementTree.ElementTree(ElementTree.fromstring(input_xml_string)).getroot() - - output_xml_testsuites = ElementTree.Element("testsuites") - output_xml_testsuites.set('name', 'AllTests') - - output_xml_testsuites_fail = 0 - output_xml_testsuites_total = 0 - - for input_xml_result in input_xml_root.iter('Result'): - for input_xml_module in input_xml_result.iter('Module'): - # e.g. CtsDeqpTestCases - input_module_name = input_xml_module.get('name') - - output_xml_module_testsuite = ElementTree.SubElement(output_xml_testsuites, "testsuite") - output_xml_module_testsuite.set('name', input_module_name) - - output_xml_module_testsuite_fail = 0 - output_xml_module_testsuite_total = 0 - - for input_xml_testcase in input_xml_module.iter('TestCase'): - # e.g. dEQP-EGL.functional.image.create - input_testcase_name = input_xml_testcase.get('name') - - output_xml_testcase_testsuite = ElementTree.SubElement(output_xml_module_testsuite, "testsuite") - output_xml_testcase_testsuite.set('name', input_testcase_name) - - output_xml_testcase_testsuite_fail = 0 - output_xml_testcase_testsuite_total = 0 - - for input_xml_test in input_xml_testcase.iter('Test'): - # e.g. gles2_android_native_rgba4_texture - input_test_name = input_xml_test.get('name') - - # e.g. dEQP-EGL.functional.image.create#gles2_android_native_rgba4_texture - output_test_name = input_testcase_name + '#' + input_test_name - - output_xml_testcase = ElementTree.SubElement(output_xml_testcase_testsuite, "testcase") - output_xml_testcase.set('name', output_test_name) - - input_test_result = input_xml_test.get('result') - if input_test_result == 'failed': - output_xml_testcase_failure = ElementTree.SubElement(output_xml_testcase, "failure") - output_xml_testcase_failure.set('name', output_test_name) - - output_xml_testcase_testsuite_fail += 1 - - output_xml_testcase_testsuite_total += 1 - - output_xml_testcase_testsuite.set('tests', str(output_xml_testcase_testsuite_total)) - output_xml_testcase_testsuite.set('failures', str(output_xml_testcase_testsuite_fail)) - - output_xml_module_testsuite_fail += output_xml_testcase_testsuite_fail - output_xml_module_testsuite_total += output_xml_testcase_testsuite_total - - output_xml_module_testsuite.set('tests', str(output_xml_module_testsuite_total)) - output_xml_module_testsuite.set('failures', str(output_xml_module_testsuite_fail)) - - output_xml_testsuites_fail += output_xml_module_testsuite_fail - output_xml_testsuites_total += output_xml_testcase_testsuite_total - - output_xml_testsuites.set('tests', str(output_xml_module_testsuite_total)) - output_xml_testsuites.set('failures', str(output_xml_module_testsuite_fail)) - - output_xml_string = to_pretty_xml(output_xml_testsuites) - - return output_xml_string - -def main(): - parser = argparse.ArgumentParser(description="Converts Android CTS test_result.xml files to Bazel result xml files") - parser.add_argument("--input_xml_file", help="The path to input Android CTS test_result.xml file.") - parser.add_argument("--output_xml_file", help="The path for the output xml file that is digestable by bazel.") - args = parser.parse_args() - - input_xml_string = None - with open(args.input_xml_file, 'r') as intput_xml_file: - input_xml_string = intput_xml_file.read() - - output_xml_string = convert_xts_xml_to_junit_xml(input_xml_string) - - with open(args.output_xml_file, 'w') as output_xml_file: - output_xml_file.write(output_xml_string) - -if __name__ == '__main__': - main() diff --git a/e2etests/cvd/convert_xts_xml_to_junit_xml_test.py b/e2etests/cvd/convert_xts_xml_to_junit_xml_test.py deleted file mode 100644 index 4c125bbc7bc..00000000000 --- a/e2etests/cvd/convert_xts_xml_to_junit_xml_test.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (C) 2025 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0(the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Tests for convert_xts_xml_to_junit_xml.py""" - -import unittest - -from xml.etree import ElementTree - -from cvd.convert_xts_xml_to_junit_xml import * - -class TestConvertXtsXmlToJunitXml(unittest.TestCase): - - def test_basic_conversion(self): - input = """ - - - - - - - - - - - - - - """ - actual = convert_xts_xml_to_junit_xml(input) - expected = """ - - - - - - - - - - - - -""" - self.assertEqual(ElementTree.canonicalize(actual), ElementTree.canonicalize(expected)) - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/e2etests/cvd/cvd_command_boot_test.sh b/e2etests/cvd/cvd_command_boot_test.sh deleted file mode 100755 index 2ff71cfbabd..00000000000 --- a/e2etests/cvd/cvd_command_boot_test.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -set -e -x - -BRANCH="" -TARGET="" -CREDENTIAL_SOURCE="${CREDENTIAL_SOURCE:-}" - -while getopts "b:c:s:t:" opt; do - case "${opt}" in - b) - BRANCH="${OPTARG}" - ;; - c) - CREDENTIAL_SOURCE="${OPTARG}" - ;; - s) - SUBSTITUTIONS="${OPTARG}" - ;; - t) - TARGET="${OPTARG}" - ;; - *) - echo "Unknown flag: -${opt}" >&2 - echo "Usage: $0 -b BRANCH -t TARGET" - exit 1 - esac -done -shift $((OPTIND-1)) - -if [[ "${BRANCH}" == "" ]]; then - echo "Missing required -b argument" -fi - -if [[ "${TARGET}" == "" ]]; then - echo "Missing required -t argument" -fi - -if [[ "${LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE}" != "" ]]; then - LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE=$(readlink -f "${LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE}") -fi - -workdir="$(mktemp -d -t cvd_command_test.XXXXXX)" - -function collect_logs_and_cleanup() { - # Don't immediately exit on failure anymore - set +e - if [[ -n "${TEST_UNDECLARED_OUTPUTS_DIR}" ]] && [[ -d "${TEST_UNDECLARED_OUTPUTS_DIR}" ]]; then - cp "${workdir}"/*.log "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/*.log "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/logcat "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/cuttlefish_config.json "${TEST_UNDECLARED_OUTPUTS_DIR}" - fi - rm -rf "${workdir}" - # Be nice, don't leave devices behind. - cvd reset -y -} - -# Regardless of whether and when a failure occurs logs must collected -trap collect_logs_and_cleanup EXIT - -# Make sure to run in a clean environment, without any devices running -cvd reset -y - -credential_arg="" -if [[ -n "$CREDENTIAL_SOURCE" ]]; then - credential_arg="--credential_source=${CREDENTIAL_SOURCE}" -fi - -cvd fetch \ - --default_build="${BRANCH}/${TARGET}" \ - --target_directory="${workdir}" \ - ${credential_arg} - -( - cd "${workdir}" - cvd create --report_anonymous_usage_stats=y --undefok=report_anonymous_usage_stats - if (( $# > 0 )); then - cvd "$@" - fi - cvd rm -) diff --git a/e2etests/cvd/cvd_create_tests/BUILD.bazel b/e2etests/cvd/cvd_create_tests/BUILD.bazel new file mode 100644 index 00000000000..d74dfe43860 --- /dev/null +++ b/e2etests/cvd/cvd_create_tests/BUILD.bazel @@ -0,0 +1,29 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "cvd_create_tests", + srcs = ["main_test.go"], + deps = [ + "//cvd/common:common", + ], + size = "enormous", + tags = [ + "exclusive", + "external", + "no-sandbox", + ], +) diff --git a/e2etests/cvd/cvd_create_tests/main_test.go b/e2etests/cvd/cvd_create_tests/main_test.go new file mode 100644 index 00000000000..5a6ac81c1a8 --- /dev/null +++ b/e2etests/cvd/cvd_create_tests/main_test.go @@ -0,0 +1,57 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "testing" + + "github.com/google/android-cuttlefish/e2etests/cvd/common" +) + +func TestCvdCreate(t *testing.T) { + testcases := []struct { + branch string + target string + }{ + { + branch: "aosp-android-latest-release", + target: "aosp_cf_x86_64_only_phone-userdebug", + }, + { + branch: "git_main", + target: "aosp_cf_x86_64_only_phone-trunk_staging-userdebug", + }, + } + for _, tc := range testcases { + t.Run(fmt.Sprintf("BUILD=%s/%s", tc.branch, tc.target), func(t *testing.T) { + c := e2etests.TestContext{} + c.SetUp(t) + + if err := c.CVDFetch(e2etests.FetchArgs{ + DefaultBuildBranch: tc.branch, + DefaultBuildTarget: tc.target, + }); err != nil { + t.Fatal(err) + } + + if err := c.CVDCreate(e2etests.CreateArgs{}); err != nil { + t.Fatal(err) + } + + c.TearDown() + }) + } +} diff --git a/e2etests/cvd/cvd_load_boot_test.sh b/e2etests/cvd/cvd_load_boot_test.sh deleted file mode 100755 index f7d915bdfab..00000000000 --- a/e2etests/cvd/cvd_load_boot_test.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -set -e -x - -usage="Usage: $0 [-c CREDENTIAL_SOURCE] ENVIRONMENT_SPECIFICATION_FILE" - -ENV_FILE="" -CREDENTIAL_SOURCE="${CREDENTIAL_SOURCE:-}" -while getopts "c:e:" opt; do - case "${opt}" in - c) - CREDENTIAL_SOURCE="${OPTARG}" - ;; - e) - ENV_FILE="${OPTARG}" - ;; - *) - echo "${usage}" - exit 1 - esac -done - -if [[ -z "${ENV_FILE}" ]]; then - echo "${usage}" - exit 1 -fi - -if [[ "${LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE}" != "" ]]; then - LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE=$(readlink -f "${LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE}") -fi - -CMD_OUT="cvd_load_stdout.txt" -CMD_ERR="cvd_load_stderr.txt" - -function collect_logs_and_cleanup() { - # Don't immediately exit on failure anymore - set +e - - if [[ -n "${TEST_UNDECLARED_OUTPUTS_DIR}" ]] && [[ -d "${TEST_UNDECLARED_OUTPUTS_DIR}" ]]; then - cp "${ENV_FILE}" "${TEST_UNDECLARED_OUTPUTS_DIR}/environment.json" - cp "${CMD_OUT}" "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${CMD_ERR}" "${TEST_UNDECLARED_OUTPUTS_DIR}" - - # TODO(b/324650975): cvd doesn't print very useful information yet so file locations must be extracted this way - home_dir="$(grep -o -E -m1 HOME="/tmp/cvd/[0-9a-zA-Z/_]+" "${CMD_ERR}" | cut -d= -f2)" - artifacts_dir="$(grep -o -E '\-\-target_directory=/tmp/cvd/[0-9a-zA-Z/_]+' "${CMD_ERR}" | cut -d= -f2 | grep -o -E '[^"]*')" - - cp "${artifacts_dir}/fetch.log" "${TEST_UNDECLARED_OUTPUTS_DIR}" - - for instance_dir in "${home_dir}"/cuttlefish/instances/*; do - if [[ -d "${instance_dir}" ]]; then - instance="$(basename "${instance_dir}")" - for log_file in "${instance_dir}"/logs/*; do - if [[ -f "${log_file}" ]]; then - base_name="$(basename "${log_file}")" - cp "${log_file}" "${TEST_UNDECLARED_OUTPUTS_DIR}/${instance}_${base_name}" - fi - done - if [[ -f "${instance_dir}/cuttlefish_config.json" ]]; then - cp "${instance_dir}"/cuttlefish_config.json "${TEST_UNDECLARED_OUTPUTS_DIR}/${instance}_cuttlefish_config.json" - else - echo "No cuttlefish_config.json found for instance ${instance}" - fi - fi - done - fi - - # Be nice, don't leave devices behind. - cvd reset -y -} - -trap collect_logs_and_cleanup EXIT - -# Make sure to run in a clean environment, without any devices running -cvd reset -y - -credential_arg="" -if [[ -n "$CREDENTIAL_SOURCE" ]]; then - credential_arg="--override=fetch.credential_source:${CREDENTIAL_SOURCE}" -fi -cvd load ${credential_arg} "${ENV_FILE}" >"${CMD_OUT}" 2>"${CMD_ERR}" diff --git a/e2etests/cvd/cvd_load_tests/BUILD.bazel b/e2etests/cvd/cvd_load_tests/BUILD.bazel new file mode 100644 index 00000000000..311319dd753 --- /dev/null +++ b/e2etests/cvd/cvd_load_tests/BUILD.bazel @@ -0,0 +1,29 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "cvd_load_tests", + srcs = ["main_test.go"], + deps = [ + "//cvd/common:common", + ], + size = "enormous", + tags = [ + "exclusive", + "external", + "no-sandbox", + ], +) diff --git a/e2etests/cvd/cvd_load_tests/main_test.go b/e2etests/cvd/cvd_load_tests/main_test.go new file mode 100644 index 00000000000..e6672d7e422 --- /dev/null +++ b/e2etests/cvd/cvd_load_tests/main_test.go @@ -0,0 +1,153 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "testing" + + "github.com/google/android-cuttlefish/e2etests/cvd/common" +) + +func TestCvdLoad(t *testing.T) { + testcases := []struct { + name string + loadconfig string + }{ + { + name: "GitMainX64Phone", + loadconfig: ` +{ + "instances": [ + { + "name": "ins-1", + "disk": { + "default_build": "@ab\/git_main\/aosp_cf_x86_64_only_phone-trunk_staging-userdebug" + }, + "vm": { + "cpus": 4, + "memory_mb": 4096, + "setupwizard_mode": "REQUIRED" + }, + "graphics": { + "displays": [ + { + "width": 720, + "height": 1280, + "dpi": 140, + "refresh_rate_hertz": 60 + } + ], + "record_screen": false + } + } + ], + "netsim_bt": false, + "metrics": { + "enable": true + }, + "common": { + "host_package": "@ab\/git_main\/aosp_cf_x86_64_only_phone-trunk_staging-userdebug" + } +}`, + }, + { + name: "AospMainX64Phone", + loadconfig: ` +{ + "instances": [ + { + "name": "ins-1", + "disk": { + "default_build": "@ab\/aosp-android-latest-release\/aosp_cf_x86_64_only_phone-userdebug", + "super": { + "system": "@ab\/aosp-android-latest-release\/aosp_cf_x86_64_only_phone-userdebug" + } + }, + "boot": { + "kernel": { + "build": "@ab\/aosp_kernel-common-android16-6.12\/kernel_virt_x86_64" + }, + "build": "@ab\/aosp-android-latest-release\/aosp_cf_x86_64_only_phone-userdebug" + }, + "vm": { + "cpus": 4, + "memory_mb": 4096, + "setupwizard_mode": "REQUIRED" + }, + "graphics": { + "displays": [ + { + "width": 720, + "height": 1280, + "dpi": 140, + "refresh_rate_hertz": 60 + } + ], + "record_screen": false + } + } + ], + "netsim_bt": false, + "metrics": { + "enable": true + }, + "common": { + "host_package": "@ab\/aosp-android-latest-release\/aosp_cf_x86_64_only_phone-userdebug" + } +}`, + }, + { + name: "AospMainX64PhoneX2", + loadconfig: ` +{ + "instances": [ + { + "name": "ins-1", + "disk": { + "default_build": "@ab\/aosp-android-latest-release\/aosp_cf_x86_64_only_phone-userdebug" + } + }, + { + "name": "ins-2", + "disk": { + "default_build": "@ab\/aosp-android-latest-release\/aosp_cf_x86_64_only_phone-userdebug" + } + } + ], + "metrics": { + "enable": true + }, + "common": { + "host_package": "@ab\/aosp-android-latest-release\/aosp_cf_x86_64_only_phone-userdebug" + } +}`, + }, + } + for _, tc := range testcases { + t.Run(fmt.Sprintf("BUILD=%s", tc.name), func(t *testing.T) { + c := e2etests.TestContext{} + c.SetUp(t) + + if err := c.CVDLoad(e2etests.LoadArgs{ + LoadConfig: tc.loadconfig, + }); err != nil { + t.Fatal(err) + } + + c.TearDown() + }) + } +} diff --git a/e2etests/cvd/cvd_xts_test.sh b/e2etests/cvd/cvd_xts_test.sh deleted file mode 100755 index 2637e6fb7ef..00000000000 --- a/e2etests/cvd/cvd_xts_test.sh +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env bash - -echo "" -echo "" -echo "" -echo "NOTE: consider using Bazel's '--test_timeout=' when running" -echo "a XTS test for the first time as the XTS download may be slow!" -echo "" -echo "'cvd fetch' caches the downloads and subsequent runs should be faster." -echo "" -echo "" -echo "" - -# Exit on failure: -set -e -# Print commands before running: -set -x - -# Parse command line flags: -CREDENTIAL_SOURCE="${CREDENTIAL_SOURCE:-}" -SUBSTITUTIONS="" -CUTTLEFISH_CREATE_ARGS="" -CUTTLEFISH_FETCH_BRANCH="" -CUTTLEFISH_FETCH_TARGET="" -XML_CONVERTER_PATH="" -XTS_ARGS="" -XTS_FETCH_BRANCH="" -XTS_FETCH_TARGET="" -XTS_TYPE="" -for arg in "$@"; do - case "${arg}" in - --cuttlefish-create-args=*) - CUTTLEFISH_CREATE_ARGS="${arg#*=}" - ;; - --cuttlefish-fetch-branch=*) - CUTTLEFISH_FETCH_BRANCH="${arg#*=}" - ;; - --cuttlefish-fetch-target=*) - CUTTLEFISH_FETCH_TARGET="${arg#*=}" - ;; - --credential-source=*) - CREDENTIAL_SOURCE="${arg#*=}" - ;; - --xml-test-result-converter-path=*) - XML_CONVERTER_PATH="${arg#*=}" - ;; - --xts-args=*) - XTS_ARGS="${arg#*=}" - ;; - --xts-fetch-branch=*) - XTS_FETCH_BRANCH="${arg#*=}" - ;; - --xts-fetch-target=*) - XTS_FETCH_TARGET="${arg#*=}" - ;; - --xts-type=*) - XTS_TYPE="${arg#*=}" - ;; - *) - echo "Unknown flag: ${arg}" >&2 - exit 1 - ;; - esac -done -if [ -z "${CUTTLEFISH_FETCH_BRANCH}" ]; then - echo "Missing required --cuttlefish-fetch-branch flag." - exit 1 -fi -if [ -z "${CUTTLEFISH_FETCH_TARGET}" ]; then - echo "Missing required --cuttlefish-fetch-targt flag." - exit 1 -fi -if [ -z "${XML_CONVERTER_PATH}" ]; then - echo "Missing required --xml-test-result-converter-path flag." - exit 1 -fi -if [ -z "${XTS_ARGS}" ]; then - echo "Missing required --xts-args flag." - exit 1 -fi -if [ -z "${XTS_FETCH_BRANCH}" ]; then - echo "Missing required --xts-fetch-branch flag." - exit 1 -fi -if [ -z "${XTS_FETCH_TARGET}" ]; then - echo "Missing required --xts-fetch-target flag." - exit 1 -fi -if [ -z "${XTS_TYPE}" ]; then - echo "Missing required --xts-type flag." - exit 1 -fi -XML_CONVERTER_PATH="$(realpath ${XML_CONVERTER_PATH})" -echo "Parsed command line args:" -echo " * CREDENTIAL_SOURCE: ${CREDENTIAL_SOURCE}" -echo " * CUTTLEFISH_CREATE_ARGS: ${CUTTLEFISH_CREATE_ARGS}" -echo " * CUTTLEFISH_FETCH_BRANCH: ${CUTTLEFISH_FETCH_BRANCH}" -echo " * CUTTLEFISH_FETCH_TARGET: ${CUTTLEFISH_FETCH_TARGET}" -echo " * XML_CONVERTER_PATH: ${XML_CONVERTER_PATH}" -echo " * XTS_ARGS: ${XTS_ARGS}" -echo " * XTS_FETCH_BRANCH: ${XTS_FETCH_BRANCH}" -echo " * XTS_FETCH_TARGET: ${XTS_FETCH_TARGET}" -echo " * XTS_TYPE: ${XTS_TYPE}" - -if [[ "${LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE}" != "" ]]; then - LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE=$(readlink -f "${LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE}") -fi - -RUN_DIRECTORY="$(pwd)" - -workdir="$(mktemp -d -t cvd_xts_test.XXXXXX)" -cd "${workdir}" - -XTS_DIRECTORY="${workdir}/test_suites" -XTS_LATEST_RESULTS_DIRECTORY="" -XTS_RUNNER="" -if [ "${XTS_TYPE}" == "cts" ]; then - XTS_LATEST_RESULTS_DIRECTORY="${XTS_DIRECTORY}/android-cts/results/latest" - XTS_RUNNER="${XTS_DIRECTORY}/android-cts/tools/cts-tradefed" -elif [ "${XTS_TYPE}" == "vts" ]; then - XTS_LATEST_RESULTS_DIRECTORY="${XTS_DIRECTORY}/android-vts/results/latest" - XTS_RUNNER="${XTS_DIRECTORY}/android-vts/tools/vts-tradefed" -else - echo "Unsupported XTS type: ${XTS_TYPE}. Failure." - exit 1 -fi -XTS_LATEST_RESULT_XML="${XTS_LATEST_RESULTS_DIRECTORY}/test_result.xml" -XTS_LATEST_RESULT_CONVERTED_XML="${XTS_LATEST_RESULTS_DIRECTORY}/test.xml" - -# Additional files to keep track of and save to the final test results directory: -CVD_CREATE_LOG_FILE="${workdir}/cvd_create_logs.txt" -CVD_FETCH_LOG_FILE="${workdir}/cvd_fetch_logs.txt" -XTS_LOG_FILE="${workdir}/xts_logs.txt" - -function collect_logs_and_cleanup() { - # Don't immediately exit on failure anymore - set +e - if [[ -n "${TEST_UNDECLARED_OUTPUTS_DIR}" ]] && [[ -d "${TEST_UNDECLARED_OUTPUTS_DIR}" ]]; then - echo "Copying logs to test output directory..." - - cp "${workdir}"/*.log "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/*.log "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/logcat "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/cuttlefish_config.json "${TEST_UNDECLARED_OUTPUTS_DIR}" - - cp "${CVD_FETCH_LOG_FILE}" "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${CVD_CREATE_LOG_FILE}" "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${XTS_LOG_FILE}" "${TEST_UNDECLARED_OUTPUTS_DIR}" - fi - - if [[ -n "${XML_OUTPUT_FILE}" ]]; then - echo "Copying converted XML results for Bazel consumption..." - cp "${XTS_LATEST_RESULT_CONVERTED_XML}" "${XML_OUTPUT_FILE}" - echo "Copied!" - fi - - rm -rf "${workdir}" - - # Be nice, don't leave devices behind. - cvd reset -y -} - -# Regardless of whether and when a failure occurs logs must collected -trap collect_logs_and_cleanup EXIT - -# Make sure to run in a clean environment, without any devices running -cvd reset -y - - -# Fetch Cuttlefish and XTS: -echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" -echo "Fetching Cuttlefish and ${XTS_TYPE}..." - -credential_arg="" -if [[ -n "$CREDENTIAL_SOURCE" ]]; then - credential_arg="--credential_source=${CREDENTIAL_SOURCE}" -fi - -cvd fetch \ - --default_build="${CUTTLEFISH_FETCH_BRANCH}/${CUTTLEFISH_FETCH_TARGET}" \ - --test_suites_build="${XTS_FETCH_BRANCH}/${XTS_FETCH_TARGET}" \ - --target_directory="${workdir}" \ - ${credential_arg} \ - 2>&1 | tee "${CVD_FETCH_LOG_FILE}" - -echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" -echo "Fetch completed!" - - -# Android CTS includes some files with a `kernel` suffix which confuses the -# Cuttlefish launcher prior to -# https://github.com/google/android-cuttlefish/commit/881728ed85329afaeb16e3b849d60c7a32fedcb7. -echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" -echo "Modifying `fetcher_config.json` for `kernel` file suffix workaround ..." -sed -i 's|_kernel"|_kernel_zzz"|g' ${workdir}/fetcher_config.json - - -# Create a new Cuttlefish device: -echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" -echo "Creating a Cuttlefish device with: ${CUTTLEFISH_CREATE_ARGS}" - -# Note: `eval` used because `CUTTLEFISH_CREATE_ARGS` might have been -# escaped by Bazel. -eval \ -HOME="$(pwd)" \ -cvd create \ - --report_anonymous_usage_stats=y \ - --undefok=report_anonymous_usage_stats \ - "${CUTTLEFISH_CREATE_ARGS}" \ - 2>&1 | tee "${CVD_CREATE_LOG_FILE}" - -echo "Cuttlefish device created!" - - -# Wait for the new Cuttlefish device to appear in adb: -echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" -echo "Waiting for the Cuttlefish device to connect to adb..." -timeout --kill-after=30s 29s adb wait-for-device -if [ $? -eq 0 ]; then - echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" - echo "Cuttlefish device connected to adb." - adb devices -else - echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" - echo "Timeout waiting for Cuttlefish device to connect to adb!" - exit 1 -fi - - -# Run XTS. Note: `eval` used because `XTS_ARGS` might have been -# escaped by Bazel. -echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" -echo "Running ${XTS_TYPE}..." -cd "${XTS_DIRECTORY}" -HOME="$(pwd)" \ -eval ${XTS_RUNNER} run commandAndExit "${XTS_TYPE}" \ - --log-level-display=INFO \ - "${XTS_ARGS}" \ - 2>&1 | tee "${XTS_LOG_FILE}" -echo "Finished running ${XTS_TYPE}!" - - -# Convert results to Bazel friendly format: -echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" -echo "Converting ${XTS_TYPE} test result output to Bazel XML format..." -python3 ${XML_CONVERTER_PATH} \ - --input_xml_file="${XTS_LATEST_RESULT_XML}" \ - --output_xml_file="${XTS_LATEST_RESULT_CONVERTED_XML}" -echo "Converted!" - - -# Determine exit code by looking at XTS results: -echo "$(date +'%Y-%m-%dT%H:%M:%S%z')" -echo "Checking if any ${XTS_TYPE} tests failed..." -failures=$(cat ${XTS_LATEST_RESULT_CONVERTED_XML} | grep "&2 - echo "Usage: $0 -b BRANCH -t TARGET" - exit 1 - esac -done - -if [[ "${BRANCH}" == "" ]]; then - echo "Missing required -b argument" -fi - -if [[ "${TARGET}" == "" ]]; then - echo "Missing required -t argument" -fi - -workdir="$(mktemp -d -t cvd_boot_test.XXXXXX)" - -function collect_logs_and_cleanup() { - # Don't immediately exit on failure anymore - set +e - if [[ -n "${TEST_UNDECLARED_OUTPUTS_DIR}" ]] && [[ -d "${TEST_UNDECLARED_OUTPUTS_DIR}" ]]; then - cp "${workdir}"/*.log "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/*.log "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/logcat "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/cuttlefish_config.json "${TEST_UNDECLARED_OUTPUTS_DIR}" - fi - rm -rf "${workdir}" - # Be nice, don't leave devices behind. - cvd reset -y -} - -# Regardless of whether and when a failure occurs logs must collected -trap collect_logs_and_cleanup EXIT - -# Make sure to run in a clean environment, without any devices running -cvd reset -y - -credential_arg="" -if [[ -n "$CREDENTIAL_SOURCE" ]]; then - credential_arg="--credential_source=${CREDENTIAL_SOURCE}" -fi - -cvd fetch \ - --default_build="${BRANCH}/${TARGET}" \ - --target_directory="${workdir}" \ - ${credential_arg} - -( - cd "${workdir}" - HOME=$(pwd) bin/launch_cvd --daemon --report_anonymous_usage_stats=y --undefok=report_anonymous_usage_stats - HOME=$(pwd) bin/stop_cvd -) diff --git a/e2etests/cvd/launch_cvd_tests/BUILD.bazel b/e2etests/cvd/launch_cvd_tests/BUILD.bazel new file mode 100644 index 00000000000..bc7780f50a6 --- /dev/null +++ b/e2etests/cvd/launch_cvd_tests/BUILD.bazel @@ -0,0 +1,29 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "launch_cvd_tests", + srcs = ["main_test.go"], + deps = [ + "//cvd/common:common", + ], + size = "enormous", + tags = [ + "exclusive", + "external", + "no-sandbox", + ], +) diff --git a/e2etests/cvd/launch_cvd_tests/main_test.go b/e2etests/cvd/launch_cvd_tests/main_test.go new file mode 100644 index 00000000000..07cba6e345f --- /dev/null +++ b/e2etests/cvd/launch_cvd_tests/main_test.go @@ -0,0 +1,74 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/google/android-cuttlefish/e2etests/cvd/common" +) + +func TestLaunchCvd(t *testing.T) { + testcases := []struct { + name string + branch string + target string + }{ + { + name: "GitMainPhone", + branch: "git_main", + target: "aosp_cf_x86_64_only_phone-trunk_staging-userdebug", + }, + { + name: "AospMainPhone", + branch: "aosp-android-latest-release", + target: "aosp_cf_x86_64_only_phone-userdebug", + }, + { + name: "Aosp14GsiPhone", + branch: "aosp-android14-gsi", + target: "aosp_cf_x86_64_phone-userdebug", + }, + { + name: "Aosp13GsiPhone", + branch: "aosp-android13-gsi", + target: "aosp_cf_x86_64_phone-userdebug", + }, + { + name: "Aosp12GsiPhone", + branch: "aosp-android12-gsi", + target: "aosp_cf_x86_64_phone-userdebug", + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + c := e2etests.TestContext{} + c.SetUp(t) + + if err := c.CVDFetch(e2etests.FetchArgs{ + DefaultBuildBranch: tc.branch, + DefaultBuildTarget: tc.target, + }); err != nil { + t.Fatal(err) + } + + if err := c.LaunchCVD(e2etests.CreateArgs{}); err != nil { + t.Fatal(err) + } + + c.TearDown() + }) + } +} diff --git a/e2etests/cvd/metrics_test.sh b/e2etests/cvd/metrics_test.sh deleted file mode 100755 index c41065e1521..00000000000 --- a/e2etests/cvd/metrics_test.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env bash - -set -e -x - -BRANCH="" -TARGET="" -CREDENTIAL_SOURCE="${CREDENTIAL_SOURCE:-}" - -while getopts "b:c:t:" opt; do - case "${opt}" in - b) - BRANCH="${OPTARG}" - ;; - c) - CREDENTIAL_SOURCE="${OPTARG}" - ;; - t) - TARGET="${OPTARG}" - ;; - *) - echo "Unknown flag: -${opt}" >&2 - echo "Usage: $0 -b BRANCH -t TARGET" - exit 1 - esac -done -shift $((OPTIND-1)) - -if [[ "${BRANCH}" == "" ]]; then - echo "Missing required -b argument" -fi - -if [[ "${TARGET}" == "" ]]; then - echo "Missing required -t argument" -fi - -if [[ "${LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE}" != "" ]]; then - LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE=$(readlink -f "${LOCAL_DEBIAN_SUBSTITUTION_MARKER_FILE}") -fi - -workdir="$(mktemp -d -t cvd_command_test.XXXXXX)" - -function collect_logs_and_cleanup() { - # Don't immediately exit on failure anymore - set +e - if [[ -n "${TEST_UNDECLARED_OUTPUTS_DIR}" ]] && [[ -d "${TEST_UNDECLARED_OUTPUTS_DIR}" ]]; then - cp "${workdir}"/*.log "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/*.log "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/logcat "${TEST_UNDECLARED_OUTPUTS_DIR}" - cp "${workdir}"/cuttlefish_runtime/cuttlefish_config.json "${TEST_UNDECLARED_OUTPUTS_DIR}" - fi - rm -rf "${workdir}" - # Be nice, don't leave devices behind. - cvd reset -y -} - -# Regardless of whether and when a failure occurs logs must collected -trap collect_logs_and_cleanup EXIT - -# Make sure to run in a clean environment, without any devices running -cvd reset -y - -credential_arg="" -if [[ -n "$CREDENTIAL_SOURCE" ]]; then - credential_arg="--credential_source=${CREDENTIAL_SOURCE}" -fi - -cvd fetch \ - --default_build="${BRANCH}/${TARGET}" \ - --target_directory="${workdir}" \ - ${credential_arg} - -( - cd "${workdir}" - - # generate instantiation, start, and boot complete events - cvd create --host_path="${workdir}" --product_path="${workdir}" - - # verify transmission by detecting existence of the metrics directory and the debug event files - metrics_dir=`cvd fleet 2> /dev/null | jq --raw-output '.groups[0].metrics_dir'` - if ! [[ -d "${metrics_dir}" ]]; then - echo "metrics directory not found" - exit 1 - fi - - # file prefixes sourced from `cuttlefish/host/libs/metrics/event_type.cc::EventTypeString` function - if ! [[ -f "`ls ${metrics_dir}/device_instantiation*.txtpb`" ]]; then - echo "metrics not transmitted for the expected instantiation event" - exit 1 - fi - if ! [[ -f "`ls ${metrics_dir}/device_boot_start*.txtpb`" ]]; then - echo "metrics not transmitted for the expected start event" - exit 1 - fi - if ! [[ -f "`ls ${metrics_dir}/device_boot_complete*.txtpb`" ]]; then - echo "metrics not transmitted for the expected boot complete event" - exit 1 - fi - - # generate stop event and verify transmission - cvd stop - if ! [[ -f "`ls ${metrics_dir}/device_stop*.txtpb`" ]]; then - echo "metrics not transmitted for the expected stop event" - exit 1 - fi -) diff --git a/e2etests/cvd/metrics_tests/BUILD.bazel b/e2etests/cvd/metrics_tests/BUILD.bazel new file mode 100644 index 00000000000..128b3674071 --- /dev/null +++ b/e2etests/cvd/metrics_tests/BUILD.bazel @@ -0,0 +1,29 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "metrics_tests", + srcs = ["main_test.go"], + deps = [ + "//cvd/common:common", + ], + size = "enormous", + tags = [ + "exclusive", + "external", + "no-sandbox", + ], +) diff --git a/e2etests/cvd/metrics_tests/main_test.go b/e2etests/cvd/metrics_tests/main_test.go new file mode 100644 index 00000000000..43123af83aa --- /dev/null +++ b/e2etests/cvd/metrics_tests/main_test.go @@ -0,0 +1,93 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "path" + "path/filepath" + "regexp" + "testing" + + "github.com/google/android-cuttlefish/e2etests/cvd/common" +) + +func anyFileExists(pattern string) bool { + matches, err := filepath.Glob(pattern) + if err != nil { + return false + } + return len(matches) > 0 +} + +func TestMetrics(t *testing.T) { + c := e2etests.TestContext{} + c.SetUp(t) + + if err := c.CVDFetch(e2etests.FetchArgs{ + DefaultBuildBranch: "aosp-android-latest-release", + DefaultBuildTarget: "aosp_cf_x86_64_only_phone-userdebug", + }); err != nil { + t.Fatal(err) + } + + if err := c.CVDCreate(e2etests.CreateArgs{}); err != nil { + t.Fatal(err) + } + + var metricsdir string + err := func() error { + output, err := c.RunCmd("cvd", "fleet") + if err != nil { + return fmt.Errorf("failed to run `cvd fleet`") + } + + re := regexp.MustCompile(`"metrics_dir" : "(.*)",`) + matches := re.FindStringSubmatch(output) + if len(matches) != 2 { + return fmt.Errorf("failed to find metrics directory.") + } + + metricsdir = matches[1] + if !e2etests.DirectoryExists(metricsdir) { + return fmt.Errorf("failed to find directory %s", metricsdir) + } + + patterns := []string{ + "device_instantiation*.txtpb", + "device_boot_start*.txtpb", + "device_boot_complete*.txtpb", + } + for _, p := range patterns { + if !anyFileExists(path.Join(metricsdir, p)) { + return fmt.Errorf("failed to find a file matching `%s`", p) + } + } + return nil + }() + if err != nil { + t.Fatal(err) + } + + if err := c.CVDStop(); err != nil { + t.Fatal(err) + } + + if !anyFileExists(path.Join(metricsdir, "device_stop*.txtpb")) { + t.Errorf("failed to find `device_stop*.txtpb` file.") + } + + c.TearDown() +} diff --git a/e2etests/go.mod b/e2etests/go.mod index 80c1c419f57..7742bd8a2b7 100644 --- a/e2etests/go.mod +++ b/e2etests/go.mod @@ -1,6 +1,8 @@ module github.com/google/android-cuttlefish/e2etests -go 1.23.1 +go 1.24.0 + +toolchain go1.24.13 require ( github.com/docker/docker v25.0.7+incompatible @@ -11,6 +13,7 @@ require ( require ( github.com/Microsoft/go-winio v0.4.14 // indirect + github.com/bazelbuild/rules_go v0.60.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -54,12 +57,12 @@ require ( go.opentelemetry.io/otel/metric v1.32.0 // indirect go.opentelemetry.io/otel/sdk v1.32.0 // indirect go.opentelemetry.io/otel/trace v1.32.0 // indirect - golang.org/x/crypto v0.35.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/sys v0.30.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.33.0 // indirect golang.org/x/time v0.8.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect ) diff --git a/e2etests/go.sum b/e2etests/go.sum index 1307b79a3cb..0cd196ef2ab 100644 --- a/e2etests/go.sum +++ b/e2etests/go.sum @@ -2,6 +2,8 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/bazelbuild/rules_go v0.60.0 h1:apGSxTTrFUyLNvX9NQmF4CbntWAO0/S5eALeVgB/6Qk= +github.com/bazelbuild/rules_go v0.60.0/go.mod h1:CYcohJVxs4n7eftbC39GCqaEJm3E1EME+6QAkGguKoI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -144,6 +146,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -161,6 +165,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -183,6 +189,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -199,6 +207,7 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -215,10 +224,14 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 h1: google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:dguCy7UOdZhTvLzDyt15+rOrawrpM4q7DD9dQ1P11P4= google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE= google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 h1:iK2jbkWL86DXjEx0qiHcRE9dE4/Ahua5k6V8OWFb//c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/testutils/runcvde2etests.sh b/tools/testutils/runcvde2etests.sh index 58715b4db39..6d674b6da5e 100755 --- a/tools/testutils/runcvde2etests.sh +++ b/tools/testutils/runcvde2etests.sh @@ -26,17 +26,25 @@ done function gather_test_results() { # Don't immediately exit on error anymore set +e - for d in "${REPO_DIR}"/e2etests/bazel-testlogs/cvd/*; do - dir="${OUTPUT_DIR}/$(basename "$d")" - mkdir -p "${dir}" - cp "${d}/test.log" "${dir}/sponge_log.log" - cp "${d}/test.xml" "${dir}/sponge_log.xml" - if [[ -f "${d}/test.outputs/outputs.zip" ]]; then - unzip "${d}/test.outputs/outputs.zip" -d "${dir}/device_logs" + + # Keep in sync with `.kokoro/presubmit*.cfg`: + output_tests_directory="${OUTPUT_DIR}/kokoro_test_results" + + tests_directory="${REPO_DIR}/e2etests/bazel-testlogs/cvd" + for file in $(find ${tests_directory} -name test.xml); do + test_directory="$(dirname ${file})" + test_directory_relative=${test_directory/#$tests_directory} + outdir="${output_tests_directory}/${test_directory_relative}" + mkdir -p "${outdir}" + cp "${test_directory}/test.log" "${outdir}/sponge_log.log" + cp "${test_directory}/test.xml" "${outdir}/sponge_log.xml" + if [[ -f "${test_directory}/test.outputs/outputs.zip" ]]; then + unzip "${test_directory}/test.outputs/outputs.zip" -d "${outdir}" fi - # Make sure everyone has access to the output files - chmod -R a+rw "${dir}" done + + # Make sure everyone has access to the output files + chmod -R a+rw "${output_tests_directory}" } cd "${REPO_DIR}/e2etests"