diff --git a/.yamato/README.md b/.yamato/README.md index 2e0a761714..7db6767555 100644 --- a/.yamato/README.md +++ b/.yamato/README.md @@ -1,7 +1,7 @@ # Netcode for GameObjects CI Documentation ## Overview -This document provides an overview of the Continuous Integration (CI) implementation for Netcode for GameObjects. +This document provides an overview of the Continuous Integration (CI) implementation for Netcode for GameObjects. Specifics of each test are described within related files (for example .yamato/package-tests.yml) and this file present high level overview related to overall implementation. ## Test Configurations @@ -9,7 +9,6 @@ CI related files are present inside .yamato/ folder and we can distinguish speci ### Helper jobs - `.yamato/package-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. -- `.yamato/project-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. This packs all packages of a given project. - `.yamato/_run-all.yml` responsible for grouping tests into groups for easier management (for example "all console tests"). - `.yamato/_triggers.yml` responsible for defining triggers (PR, nightly, weekly etc.) and defining which tests to run. - `disable-burst-if-requested.py` responsible for helping to disable burst if needed. @@ -77,4 +76,4 @@ Currently, the CI implementation supports the following platforms: ## Design Considerations In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. -The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. \ No newline at end of file +The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index 089a8b5247..1832685b89 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -15,10 +15,10 @@ run_quick_checks: name: Run Quick Initial Checks dependencies: - - .yamato/package-pack.yml#package_pack_-_ngo_ubuntu + # Ensure the code is running to our current standards - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk - # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version - - .yamato/vetting-test.yml#vetting_test + # This is needed for most of the jobs to execute tests + it runs xray PVP checks (all fast checks) + - .yamato/package-pack.yml#package_pack_-_ngo_win # Runs all package tests @@ -253,3 +253,44 @@ run_all_project_tests_console_standalone_6000: - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_6000.0 {% endfor -%} {% endfor -%} + + +# Runs all CMB service tests +run_all_project_tests_cmb_service: + name: Run All CMB Service Tests + dependencies: +{% for project in projects.default -%} +{% for platform in test_platforms.default -%} +{% for editor in validation_editors.all -%} +{% for backend in scripting_backends -%} + - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }} +{% endfor -%} +{% endfor -%} +{% endfor -%} +{% endfor -%} + +# Runs all CMB service tests on trunk editor +run_all_project_tests_cmb_service_trunk: + name: Run All CMB Service Tests [Trunk only] + dependencies: +{% for project in projects.default -%} +{% for platform in test_platforms.default -%} +{% for editor in validation_editors.default -%} +{% for backend in scripting_backends -%} + - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }} +{% endfor -%} +{% endfor -%} +{% endfor -%} +{% endfor -%} + +# Runs all CMB service tests on mimimum supported editor (6000.0 in case of NGOv2.X) +run_all_project_tests_cmb_service_6000: + name: Run All CMB Service Tests [6000.0] + dependencies: +{% for project in projects.default -%} +{% for platform in test_platforms.default -%} +{% for backend in scripting_backends -%} + - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_6000.0 +{% endfor -%} +{% endfor -%} +{% endfor -%} diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index ec379d091b..72145210c2 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -44,15 +44,15 @@ # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job. -#----------------------------------------------------------------------------------- +#----------------------------------------------------------------------------------- # After some experimenting with CI setups we discovered that even though sometimes we don't need CI to run (no reason to run package tests if only Documentation is changed) there are some checks that devs may not realize but changes in seemingly unrelated files will cause their failures # This trigger was created to ensure that ALL PRs run this minimal check even when we don't need to run full tests pr_minimal_required_checks: name: Minimal PR checks dependencies: - - .yamato/package-pack.yml#package_pack_-_ngo_win - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk + - .yamato/package-pack.yml#package_pack_-_ngo_win triggers: expression: |- (pull_request.comment eq "ngo" OR @@ -72,6 +72,9 @@ pr_code_changes_checks: name: Code changes PR checks # Run the following tests on a selection of different desktop platforms dependencies: + # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version + - .yamato/vetting-test.yml#vetting_test + # Run package EditMode and Playmode package tests on trunk and an older supported editor (6000.0) - .yamato/package-tests.yml#package_test_-_ngo_trunk_mac - .yamato/package-tests.yml#package_test_-_ngo_6000.0_win @@ -82,7 +85,9 @@ pr_code_changes_checks: # Run standalone test. We run it only on Ubuntu since it's the fastest machine, and it was noted that for example distribution on macOS is taking 40m since we switched to Apple Silicon # Coverage on other standalone machines is present in Nightly job so it's enough to not run all of them for PRs - - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_win_il2cpp_6000.0 + # desktop_standalone_test and cmb_service_standalone_test are both reusing desktop_standalone_build dependency so we run those in the same configuration on PRs to reduce waiting time. + # Note that our daily tests will anyway run both test configurations in "minimal supported" and "trunk" configurations + - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_ubuntu_il2cpp_trunk - .yamato/cmb-service-standalone-tests.yml#cmb_service_standalone_test_testproject_ubuntu_il2cpp_trunk triggers: expression: |- @@ -104,9 +109,9 @@ pr_code_changes_checks: "**/*.md" ] cancel_old_ci: true - - - + + + @@ -144,6 +149,9 @@ develop_nightly: # Build player for webgl platform on trunk and 6000.0 editors - .yamato/_run-all.yml#run_all_webgl_builds_trunk - .yamato/_run-all.yml#run_all_webgl_builds_6000 + # Run Runtime tests against cmb service on trunk and 6000.0 editors + - .yamato/_run-all.yml#run_all_project_tests_cmb_service_trunk + - .yamato/_run-all.yml#run_all_project_tests_cmb_service_6000 # Build player for webgl platform on trunk and 6000.0 editors - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_ubuntu_trunk - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_win_6000.0 @@ -177,5 +185,7 @@ develop_weekly_trunk: - .yamato/_run-all.yml#run_all_project_tests_console_standalone # Build player for webgl platform on trunk - .yamato/_run-all.yml#run_all_webgl_builds + # Run Runtime tests against CMB service + - .yamato/_run-all.yml#run_all_project_tests_cmb_service # Run code coverage test - .yamato/code-coverage.yml#code_coverage_ubuntu_trunk diff --git a/.yamato/cmb-service-standalone-tests.yml b/.yamato/cmb-service-standalone-tests.yml index 3a9cc76b5d..9bcc0f2bf2 100644 --- a/.yamato/cmb-service-standalone-tests.yml +++ b/.yamato/cmb-service-standalone-tests.yml @@ -57,7 +57,7 @@ cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }} - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} artifacts: logs: paths: diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 688fa2e03f..70330ac51e 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -1,6 +1,6 @@ {% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- - + # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for executing package tests with code coverage analysis enabled. # Coverage analysis provides insights into: @@ -8,7 +8,7 @@ # Line and branch coverage statistics # Generated HTML reports for coverage visualization # Additional metrics for coverage analysis - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) @@ -19,12 +19,12 @@ # Requires Unity Editor installation # Burst compilation is disabled to ensure accurate coverage measurement # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise coverage results will not be visible - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - - + + + {% for platform in test_platforms.default -%} {% for editor in validation_editors.default -%} code_coverage_{{ platform.name }}_{{ editor }}: @@ -39,7 +39,7 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout={ test_timeout }} --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: @@ -47,4 +47,4 @@ code_coverage_{{ platform.name }}_{{ editor }}: dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 999651ef2c..3690585c66 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -4,7 +4,7 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for Console platform test validation. # Those tests cover both PlayMode and EditMode tests from package test assemblies. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) @@ -16,10 +16,10 @@ # 1. Build Phase: Creates standalone players for console platforms # 2. Run Phase: Executes runtime tests on actual console devices # The Run phase uses build job as dependency - + # Note: More of a Unity specific but test assemblies need to be included in the build phase command # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - + # PLATFORM SPECIFICS----------------------------------------------------------------- # Common Requirements: # All consoles require IL2CPP scripting backend @@ -29,13 +29,13 @@ # Switch: ARM64 architecture only # Other Consoles: x64 architecture # Each console requires specific SDK paths and tools - + # QUALITY THOUGHTS-------------------------------------------------------------------- # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - + + # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_build -%} @@ -51,7 +51,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout}} variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -72,9 +72,9 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endfor -%} {% endfor -%} {% endfor -%} - - - + + + # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} @@ -90,7 +90,7 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -109,4 +109,4 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: - .yamato/console-standalone-test.yml#console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 1b58afc459..32aeef21d3 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -44,7 +44,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }} artifacts: players: paths: @@ -53,7 +53,8 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ paths: - "artifacts/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -79,7 +80,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} artifacts: logs: paths: diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index e7c270948e..55fa6025b7 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -4,13 +4,13 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for Mobile platform test validation. # Those tests cover both PlayMode and EditMode tests from package test assemblies. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For all mobile platform (Android, iOS) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) # 3. For the default project. - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For mobile devices a split is required into two phases: # 1. Build Phase: Creates standalone players for mobile platforms @@ -18,7 +18,7 @@ # The Run phase uses build job as dependency # Note: More of a Unity specific but test assemblies need to be included in the build phase command # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - + # PLATFORM SPECIFICS-------------------------------------------------------------------- # iOS Requirements: # Must use IL2CPP scripting backend @@ -29,12 +29,12 @@ # Uses IL2CPP scripting backend (recommended over Mono) # Supports both ARM64 and ARMv7 architectures # Can be build on any desktop platform - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - + + # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_build -%} @@ -48,14 +48,14 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} - # Automatic UI interruption handling is available for iPhones running iOS 15 and above (models SE-Gen3 and 13). + # Automatic UI interruption handling is available for iPhones running iOS 15 and above (models SE-Gen3 and 13). # It is enabled by default when using those devices. Otherwise, system alerts (e.g. “Local Network Access” permission alert, introduced in iOS 14) might cause disruptions during test execution. # If building of the test app is done on a separate (“Build”) job, please make sure that that job has environment variable UNITY_HANDLEUIINTERRUPTIONS set to 1. variables: UNITY_HANDLEUIINTERRUPTIONS: 1 commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }} artifacts: players: paths: @@ -66,8 +66,8 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endfor -%} {% endfor -%} {% endfor -%} - - + + # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} @@ -84,7 +84,7 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: commands: # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - + {% if platform.standalone == "Android" %} # Download standalone UnityTestRunner and ADB setup - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey.pub -O %USERPROFILE%/.android/adbkey.pub @@ -97,11 +97,11 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: # Run tests for Android devices - | set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP% - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 + UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} {% else %} # Run tests for non-Android devices - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 --device-id=%BOKKEN_DEVICE_ID% + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} --device-id=%BOKKEN_DEVICE_ID% {% endif %} artifacts: logs: diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index c6c0643889..94c940b74b 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -7,6 +7,11 @@ # The job performs additional validation by using Package Verification Pipeline (PVP). It includes x-ray validation for quick package structure verification # Because of that validation we can detect errors at the early stage of testing so not to waste CI resources +# NOTES-------------------------------------------------------------------------------- + # For NGO we use package_pack everywhere rather than project_pack + # This is because only the NGO package is being referenced as a file inside testproject + # This means both project_pack and package_pack produce same package artifacts + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) @@ -29,7 +34,7 @@ package_pack_-_ngo_{{ platform.name }}: agent: type: {{ platform.type }} image: {{ platform.image }} - flavor: {{ platform.flavor }} + flavor: {{ platform.smaller_flavor }} {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index cbadbfba44..e9df795f10 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -5,23 +5,23 @@ # This job is responsible for execution of package-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from package test assemblies # Additionally it combines Package Verification Pipeline (PVP) validation. This ensures that package is compatible with Unity standards - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds required) # No scripting backend variations needed (Editor context) # Architecture variations not applicable (Editor context) # Requires project packaging as prerequisite (dependency job) # Uses PVP for package validation. Specifically it looks for supported profiles which we should conform to but takes ./pvpExceptions.json file into consideration where we note our known issues related to PVP checks - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file # TODO: we should aim to replace target PVP profile from supported to gold - + #------------------------------------------------------------------------------------ {% for platform in test_platforms.desktop -%} @@ -31,7 +31,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: agent: type: {{ platform.type }} image: {{ platform.image }} - flavor: {{ platform.flavor }} + flavor: {{ platform.smaller_flavor }} {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} @@ -40,14 +40,14 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: UNITY_EXT_LOGGING: 1 commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. - + # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results - upm-pvp require {% if platform.name == "win" %}"%XRAY_PROFILE%"{% else %}"$XRAY_PROFILE"{% endif %} --results pvp-results # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --reruncount=1 --clean-library-on-rerun + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} artifacts: logs: paths: @@ -57,4 +57,4 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index 26bf8102f4..8e61084715 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -5,21 +5,21 @@ # This job is responsible for executing performance tests for NGO package. # Its configuration is set to not report any data but just to give results (at least in current state since we don't have any tests to run). # Currently because of lack of performance tests this job will always return "no tests have been selected" and because oif that it's not included in any trigger jobs. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (For NGOv2.X it means 6000+) # 3. For the default project (project is used only as a context for the build). TODO-comment: if performance tests would be included in projects then we should make an approperiate split. - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # Tests are run in Editor context only # No performance metrics are reported to monitoring systems - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # TODO: Currently NGO don't have any performance tests so this job is a placeholder for the future. We should discuss how to approach the topic of performance testing # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #------------------------------------------------------------------------------------ {% for platform in test_platforms.desktop -%} @@ -36,7 +36,7 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin {% endif %} commands: - unity-downloader-cli -u {{ editor }} -c Editor --wait {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor - - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata + - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --dontreportperformancedata # TODO: when performance tests will be present we need to add actuall execution of this test artifacts: logs: @@ -44,4 +44,4 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin - "artifacts/**/*" {% endfor -%} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml deleted file mode 100644 index 647d395bde..0000000000 --- a/.yamato/project-pack.yml +++ /dev/null @@ -1,45 +0,0 @@ -{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. ---- - -# DESCRIPTION-------------------------------------------------------------------------- - # This job is responsible for packing a specific project. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. - # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. - -# CONFIGURATION STRUCTURE-------------------------------------------------------------- - # Jobs configurations are generated using nested loops through: - # 1. For all projects (testproject, minimalproject, testproject-tools-integration). - # 2. For all desktop platforms (Win, Ubuntu, Mac) - -# TECHNICAL CONSIDERATIONS-------------------------------------------------------------------- - # Job does not require Unity Editor in order to perform packing. - # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. - -# QUALITY CONSIDERATIONS-------------------------------------------------------------------- - # To see where this job is included (in trigger job definitions) look into _triggers.yml file - # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp. Additionally this would allow us to run PVP checks on projects - - -#-------------------------------------------------------------------------------------- - -{% for project in projects.all -%} -{% for platform in test_platforms.desktop -%} -project_pack_-_{{ project.name }}_{{ platform.name }}: - name: Project Pack - {{ project.name }} [{{ platform.name }}] - agent: - type: {{ platform.type }} - image: {{ platform.image }} - flavor: {{ platform.flavor }} -{% if platform.model %} - model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) -{% endif %} - commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - - upm-ci project pack --project-path {{ project.path }} - dependencies: - - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - artifacts: - packages: - paths: - - "upm-ci~/packages/**/*" -{% endfor -%} -{% endfor -%} \ No newline at end of file diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index a39f957480..e00c832fe1 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -4,24 +4,23 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job executes project-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from project test assemblies - # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) - + # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) ?? + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all projects WITH TESTS (filtered by has_tests flag) (testproject, testproject-tools-interation) [For more info look into project.metafile configuration] # 2. For all desktop platforms (Windows, Ubuntu, macOS) # 3. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds is required) # No scripting backend variations needed (Editor context) # Architecture variations not applicable (Editor context) # Requires project packaging as prerequisite (dependency job) - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- - # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #------------------------------------------------------------------------------------ {% for project in projects.all -%} @@ -38,16 +37,16 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} artifacts: logs: paths: - - "upm-ci~/test-results/**/*" + - "test-results/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} {% endif -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index 19ae953e27..7d749ce18a 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -4,7 +4,7 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible fo validating package compatibility with latest dependency versions. # This helps detect potential breaking changes from dependency updates early - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all projects (testproject, minimalproject, testproject-tools-integration). @@ -15,17 +15,17 @@ # This job requires successful project packaging before execution (job dependency) # This job tests only NGO package dependencies (com.unity.netcode.gameobjects) # The results are being generated in upm-ci~/test-results directory (specific of upm-ci) - + # QUALITY CONSIDERATIONS--------------------------------------------------------------------- # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp # To see where this job is included (in trigger job definitions) look into _triggers.yml file - -#-------------------------------------------------------------------------------------- - + +#-------------------------------------------------------------------------------------- + {% for project in projects.all -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} -updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: +updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: name : Updated Dependencies Test - NGO {{ project.name }} [{{ platform.name }}, {{ editor }}] agent: type: {{ platform.type }} @@ -42,7 +42,7 @@ updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: paths: - "upm-ci~/test-results/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index d6c93e86ff..10d3ecd736 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -9,6 +9,7 @@ # type --> Specifies the Bokken agent type (e.g., Unity::VM, Unity::VM::osx, Unity::mobile::shield) # image --> Defines the package-ci Bokken image to use for the environment (e.g., package-ci/ubuntu-22.04:v4). This is basically a device configuration # flavor --> Determines the VM size/resources (e.g., b1.small, b1.large, m1.mac) + # smaller_flavor --> An override for flavor that determines the VM size/resources for lighter weight jobs that can run on a smaller vm # standalone --> Specifies the build target platform (e.g., StandaloneLinux64, Android, IOS) # model --> Defines specific hardware model requirements (e.g., rtx2080, iPhone model 13). Notice that trunk currently (19.08.2025) has 13.0 as minimal iOS version which devices below this are not supporting # base --> Indicates the base operating system for build operations (e.g., win, mac) @@ -45,18 +46,21 @@ test_platforms: type: Unity::VM image: package-ci/ubuntu-22.04:v4 flavor: b1.large + smaller_flavor: b1.medium standalone: StandaloneLinux64 model: rtx2080 - name: win type: Unity::VM image: package-ci/win10:v4 flavor: b1.large + smaller_flavor: b1.medium standalone: StandaloneWindows64 model: rtx2080 - name: mac type: Unity::VM::osx image: package-ci/macos-13-arm64:v4 # ARM64 to support M1 model (below) flavor: m1.mac + smaller_flavor: m1.mac # mac doesn't have a smaller vm size. We define it anyway as it simplifies the yaml templating to have it defined. standalone: StandaloneOSX model: M1 # The default model (an x64 Intel Mac VM) quite often caused a known issue of doing all the bitflips in packages resulting in failures # For mobile devices there is a split between the build and run phase so there is a need of splitting specification for both @@ -187,3 +191,16 @@ projects: - name: minimalproject path: minimalproject has_tests: false + + +# UNIFIED TEST RUNNER CONFIGURATIONS-------------------------------------------------------------------- +# Shared settings for the UnifiedTestRunner + +# Configures the number of times a failed test is retried before it is considered failed. +# Helps handle flaky tests +# Passed into --retry +num_test_retries: 1 + +# Number of seconds before a single test times out +# Passed into --timeout +test_timeout: 1800 diff --git a/.yamato/vetting-test.yml b/.yamato/vetting-test.yml index d58f03e817..5d26deb39e 100644 --- a/.yamato/vetting-test.yml +++ b/.yamato/vetting-test.yml @@ -8,7 +8,7 @@ {% for editor in validation_editors.minimal -%} vetting_test: name: NGO - Vetting Test (Win, {{editor}} LTS) - agent: { type: Unity::VM, flavor: b1.large, image: package-ci/win11:v4 } + agent: { type: Unity::VM, flavor: b1.medium, image: package-ci/win11:v4 } commands: - npm install -g "upm-ci-utils@stable" --registry https://artifactory.prd.it.unity3d.com/artifactory/api/npm/upm-npm - unity-downloader-cli --fast --wait --unity-version {{ editor }} --components editor --arch x64 diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index c3d4f65376..1f028ce10f 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -2,28 +2,28 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This job is responsible for validating a successful building of a player on WebGl standalone platform. + # This job is responsible for validating a successful building of a player on WebGl standalone platform. # This job WILL NOT execute any tests, we only validate the building process. - # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. - + # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For the default project (project is used only as a context for the build). # 2. For all desktop platforms (Windows, Ubuntu, macOS) # 3. For all supported Unity Editor versions (For NGOv2.X it means 6000+) - + # TECHNICAL CONSIDERATIONS---------------------------------------------------------------- # WebGL requires IL2CPP scripting backend (Mono is not supported) # We are not using ARM64 architectures since we only perform a build action. x64 architectures are preferred for build phase (in order to optimize available resource usage) # We only perform build validation (no runtime testing) - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # In the future we could try to implement an infrastructure to run test in webgl context but this could be quite complicated and would need to be evaluated if it's worth it # To see where this job is included (in trigger job definitions) look into _triggers.yml file # WebGL jobs were timing up more often in develop-2.0.0 branch (especially on trunk) so we increased the timeout from 1800 to 3600. We should investigate why it times more often in develop-2.0.0/ubuntu/trunk - -#-------------------------------------------------------------------------------------- - + +#-------------------------------------------------------------------------------------- + {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} @@ -38,14 +38,14 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c webgl -c il2cpp {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloading the editor with additional webgl and il2cpp components - + # The following step builds the player with defined options such as: - # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) - # Editor is run in batchmode, which means that Unity runs command line arguments without the need for human interaction. It also suppresses pop-up windows that require human interaction (such as the Save Scene window). We should always run Unity in batch mode when using command line arguments, because it allows automation to run without interruption. (--extra-editor-arg=-batchmode) - # Engine is initialized in ‘nographics’ mode since we don't need any graphics for this case (--extra-editor-arg=-nographics) - # In case of failure the job will be rerunned once (--reruncount=1) with clean library (--clean-library-on-rerun) - # This will perform only building phase (--build-only) with a timeout of 3m (--timeout=1800) - - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=3600 + # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) + # Editor is run in batchmode, which means that Unity runs command line arguments without the need for human interaction. It also suppresses pop-up windows that require human interaction (such as the Save Scene window). We should always run Unity in batch mode when using command line arguments, because it allows automation to run without interruption. (--extra-editor-arg=-batchmode) + # Engine is initialized in ‘nographics’ mode since we don't need any graphics for this case (--extra-editor-arg=-nographics) + # In case of failure the job will be rerunned once (--reruncount=1) with clean library (--clean-library-on-rerun). Note that this is build only step so no tests are being executed + # This will perform only building phase (--build-only) with a timeout set by the test_timeout variable in the project metafile. + - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }} artifacts: logs: paths: diff --git a/testproject/Assets/Resources.meta b/testproject/Assets/Resources.meta deleted file mode 100644 index 52fd5df24a..0000000000 --- a/testproject/Assets/Resources.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 3f61bb7a9cfe67241ac932bf19d039c4 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs b/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs index b641a1a3e6..ddccd7c029 100644 --- a/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs +++ b/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs @@ -1,6 +1,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Text; using NUnit.Framework; using Unity.Netcode; using Unity.Netcode.TestHelpers.Runtime; @@ -26,12 +27,7 @@ public class NetworkSceneManagerEventNotifications : NetcodeIntegrationTest private LoadSceneMode m_LoadSceneMode; private bool m_CanStartServerOrClients = false; private bool m_LoadEventCompleted = false; - - // TODO: [CmbServiceTests] Adapt to run with the service - protected override bool UseCMBService() - { - return false; - } + private NetworkManager m_ClientToTestLoading; internal class SceneTestInfo { @@ -78,11 +74,16 @@ protected override IEnumerator OnTearDown() protected override IEnumerator OnStartedServerAndClients() { - m_ServerNetworkManager.SceneManager.OnSceneEvent += ServerSceneManager_OnSceneEvent; - foreach (var client in m_ClientNetworkManagers) + foreach (var manager in m_NetworkManagers) { - client.SceneManager.ClientSynchronizationMode = m_LoadSceneMode; - client.SceneManager.OnSceneEvent += ClientSceneManager_OnSceneEvent; + if (manager.IsServer || manager.LocalClient.IsSessionOwner) + { + manager.SceneManager.OnSceneEvent += ServerSceneManager_OnSceneEvent; + continue; + } + + manager.SceneManager.ClientSynchronizationMode = m_LoadSceneMode; + manager.SceneManager.OnSceneEvent += ClientSceneManager_OnSceneEvent; } return base.OnStartedServerAndClients(); } @@ -94,9 +95,10 @@ private void ClientSceneManager_OnSceneEvent(SceneEvent sceneEvent) // Validate that the clients finish synchronization and they used the proper synchronization mode case SceneEventType.SynchronizeComplete: { - var matchedClient = m_ClientNetworkManagers.Where(c => c.LocalClientId == sceneEvent.ClientId); - Assert.True(matchedClient.Count() > 0, $"Found no client {nameof(NetworkManager)}s that had a {nameof(NetworkManager.LocalClientId)} of {sceneEvent.ClientId}"); - Assert.AreEqual(matchedClient.First().SceneManager.ClientSynchronizationMode, m_ServerNetworkManager.SceneManager.ClientSynchronizationMode); + var authority = GetAuthorityNetworkManager(); + var matchedClient = m_ClientNetworkManagers.FirstOrDefault(c => c.LocalClientId == sceneEvent.ClientId); + Assert.That(matchedClient, Is.Not.Null, $"Found no client {nameof(NetworkManager)}s that had a {nameof(NetworkManager.LocalClientId)} of {sceneEvent.ClientId}"); + Assert.AreEqual(matchedClient.SceneManager.ClientSynchronizationMode, authority.SceneManager.ClientSynchronizationMode); break; } } @@ -104,6 +106,7 @@ private void ClientSceneManager_OnSceneEvent(SceneEvent sceneEvent) private void ServerSceneManager_OnSceneEvent(SceneEvent sceneEvent) { + var authority = GetAuthorityNetworkManager(); VerboseDebug($"[SceneEvent] ClientId:{sceneEvent.ClientId} | EventType: {sceneEvent.SceneEventType}"); switch (sceneEvent.SceneEventType) { @@ -126,16 +129,16 @@ private void ServerSceneManager_OnSceneEvent(SceneEvent sceneEvent) } case SceneEventType.LoadComplete: { - if (sceneEvent.ClientId == NetworkManager.ServerClientId) + if (sceneEvent.ClientId == authority.LocalClientId) { var scene = sceneEvent.Scene; m_CurrentScene = scene; } - if (sceneEvent.ClientId == m_ClientNetworkManagers[0].LocalClientId) + if (sceneEvent.ClientId == m_ClientToTestLoading.LocalClientId) { if (!m_ScenesLoaded.Contains(sceneEvent.SceneName)) { - Debug.Log($"Loaded {sceneEvent.SceneName}"); + VerboseLog($"Loaded {sceneEvent.SceneName}"); m_ScenesLoaded.Add(sceneEvent.SceneName); } } @@ -162,10 +165,10 @@ private void ServerSceneManager_OnSceneEvent(SceneEvent sceneEvent) // If we are a server and this is being processed by the server, then add the server to the completed list // to validate that the event completed on all clients (and the server). - if (!m_ServerNetworkManager.IsHost && sceneEvent.ClientId == m_ServerNetworkManager.LocalClientId && - !sceneEvent.ClientsThatCompleted.Contains(m_ServerNetworkManager.LocalClientId)) + if (!authority.IsHost && sceneEvent.ClientId == authority.LocalClientId && + !sceneEvent.ClientsThatCompleted.Contains(authority.LocalClientId)) { - sceneEvent.ClientsThatCompleted.Add(m_ServerNetworkManager.LocalClientId); + sceneEvent.ClientsThatCompleted.Add(authority.LocalClientId); } if (sceneEvent.SceneEventType == SceneEventType.LoadEventCompleted) { @@ -180,16 +183,16 @@ private void ServerSceneManager_OnSceneEvent(SceneEvent sceneEvent) } private void SceneManager_OnSceneEvent(SceneEvent sceneEvent) { - Debug.Log($"[SceneEvent] ClientId:{sceneEvent.ClientId} | EventType: {sceneEvent.SceneEventType}"); + VerboseLog($"[SceneEvent] ClientId:{sceneEvent.ClientId} | EventType: {sceneEvent.SceneEventType}"); switch (sceneEvent.SceneEventType) { case SceneEventType.LoadComplete: { - if (sceneEvent.ClientId == m_ClientNetworkManagers[0].LocalClientId) + if (sceneEvent.ClientId == m_ClientToTestLoading.LocalClientId) { if (!m_ScenesLoaded.Contains(sceneEvent.SceneName)) { - Debug.Log($"Loaded {sceneEvent.SceneName}"); + VerboseLog($"Loaded {sceneEvent.SceneName}"); m_ScenesLoaded.Add(sceneEvent.SceneName); } } @@ -198,11 +201,11 @@ private void SceneManager_OnSceneEvent(SceneEvent sceneEvent) case SceneEventType.UnloadComplete: { - if (sceneEvent.ClientId == m_ClientNetworkManagers[0].LocalClientId) + if (sceneEvent.ClientId == m_ClientToTestLoading.LocalClientId) { if (m_ScenesLoaded.Contains(sceneEvent.SceneName)) { - Debug.Log($"Unloaded {sceneEvent.SceneName}"); + VerboseLog($"Unloaded {sceneEvent.SceneName}"); // We check here for single mode because the final scene event // will be SceneEventType.LoadEventCompleted (easier to trap for it here) m_ScenesLoaded.Remove(sceneEvent.SceneName); @@ -226,26 +229,29 @@ protected override bool CanStartServerAndClients() [UnityTest] public IEnumerator SceneLoadingAndNotifications([Values] LoadSceneMode loadSceneMode) { + var authority = GetAuthorityNetworkManager(); + var nonAuthority = GetNonAuthorityNetworkManager(); + m_ClientToTestLoading = nonAuthority; m_LoadSceneMode = loadSceneMode; m_CurrentSceneName = k_SceneToLoad; m_CanStartServerOrClients = true; yield return StartServerAndClients(); - yield return WaitForConditionOrTimeOut(() => m_ClientsReceivedSynchronize.Count == (m_ClientNetworkManagers.Length)); - Assert.False(s_GlobalTimeoutHelper.TimedOut, $"Timed out waiting for all clients to receive synchronization event! Received: {m_ClientsReceivedSynchronize.Count} | Expected: {m_ClientNetworkManagers.Length}"); + yield return WaitForConditionOrTimeOut(() => m_ClientsReceivedSynchronize.Count == NumberOfClients); + AssertOnTimeout($"Timed out waiting for all clients to receive synchronization event! Received: {m_ClientsReceivedSynchronize.Count} | Expected: {NumberOfClients}"); if (loadSceneMode == LoadSceneMode.Single) { - m_ClientNetworkManagers[0].SceneManager.OnSceneEvent += SceneManager_OnSceneEvent; + m_ClientToTestLoading.SceneManager.OnSceneEvent += SceneManager_OnSceneEvent; } // Now prepare for the scene testing InitializeSceneTestInfo(); // Test loading scenes and the associated event messaging and notification pipelines ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.Started); + Assert.AreEqual(authority.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.Started); // Check error status for trying to load during an already in progress scene event - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.SceneEventInProgress); + Assert.AreEqual(authority.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.SceneEventInProgress); // Wait for all clients to load the scene yield return WaitForConditionOrTimeOut(ConditionPassed); @@ -262,7 +268,7 @@ public IEnumerator SceneLoadingAndNotifications([Values] LoadSceneMode loadScene m_CurrentSceneName = k_InSceneNetworkObject; ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(k_InSceneNetworkObject, LoadSceneMode.Additive), SceneEventProgressStatus.Started); + Assert.AreEqual(authority.SceneManager.LoadScene(k_InSceneNetworkObject, LoadSceneMode.Additive), SceneEventProgressStatus.Started); // Wait for all clients to additively load this additional scene yield return WaitForConditionOrTimeOut(ConditionPassed); @@ -272,40 +278,39 @@ public IEnumerator SceneLoadingAndNotifications([Values] LoadSceneMode loadScene // Now single mode load a new scene (i.e. "scene switch") m_CurrentSceneName = k_BaseUnitTestSceneName; ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.Started); + Assert.AreEqual(authority.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.Started); // Wait for all clients to perform scene switch yield return WaitForConditionOrTimeOut(ConditionPassed); AssertOnTimeout($"Timed out waiting for all clients to switch to scene {m_CurrentSceneName}!"); // Make sure the server scene is the active scene SceneManager.SetActiveScene(m_CurrentScene); - yield return WaitForConditionOrTimeOut(() => !m_ScenesLoaded.Contains(k_SceneToLoad) && !m_ScenesLoaded.Contains(k_InSceneNetworkObject)); - var additionalInfo = string.Empty; - if (s_GlobalTimeoutHelper.TimedOut) + var helper = new TimeoutHelper(); + yield return WaitForConditionOrTimeOut(() => !m_ScenesLoaded.Contains(k_SceneToLoad) && !m_ScenesLoaded.Contains(k_InSceneNetworkObject), helper); + var additionalInfo = new StringBuilder(); + if (helper.TimedOut) { - foreach (var sceneName in m_ScenesLoaded) - { - additionalInfo += $"{sceneName},"; - } - Debug.Break(); + additionalInfo.Append("Scenes currently loaded: ["); + additionalInfo.AppendJoin(", ", m_ScenesLoaded); + additionalInfo.Append("]"); } - AssertOnTimeout($"{nameof(m_ScenesLoaded)} still contains some of the scenes that were expected to be unloaded!\n {additionalInfo}"); + AssertOnTimeout($"{nameof(m_ScenesLoaded)} still contains some of the scenes that were expected to be unloaded!\n {additionalInfo}", helper); } // Test unloading additive scenes and the associated event messaging and notification pipelines ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.UnloadScene(m_CurrentScene), SceneEventProgressStatus.Started); + Assert.AreEqual(authority.SceneManager.UnloadScene(m_CurrentScene), SceneEventProgressStatus.Started); yield return WaitForConditionOrTimeOut(ConditionPassed); AssertOnTimeout($"Timed out waiting for all clients to unload {m_CurrentSceneName}!"); // Check error status for trying to unloading something not loaded ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.UnloadScene(m_CurrentScene), SceneEventProgressStatus.SceneNotLoaded); + Assert.AreEqual(authority.SceneManager.UnloadScene(m_CurrentScene), SceneEventProgressStatus.SceneNotLoaded); // Check error status for trying to load an invalid scene name LogAssert.Expect(LogType.Error, $"Scene '{k_InvalidSceneName}' couldn't be loaded because it has not been added to the build settings scenes in build list."); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(k_InvalidSceneName, LoadSceneMode.Additive), SceneEventProgressStatus.InvalidSceneName); + Assert.AreEqual(authority.SceneManager.LoadScene(k_InvalidSceneName, LoadSceneMode.Additive), SceneEventProgressStatus.InvalidSceneName); } @@ -329,9 +334,7 @@ private void ResetWait() /// private void InitializeSceneTestInfo() { - m_ShouldWaitList.Add(new SceneTestInfo() { ClientId = NetworkManager.ServerClientId, ShouldWait = false }); - - foreach (var manager in m_ClientNetworkManagers) + foreach (var manager in m_NetworkManagers) { m_ShouldWaitList.Add(new SceneTestInfo() { ClientId = manager.LocalClientId, ShouldWait = false }); } @@ -343,12 +346,20 @@ private void InitializeSceneTestInfo() /// private bool ConditionPassed() { - var completed = true; - if (m_LoadSceneMode == LoadSceneMode.Single) + if (m_LoadSceneMode == LoadSceneMode.Single && !m_LoadEventCompleted) { - completed = m_LoadEventCompleted; + return false; } - return completed && !(m_ShouldWaitList.Select(c => c).Where(c => c.ProcessedEvent != true && c.ShouldWait == true).Count() > 0); + + foreach (var client in m_ShouldWaitList) + { + if (!client.ProcessedEvent || client.ShouldWait) + { + return false; + } + } + + return true; } /// @@ -356,7 +367,15 @@ private bool ConditionPassed() /// private bool ContainsClient(ulong clientId) { - return m_ShouldWaitList.Select(c => c.ClientId).Where(c => c == clientId).Count() > 0; + foreach (var client in m_ShouldWaitList) + { + if (client.ClientId == clientId) + { + return true; + } + } + + return false; } /// @@ -364,7 +383,13 @@ private bool ContainsClient(ulong clientId) /// private void SetClientProcessedEvent(ulong clientId) { - m_ShouldWaitList.Select(c => c).Where(c => c.ClientId == clientId).First().ProcessedEvent = true; + foreach (var client in m_ShouldWaitList) + { + if (client.ClientId == clientId) + { + client.ProcessedEvent = true; + } + } } /// @@ -372,9 +397,13 @@ private void SetClientProcessedEvent(ulong clientId) /// private void SetClientWaitDone(List clients) { - foreach (var clientId in clients) + var lookup = clients.ToHashSet(); + foreach (var client in m_ShouldWaitList) { - m_ShouldWaitList.Select(c => c).Where(c => c.ClientId == clientId).First().ShouldWait = false; + if (lookup.Contains(client.ClientId)) + { + client.ShouldWait = false; + } } } @@ -401,5 +430,13 @@ private bool ContainsAllClients(List clients) } return true; } + + private void VerboseLog(string message) + { + if (OnSetVerboseDebug()) + { + Debug.Log(message); + } + } } }