diff --git a/.yamato/README.md b/.yamato/README.md index 4838352f26..fed424413e 100644 --- a/.yamato/README.md +++ b/.yamato/README.md @@ -9,7 +9,6 @@ CI related files are present inside .yamato/ folder and we can distinguish speci ### Helper jobs - `.yamato/package-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. -- `.yamato/project-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. This packs all packages of a given project. - `.yamato/_run-all.yml` responsible for grouping tests into groups for easier management (for example "all console tests"). - `.yamato/_triggers.yml` responsible for defining triggers (PR, nightly, weekly etc.) and defining which tests to run. - `disable-burst-if-requested.py` responsible for helping to disable burst if needed. @@ -77,4 +76,4 @@ Currently, the CI implementation supports the following platforms: ## Design Considerations In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. -The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. \ No newline at end of file +The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index 194f162878..4eda4fb597 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -17,8 +17,6 @@ run_quick_checks: dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_ubuntu - .yamato/project-standards.yml#standards_ubuntu_testproject_6000.2 - # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version - - .yamato/vetting-test.yml#vetting_test # Runs all package tests run_all_package_tests: diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index 9624533cb4..2483e600d0 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -41,7 +41,7 @@ # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job. -#----------------------------------------------------------------------------------- +#----------------------------------------------------------------------------------- # After some experimenting with CI setups we discovered that even though sometimes we don't need CI to run (no reason to run package tests if only Documentation is changed) there are some checks that devs may not realize but changes in seemingly unrelated files will cause their failures # This trigger was created to ensure that ALL PRs run this minimal check even when we don't need to run full tests @@ -69,6 +69,9 @@ pr_code_changes_checks: name: Code changes PR checks # Run the following tests on a selection of different desktop platforms dependencies: + # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version + - .yamato/vetting-test.yml#vetting_test + # Run package EditMode and Playmode package tests on 6000.2 (latest supported editor) and an older supported editor (2022.3) (2022.3 will soon be a minimum supported editor) - .yamato/package-tests.yml#package_test_-_ngo_6000.2_mac - .yamato/package-tests.yml#package_test_-_ngo_2022.3_win @@ -104,9 +107,9 @@ pr_code_changes_checks: "**/*.md" ] cancel_old_ci: true - - - + + + @@ -122,6 +125,8 @@ develop_nightly: frequency: daily rerun: always dependencies: + # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version + - .yamato/vetting-test.yml#vetting_test # Run project standards to verify package/default project - .yamato/project-standards.yml#standards_ubuntu_testproject_6000.2 - .yamato/project-standards.yml#standards_ubuntu_testproject_2022.3 @@ -164,6 +169,8 @@ develop_weekly_trunk: frequency: weekly rerun: always dependencies: + # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version + - .yamato/vetting-test.yml#vetting_test # Run project standards to verify package/default project - .yamato/_run-all.yml#run_all_projects_standards # Run package EditMode and Playmode tests on desktop platforms diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 5702f905c1..8449d3114b 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -8,22 +8,22 @@ # Line and branch coverage statistics # Generated HTML reports for coverage visualization # Additional metrics for coverage analysis - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) # 2. For default editor version (6000.2) since coverage would not vary between editors (no need for checks on more editors) - + #TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # In theory this job also runs package tests, but we don't want to use it as default since is heavier (because of added coverage analysis) and coverage is not changing that often # Requires Unity Editor installation # Burst compilation is disabled to ensure accurate coverage measurement # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise, coverage results will not be visible - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - + + {% for platform in test_platforms.default -%} {% for editor in validation_editors.default -%} code_coverage_{{ platform.name }}_{{ editor }}: @@ -38,7 +38,7 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv1_{{ platform.name }}_{{ editor }};flags:NGOv1_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv1_{{ platform.name }}_{{ editor }};flags:NGOv1_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout={ test_timeout }} --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: @@ -46,4 +46,4 @@ code_coverage_{{ platform.name }}_{{ editor }}: dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_ubuntu {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index fa0e71885f..31f858fe07 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -4,22 +4,22 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for Console platform test validation. # Those tests cover both PlayMode and EditMode tests from package test assemblies. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) # 2. For all supported Unity Editor versions (for NGOv1.X this means 2022.3+ editors) # 3. For the default project. - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For console devices a split is required into two phases: # 1. Build Phase: Creates standalone players for console platforms # 2. Run Phase: Executes runtime tests on actual console devices # The Run phase uses build job as dependency - + # Note: More of a Unity specific but test assemblies need to be included in the build phase command # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - + # PLATFORM SPECIFICS----------------------------------------------------------------- # Common Requirements: # All consoles require IL2CPP scripting backend @@ -29,13 +29,13 @@ # Switch: ARM64 architecture only # Other Consoles: x64 architecture # Each console requires specific SDK paths and tools - + # QUALITY THOUGHTS-------------------------------------------------------------------- # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - - + + + # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_build -%} @@ -51,7 +51,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout}} variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -72,9 +72,9 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endfor -%} {% endfor -%} {% endfor -%} - - - + + + # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} @@ -109,4 +109,4 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: - .yamato/console-standalone-test.yml#console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index b1b34c15e9..7f26534cf6 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -4,30 +4,30 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for Desktop platform test validation. # Those tests cover both PlayMode and EditMode tests from package test assemblies. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: # 1. For all desktop platform (Windows, macOS, Ubuntu) # 2. For all supported Unity Editor versions (for NGOv1.X this means 2022.3+ editors) # 3. For the default project. # 4. For all scripting backends (mono, il2cpp) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For desktop devices a split is into two phases is not required but we use it for consistency with setup of others standalone platforms: # 1. Build Phase: Creates standalone players for desktop platforms # 2. Run Phase: Executes runtime tests on actual desktop devices # The Run phase uses build job as dependency - + # Note: More of a Unity specific but test assemblies need to be included in the build phase command # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - + # QUALITY THOUGHTS-------------------------------------------------------------------- # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #----------------------------------------------------------------------------------- - - + + # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} @@ -53,15 +53,16 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ paths: - "artifacts/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} {% endfor -%} {% endfor -%} - - - - + + + + # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} @@ -78,7 +79,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} artifacts: logs: paths: diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 70152ececb..e5d9f8bcf2 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -4,13 +4,13 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for Mobile platform test validation. # Those tests cover both PlayMode and EditMode tests from package test assemblies. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For all mobile platform (Android, iOS) # 2. For all supported Unity Editor versions (for NGOv1.X this means 2022.3+ editors) # 3. For the default project. - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For mobile devices a split is required into two phases: # 1. Build Phase: Creates standalone players for mobile platforms @@ -18,23 +18,23 @@ # The Run phase uses build job as dependency # Note: More of a Unity specific but test assemblies need to be included in the build phase command # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - + # PLATFORM SPECIFICS-------------------------------------------------------------------- # iOS Requirements: # Must use IL2CPP scripting backend # Only supports ARM64 architecture # Builds on macOS agents only - + # Android Requirements: # Uses IL2CPP scripting backend (recommended over Mono) # Supports both ARM64 and ARMv7 architectures # Can be built on any desktop platform - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - + + # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_build -%} @@ -48,14 +48,14 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} - # Automatic UI interruption handling is available for iPhones running iOS 15 and above (models SE-Gen3 and 13). + # Automatic UI interruption handling is available for iPhones running iOS 15 and above (models SE-Gen3 and 13). # It is enabled by default when using those devices. Otherwise, system alerts (e.g. “Local Network Access” permission alert, introduced in iOS 14) might cause disruptions during test execution. # If building of the test app is done on a separate (“Build”) job, please make sure that that job has environment variable UNITY_HANDLEUIINTERRUPTIONS set to 1. variables: UNITY_HANDLEUIINTERRUPTIONS: 1 commands: - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --wait - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }} artifacts: players: paths: @@ -66,8 +66,8 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endfor -%} {% endfor -%} {% endfor -%} - - + + # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} @@ -84,24 +84,24 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: commands: # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - + {% if platform.standalone == "Android" %} # Download standalone UnityTestRunner and ADB setup - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey.pub -O %USERPROFILE%/.android/adbkey.pub - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey -O %USERPROFILE%/.android/adbkey - command: gsudo NetSh Advfirewall set allprofiles state off - + # Connect to Android device - command: '"%ANDROID_SDK_ROOT%\platform-tools\adb.exe" connect %BOKKEN_DEVICE_IP%' - + # Run tests for Android devices - | set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP% - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 + UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} {% else %} - + # Run tests for non-Android devices - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 --device-id=%BOKKEN_DEVICE_ID% + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} --device-id=%BOKKEN_DEVICE_ID% {% endif %} artifacts: logs: diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index 55a0f8e53c..a2afe80c08 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -7,6 +7,11 @@ # The job performs additional validation by using Package Verification Pipeline (PVP). It includes x-ray validation for quick package structure verification # Because of that validation we can detect errors at the early stage of testing so not to waste CI resources +# NOTES-------------------------------------------------------------------------------- + # For NGO we use package_pack everywhere rather than project_pack + # This is because only the NGO package is being referenced as a file inside testproject + # This means both project_pack and package_pack produce same package artifacts + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 3394bdc2a0..f41c0d1f6e 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -5,25 +5,25 @@ # This job is responsible for execution of package-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from package test assemblies # Additionally it combines Package Verification Pipeline (PVP) validation. This ensures that package is compatible with Unity standards - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (for NGOv1.X this means 2022.3+ editors) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds required) # No scripting backend variations needed (Editor context) # Architecture variations not applicable (Editor context) # Requires project packaging as prerequisite (dependency job) # Uses PVP for package validation. Specifically it looks for supported profiles which we should conform to but takes ./pvpExceptions.json file into consideration where we note our known issues related to PVP checks - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file # TODO: we should aim to replace target PVP profile from supported to gold - + #------------------------------------------------------------------------------------ - + {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} package_test_-_ngo_{{ editor }}_{{ platform.name }}: @@ -31,7 +31,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: agent: type: {{ platform.type }} image: {{ platform.image }} - flavor: {{ platform.flavor }} + flavor: {{ platform.smaller_flavor }} {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} @@ -40,14 +40,14 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: UNITY_EXT_LOGGING: 1 commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. - + # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results - upm-pvp require {% if platform.name == "win" %}"%XRAY_PROFILE%"{% else %}"$XRAY_PROFILE"{% endif %} --results pvp-results # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} artifacts: logs: paths: @@ -57,4 +57,4 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - .yamato/package-pack.yml#package_pack_-_ngo_ubuntu {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index b5e8cea458..63644f97b9 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -5,23 +5,23 @@ # This job is responsible for executing performance tests for NGO package. # Its configuration is set to not report any data but just to give results (at least in current state since we don't have any tests to run). # Currently, because of lack of performance tests this job will always return "no tests have been selected" and because oif that it's not included in any trigger jobs. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (For NGOv1.X it means 2022.3+) # 3. For the default project (project is used only as a context for the build). TODO-comment: if performance tests would be included in projects then we should make an appropriate split. - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # Tests are run in Editor context only # No performance metrics are reported to monitoring systems - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # TODO: Currently NGO don't have any performance tests so this job is a placeholder for the future. We should discuss how to approach the topic of performance testing # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #------------------------------------------------------------------------------------ - + {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} {% for project in projects.default -%} @@ -36,7 +36,7 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin {% endif %} commands: - unity-downloader-cli -u {{ editor }} -c Editor --wait {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor - - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata + - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --dontreportperformancedata # TODO: when performance tests will be present we need to add actuall execution of this test artifacts: logs: @@ -44,4 +44,4 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin - "artifacts/**/*" {% endfor -%} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml deleted file mode 100644 index d3da73ad24..0000000000 --- a/.yamato/project-pack.yml +++ /dev/null @@ -1,45 +0,0 @@ -{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. ---- - -# DESCRIPTION-------------------------------------------------------------------------- - # This job is responsible for packing a specific project. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. - # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. - -# CONFIGURATION STRUCTURE-------------------------------------------------------------- - # Jobs configurations are generated using nested loops through: - # 1. For all projects (testproject, minimalproject, testproject-tools-integration). - # 2. For all desktop platforms (Win, Ubuntu, Mac) - -# TECHNICAL CONSIDERATIONS-------------------------------------------------------------------- - # Job does not require Unity Editor in order to perform packing. - # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. - -# QUALITY CONSIDERATIONS-------------------------------------------------------------------- - # To see where this job is included (in trigger job definitions) look into _triggers.yml file - # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp. Additionally this would allow us to run PVP checks on projects - - -#-------------------------------------------------------------------------------------- - -{% for project in projects.all -%} -{% for platform in test_platforms.desktop -%} -project_pack_-_{{ project.name }}_{{ platform.name }}: - name: Project Pack - {{ project.name }} [{{ platform.name }}] - agent: - type: {{ platform.type }} - image: {{ platform.image }} - flavor: {{ platform.flavor }} -{% if platform.model %} - model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) -{% endif %} - commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - - upm-ci project pack --project-path {{ project.path }} - dependencies: - - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - artifacts: - packages: - paths: - - "upm-ci~/packages/**/*" -{% endfor -%} -{% endfor -%} \ No newline at end of file diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 45181874c9..e61ee790bc 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -5,25 +5,25 @@ # This job executes project-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from project test assemblies # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all projects WITH TESTS (filtered by has_tests flag) (testproject, testproject-tools-interation) [For more info look into project.metafile configuration] # 2. For all desktop platforms (Windows, Ubuntu, macOS) # 3. For all supported Unity Editor versions (for NGOv1.X this means 2022.3+ editors) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds is required) # No scripting backend variations needed (Editor context) # Architecture variations not applicable (Editor context) # Requires project packaging as prerequisite (dependency job) - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #------------------------------------------------------------------------------------ - + {% for project in projects.all -%} {% if project.has_tests == "true" -%} {% for platform in test_platforms.desktop -%} @@ -38,16 +38,15 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} artifacts: logs: paths: - "upm-ci~/test-results/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} -{% endfor -%} + - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}{% endfor -%} {% endfor -%} {% endif -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index 38671cf496..1b3175443f 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -4,24 +4,24 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible fo validating package compatibility with latest dependency versions. # This helps detect potential breaking changes from dependency updates early - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all projects (testproject, minimalproject, testproject-tools-integration). # 2. For all desktop platforms (Win, Ubuntu, Mac) # 3. For all supported editors (For NGOv1.X it means 2022.3+) - + # TECHNICAL CONSIDERATIONS---------------------------------------------------------------- # This job requires successful project packaging before execution (job dependency) # This job tests only NGO package dependencies (com.unity.netcode.gameobjects) # The results are being generated in upm-ci~/test-results directory (specific of upm-ci) - + # QUALITY CONSIDERATIONS--------------------------------------------------------------------- # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp # To see where this job is included (in trigger job definitions) look into _triggers.yml file - -#-------------------------------------------------------------------------------------- - + +#-------------------------------------------------------------------------------------- + {% for project in projects.all -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} @@ -42,7 +42,7 @@ updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: paths: - "upm-ci~/test-results/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index 313f5a0e2b..47a2f8f3ce 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -9,6 +9,7 @@ # type --> Specifies the Bokken agent type (e.g., Unity::VM, Unity::VM::osx, Unity::mobile::shield) # image --> Defines the package-ci Bokken image to use for the environment (e.g., package-ci/ubuntu-22.04:v4). This is basically a device configuration # flavor --> Determines the VM size/resources (e.g., b1.small, b1.large, m1.mac) + # smaller_flavor --> An override for flavor that determines the VM size/resources for lighter weight jobs that can run on a smaller vm # standalone --> Specifies the build target platform (e.g., StandaloneLinux64, Android, IOS) # model --> Defines specific hardware model requirements (e.g., rtx2080, iPhone model 13). Notice that trunk currently (19.08.2025) has 13.0 as minimal iOS version which devices below this are not supporting # base --> Indicates the base operating system for build operations (e.g., win, mac) @@ -19,10 +20,10 @@ # The small agent was created to handle jobs that don't involve running Unity and are in general super light when it comes to resource usage (for example pack job). # It uses ubuntu since Linux VMs are faster and cheaper to provision than Mac or Windows Virtual Machines (VMs). # The b1.small flavour is nearly always sufficient for jobs that don’t involve running Unity Editor. -small_agent_platform: +small_agent_platform: - name: ubuntu type: Unity::VM - image: package-ci/ubuntu-22.04:v4 + image: package-ci/ubuntu-22.04:v4.77.0 flavor: b1.small @@ -37,26 +38,29 @@ test_platforms: default: - name: ubuntu type: Unity::VM - image: package-ci/ubuntu-22.04:v4 + image: package-ci/ubuntu-22.04:v4.77.0 flavor: b1.large standalone: StandaloneLinux64 - desktop: + desktop: - name: ubuntu type: Unity::VM - image: package-ci/ubuntu-22.04:v4 + image: package-ci/ubuntu-22.04:v4.77.0 flavor: b1.large + smaller_flavor: b1.medium standalone: StandaloneLinux64 model: rtx2080 - name: win type: Unity::VM image: package-ci/win10:v4 flavor: b1.large + smaller_flavor: b1.medium standalone: StandaloneWindows64 model: rtx2080 - name: mac type: Unity::VM::osx image: package-ci/macos-13-arm64:v4 # ARM64 to support M1 model (below) - flavor: m1.mac + smaller_flavor: m1.mac # mac doesn't have a smaller vm size. We define it anyway as it simplifies the yaml templating to have it defined. + smaller_flavor: b1.medium standalone: StandaloneOSX model: M1 # The default model (an x64 Intel Mac VM) quite often caused a known issue of doing all the bitflips in packages resulting in failures # For mobile devices there is a split between the build and run phase so there is a need of splitting specification for both @@ -146,7 +150,7 @@ test_platforms: image: package-ci/win10-xbox:v4 flavor: b1.large standalone: GameCoreScarlett - + # EDITOR CONFIGURATIONS------------------------------------------------------------------------------- # Editors to be used for testing. NGOv1.X official support started from 2021.3 editor # TODO: When a new editor will be released it should be added to this list. Correct also if we will stop supporting one of the editors present here @@ -163,7 +167,7 @@ validation_editors: # Scripting backends used by Standalone RunTimeTests--------------------------------------------------- -scripting_backends: +scripting_backends: - mono - il2cpp @@ -188,3 +192,16 @@ projects: - name: testproject-tools-integration path: testproject-tools-integration has_tests: true + + +# UNIFIED TEST RUNNER CONFIGURATIONS-------------------------------------------------------------------- +# Shared settings for the UnifiedTestRunner + +# Configures the number of times a failed test is retried before it is considered failed. +# Helps handle flaky tests +# Passed into --retry +num_test_retries: 1 + +# Number of seconds before a single test times out +# Passed into --timeout +test_timeout: 1800 diff --git a/.yamato/vetting-test.yml b/.yamato/vetting-test.yml index d58f03e817..5d26deb39e 100644 --- a/.yamato/vetting-test.yml +++ b/.yamato/vetting-test.yml @@ -8,7 +8,7 @@ {% for editor in validation_editors.minimal -%} vetting_test: name: NGO - Vetting Test (Win, {{editor}} LTS) - agent: { type: Unity::VM, flavor: b1.large, image: package-ci/win11:v4 } + agent: { type: Unity::VM, flavor: b1.medium, image: package-ci/win11:v4 } commands: - npm install -g "upm-ci-utils@stable" --registry https://artifactory.prd.it.unity3d.com/artifactory/api/npm/upm-npm - unity-downloader-cli --fast --wait --unity-version {{ editor }} --components editor --arch x64 diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index 8abca34c5f..d258fa61a1 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -2,27 +2,27 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This job is responsible for validating a successful building of a player on WebGl standalone platform. + # This job is responsible for validating a successful building of a player on WebGl standalone platform. # This job WILL NOT execute any tests, we only validate the building process. - # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. - + # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For the default project (project is used only as a context for the build). # 2. For all desktop platforms (Windows, Ubuntu, macOS) # 3. For all supported Unity Editor versions (For NGOv1.X it means 2022.3+) - + # TECHNICAL CONSIDERATIONS---------------------------------------------------------------- # WebGL requires IL2CPP scripting backend (Mono is not supported) # We are not using ARM64 architectures since we only perform a build action. x64 architectures are preferred for build phase (in order to optimize available resource usage) # We only perform build validation (no runtime testing) - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # In the future we could try to implement an infrastructure to run test in webgl context but this could be quite complicated and would need to be evaluated if it's worth it # To see where this job is included (in trigger job definitions) look into _triggers.yml file - -#-------------------------------------------------------------------------------------- - + +#-------------------------------------------------------------------------------------- + {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} @@ -37,14 +37,14 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c webgl -c il2cpp {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloading the editor with additional webgl and il2cpp components - + # The following step builds the player with defined options such as: # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) # Editor is run in batchmode, which means that Unity runs command line arguments without the need for human interaction. It also suppresses pop-up windows that require human interaction (such as the Save Scene window). We should always run Unity in batch mode when using command line arguments, because it allows automation to run without interruption. (--extra-editor-arg=-batchmode) # Engine is initialized in ‘nographics’ mode since we don't need any graphics for this case (--extra-editor-arg=-nographics) - # In case of failure the job will be rerunned once (--reruncount=1) with clean library (--clean-library-on-rerun) - # This will perform only building phase (--build-only) with a timeout of 3m (--timeout=1800) - - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + # In case of failure the job will be rerunned once (--reruncount=1) with clean library (--clean-library-on-rerun). Note that this is build only step so no tests are being executed + # This will perform only building phase (--build-only) with a timeout set by the test_timeout variable in the project metafile. + - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }} artifacts: logs: paths: