chore: Add Standalone Tests for IL2CPP backend (#1923)

Author: ashwinimurt

.yamato/_run-all.yml

@@ -7,6 +7,7 @@ run_all_tests:
     - .yamato/_triggers.yml#badges_test_trigger
     - .yamato/mobile-build-and-test.yml#2021.2_Run_iOS_Player_With_Tests
     - .yamato/mobile-build-and-test.yml#2021.2_Run_Android_Player_With_Tests
+    # - .yamato/_run-all.yml#all_project_tests_standalone
 {% for platform in test_platforms -%}
 {% for project in projects -%}
 {% for editor in project.test_editors -%}
@@ -39,7 +40,6 @@ run_all_tests_trunk:
 all_project_tests:
   name: Run All Project Tests
   dependencies:
-    # Pull in package and validate jobs through the badges job
     - .yamato/_triggers.yml#badges_test_trigger
 {% for platform in test_platforms -%}
 {% for project in projects -%}
@@ -52,7 +52,6 @@ all_project_tests:
 all_package_tests:
   name: Run All Package Tests
   dependencies:
-    # Pull in package and validate jobs through the badges job
     - .yamato/_triggers.yml#badges_test_trigger
 {% for platform in test_platforms -%}
 {% for project in projects -%}
@@ -67,7 +66,6 @@ all_package_tests:
 all_singlenode_multiprocess_tests:
   name: Run All Multiprocess Tests - Single Node
   dependencies:
-    # Pull in package and validate jobs through the badges job
 {% for platform in test_platforms -%}
 {% for project in projects -%}
 {% for editor in project.test_editors -%}
@@ -77,3 +75,18 @@ all_singlenode_multiprocess_tests:
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
+
+all_project_tests_standalone:
+  name: Run All Project Tests - Standalone
+  dependencies:
+{% for platform in test_platforms -%}
+{% for project in projects -%}
+{% if project.has_tests == "true" -%}
+{% for editor in project.test_editors -%}
+{% for backend in scripting_backends -%}
+    - .yamato/standalone-project-tests.yml#standalone_tests_{{ project.name }}_{{ backend }}_{{ editor }}_{{ platform.name }}
+{% endfor -%}
+{% endfor -%}
+{% endif -%}
+{% endfor -%}
+{% endfor -%}

.yamato/_triggers.yml

@@ -1,7 +1,7 @@
 {% metadata_file .yamato/project.metafile %}
 ---
 develop_nightly:
-  name: "[Nightly] Run All Tests"
+  name: "\U0001F319 [Nightly] Run All Tests"
   triggers:
     recurring:
     - branch: develop
@@ -14,7 +14,7 @@ develop_nightly:
 {% endfor -%}
 
 develop_weekly_trunk:
-  name: "[Weekly] Run All Tests [Trunk]"
+  name: "\U0001F319 [Weekly] Run All Tests [Trunk]"
   triggers:
     recurring:
     - branch: develop
@@ -24,7 +24,7 @@ develop_weekly_trunk:
     - .yamato/_run-all.yml#run_all_tests_trunk
 
 multiprocess_nightly:
-  name: "[Nightly] Run Multiprocess Tests"
+  name: "\U0001F319 [Nightly] Run Multiprocess Tests"
   triggers:
     recurring:
     - branch: develop
@@ -65,7 +65,7 @@ pull_request_trigger:
 # Currently, we need to have a trigger to updated badges
 # Only package badges currently exist
 badges_test_trigger:
-  name: Badges Tests Trigger
+  name: ⚡ Badges Tests Trigger
   agent:
     type: Unity::VM
     image: package-ci/ubuntu:stable

.yamato/code-coverage.yml

@@ -1,6 +1,7 @@
 {% metadata_file .yamato/project.metafile %}
 ---
 {% for project in projects -%}
+{% if project.has_tests == "true" -%}
 code_coverage_win_{{ project.name }}_{{ validation_editor }}:
   name: Code Coverage Report - Windows - {{ project.name }} - {{ validation_editor }}
   agent:
@@ -18,4 +19,5 @@ code_coverage_win_{{ project.name }}_{{ validation_editor }}:
         - "upm-ci~/test-results/**/*"
   dependencies:
     - .yamato/project-pack.yml#pack_{{ project.name }}
+{% endif -%}
 {% endfor -%}

.yamato/project.metafile

@@ -35,6 +35,7 @@ projects:
     path: testproject
     validate: true
     publish: true
+    has_tests: true
     # Packages within a project that will be tested
     packages:
       - name: com.unity.netcode.gameobjects
@@ -47,6 +48,7 @@ projects:
     path: minimalproject
     validate: false
     publish: false
+    has_tests: false
     packages:
       - name: com.unity.netcode.gameobjects
         path: com.unity.netcode.gameobjects
@@ -56,6 +58,15 @@ projects:
     path: testproject-tools-integration
     validate: false
     publish: false
+    has_tests: true
     test_editors:
       - 2021.3
       - trunk
+
+# Scripting backends used by Standalone Playmode Tests
+scripting_backends: 
+  - mono
+  - il2cpp
+
+# Images with build-tools installed required for Standalone Tests - IL2CPP
+win_il2cpp_test_image: dots-player/windows10:latest

.yamato/standalone-project-tests.yml

@@ -1,30 +1,30 @@
 {% metadata_file .yamato/project.metafile %}
 ---
 
-# For every platform and editor version, run its project tests without
-# running package tests too since they are handled on their respective
-# jobs
 {% for project in projects -%}
+{% if project.has_tests == "true" -%}
 {% for editor in project.test_editors -%}
 {% for platform in test_platforms -%}
-standalone_test_{{ project.name }}_{{ editor }}_{{ platform.name }}:
-  name : standalone {{ project.name }} tests - {{ editor }} on {{ platform.name }}
+{% for backend in scripting_backends -%}
+standalone_tests_{{ project.name }}_{{ backend }}_{{ editor }}_{{ platform.name }}:
+  name : Standalone Tests - {{ project.name }} - [{{ backend }}, {{ platform.name }}, {{ editor }}]
   agent:
     type: {{ platform.type }}{% if platform.name == "ubuntu" %}::GPU{% endif %}
 {% if platform.name == "ubuntu" %}    model: rtx2080{% endif %}
-    image: {{ platform.image }}
+    image: {% if platform.name == 'win' and backend == 'il2cpp' %}{{ win_il2cpp_test_image }} {% else %} {{ platform.image }} {% endif %}
     flavor: {{ platform.flavor}}
   commands:
     - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
     - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
     - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr{% if platform.name == "win" %}.bat{% endif %} --output utr{% if platform.name == "win" %}.bat{% endif %}
-{% if platform.name != "win" %}    - chmod +x ./utr{% endif %}
-    - unity-downloader-cli -u {{editor}} -c Editor --fast --wait
-    - {% if platform.name != "win" %}./{% endif %}utr --suite=playmode --platform={{platform.standalone}}  --editor-location=.Editor --testproject=testproject --player-save-path=build/players --artifacts_path=build/logs --scripting-backend=mono --build-only --testfilter=Unity.Netcode.RuntimeTests --extra-editor-arg=-batchmode --extra-editor-arg=-nographics
+{% if platform.name != "win" %}
+    - chmod +x ./utr
+{% endif %}
+    - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp{% endif %} --fast --wait
+    - {% if platform.name != "win" %}./{% endif %}utr --suite=playmode --platform={{ platform.standalone }}  --editor-location=.Editor --testproject=testproject --player-save-path=build/players --artifacts_path=build/logs --scripting-backend={{ backend }} --build-only --testfilter=Unity.Netcode.RuntimeTests --extra-editor-arg=-batchmode --extra-editor-arg=-nographics
     - |
       {% if platform.name == "win" %}set{% endif %}{% if platform.name != "win" %}export{% endif %} UTR_VERSION=0.12.0
-      {% if platform.name != "win" %}./{% endif %}utr --suite=playmode --platform={{platform.standalone}}  --player-load-path=build/players --artifacts_path=build/test-results --testfilter=Unity.Netcode.RuntimeTests --playergraphicsapi=Null
-    - echo {{ platform.name }}
+      {% if platform.name != "win" %}./{% endif %}utr --suite=playmode --platform={{ platform.standalone }}  --player-load-path=build/players --artifacts_path=build/test-results --scripting-backend={{ backend }} --testfilter=Unity.Netcode.RuntimeTests --playergraphicsapi=Null
   artifacts:
     logs:
       paths:
@@ -36,5 +36,5 @@ standalone_test_{{ project.name }}_{{ editor }}_{{ platform.name }}:
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
-
-
+{% endif -%}
+{% endfor -%}