test: Allow for the detection of whether to run locally or remotely. (#2054)

* Set up for launching remotely

* Set up for launching remotely

* Report on WorkerCount and LaunchRemotely state

* Store the Process as it will complete later

* add a little more logging

* Simplified name

* Update GetWorkerCount to no longer set internal state

* Update BaseMultiprocessTests.cs

respond to PR feedback

Co-authored-by: ashwini <36935028+ashwinimurt@users.noreply.github.com>

Author: zain-mecklai

testproject/Assets/Tests/Runtime/MultiprocessRuntime/BaseMultiprocessTests.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
 using NUnit.Framework;
 using UnityEngine;
 using UnityEngine.SceneManagement;
@@ -22,6 +24,14 @@ public MultiprocessTestsAttribute() : base(MultiprocessCategoryName) { }
     [MultiprocessTests]
     public abstract class BaseMultiprocessTests
     {
+        protected string[] platformList { get; set; }
+
+        protected int GetWorkerCount()
+        {
+            platformList = MultiprocessOrchestration.GetRemotePlatformList();
+            return platformList == null ? WorkerCount : platformList.Length;
+        }
+        protected bool m_LaunchRemotely;
         private bool m_HasSceneLoaded = false;
         // TODO: Remove UTR check once we have Multiprocess tests fully working
         protected bool IgnoreMultiprocessTests => MultiprocessOrchestration.ShouldIgnoreUTRTests();
@@ -30,7 +40,7 @@ public abstract class BaseMultiprocessTests
 
         /// <summary>
         /// Implement this to specify the amount of workers to spawn from your main test runner
-        /// TODO there's a good chance this will be re-factored with something fancier once we start integrating with bokken
+        /// Note: If using remote workers, the woorker count will come from the environment variable 
         /// </summary>
         protected abstract int WorkerCount { get; }
 
@@ -131,16 +141,16 @@ private bool VerifySceneIsValidForClientsToLoad(int sceneIndex, string sceneName
         public virtual IEnumerator Setup()
         {
             yield return new WaitUntil(() => NetworkManager.Singleton != null);
-            MultiprocessLogger.Log("NetworkManager.Singleton != null");
             yield return new WaitUntil(() => NetworkManager.Singleton.IsServer);
-            MultiprocessLogger.Log("NetworkManager.Singleton.IsServer");
             yield return new WaitUntil(() => NetworkManager.Singleton.IsListening);
-            MultiprocessLogger.Log("NetworkManager.Singleton.IsListening");
             yield return new WaitUntil(() => m_HasSceneLoaded == true);
-            MultiprocessLogger.Log("m_HasSceneLoaded");
             var startTime = Time.time;
+            m_LaunchRemotely = MultiprocessOrchestration.IsRemoteOperationEnabled();
 
-            MultiprocessLogger.Log($"Active Worker Count is {MultiprocessOrchestration.ActiveWorkerCount()} and connected client count is {NetworkManager.Singleton.ConnectedClients.Count}");
+            MultiprocessLogger.Log($"Active Worker Count is {MultiprocessOrchestration.ActiveWorkerCount()}" +
+                $" and connected client count is {NetworkManager.Singleton.ConnectedClients.Count} " +
+                $" and WorkerCount is {GetWorkerCount()} " +
+                $" and LaunchRemotely is {m_LaunchRemotely}");
             if (MultiprocessOrchestration.ActiveWorkerCount() + 1 < NetworkManager.Singleton.ConnectedClients.Count)
             {
                 MultiprocessLogger.Log("Is this a bad state?");
@@ -149,20 +159,32 @@ public virtual IEnumerator Setup()
             // Moved this out of OnSceneLoaded as OnSceneLoaded is a callback from the SceneManager and just wanted to avoid creating
             // processes from within the same callstack/context as the SceneManager.  This will instantiate up to the WorkerCount and
             // then any subsequent calls to Setup if there are already workers it will skip this step
-            if (NetworkManager.Singleton.ConnectedClients.Count - 1 < WorkerCount)
+            if (!m_LaunchRemotely)
             {
-                var numProcessesToCreate = WorkerCount - (NetworkManager.Singleton.ConnectedClients.Count - 1);
-                for (int i = 1; i <= numProcessesToCreate; i++)
+                if (NetworkManager.Singleton.ConnectedClients.Count - 1 < WorkerCount)
                 {
-                    MultiprocessLogger.Log($"Spawning testplayer {i} since connected client count is {NetworkManager.Singleton.ConnectedClients.Count} is less than {WorkerCount} and Number of spawned external players is {MultiprocessOrchestration.ActiveWorkerCount()} ");
-                    string logPath = MultiprocessOrchestration.StartWorkerNode(); // will automatically start built player as clients
-                    MultiprocessLogger.Log($"logPath to new process is {logPath}");
-                    MultiprocessLogger.Log($"Active Worker Count {MultiprocessOrchestration.ActiveWorkerCount()} and connected client count is {NetworkManager.Singleton.ConnectedClients.Count}");
+                    var numProcessesToCreate = WorkerCount - (NetworkManager.Singleton.ConnectedClients.Count - 1);
+                    for (int i = 1; i <= numProcessesToCreate; i++)
+                    {
+                        MultiprocessLogger.Log($"Spawning testplayer {i} since connected client count is {NetworkManager.Singleton.ConnectedClients.Count} is less than {WorkerCount} and Number of spawned external players is {MultiprocessOrchestration.ActiveWorkerCount()} ");
+                        string logPath = MultiprocessOrchestration.StartWorkerNode(); // will automatically start built player as clients
+                        MultiprocessLogger.Log($"logPath to new process is {logPath}");
+                        MultiprocessLogger.Log($"Active Worker Count {MultiprocessOrchestration.ActiveWorkerCount()} and connected client count is {NetworkManager.Singleton.ConnectedClients.Count}");
+                    }
                 }
             }
             else
             {
-                MultiprocessLogger.Log($"No need to spawn a new test player as there are already existing processes {MultiprocessOrchestration.ActiveWorkerCount()} and connected clients {NetworkManager.Singleton.ConnectedClients.Count}");
+                var launchProcessList = new List<Process>();
+                if (NetworkManager.Singleton.ConnectedClients.Count - 1 < GetWorkerCount())
+                {
+                    var machines = MultiprocessOrchestration.GetRemoteMachineList();
+                    foreach (var machine in machines)
+                    {
+                        MultiprocessLogger.Log($"Would launch on {machine.Name} too get worker count to {GetWorkerCount()} from {NetworkManager.Singleton.ConnectedClients.Count - 1}");
+                        launchProcessList.Add(MultiprocessOrchestration.StartWorkersOnRemoteNodes(machine));
+                    }
+                }
             }
 
             var timeOutTime = Time.realtimeSinceStartup + TestCoordinator.MaxWaitTimeoutSec;

testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/MultiprocessLogger.cs

@@ -62,7 +62,7 @@ public void LogFormat(LogType logType, UnityEngine.Object context, string format
                 testName = "unknown";
             }
 
-            Debug.unityLogger.logHandler.LogFormat(logType, context, $"MPLOG({DateTime.Now:T}) : {testName} : {format}", args);
+            Debug.LogFormat(logType, LogOption.NoStacktrace, context, $"MPLOG({DateTime.Now:T}) : {testName} : {format}", args);
         }
     }
 }

testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/MultiprocessOrchestration.cs

@@ -21,6 +21,9 @@ public static DirectoryInfo MultiprocessDirInfo
     }
     private static List<Process> s_Processes = new List<Process>();
     private static int s_TotalProcessCounter = 0;
+    public static string PathToDll { get; private set; }
+    public static List<Process> ProcessList = new List<Process>();
+    private static FileInfo s_Localip_fileinfo;
 
     private static DirectoryInfo initMultiprocessDirinfo()
     {
@@ -39,12 +42,27 @@ private static DirectoryInfo initMultiprocessDirinfo()
         {
             MultiprocessDirInfo.Create();
         }
+        s_Localip_fileinfo = new FileInfo(Path.Combine(s_MultiprocessDirInfo.FullName, "localip"));
+
         return s_MultiprocessDirInfo;
     }
 
     static MultiprocessOrchestration()
     {
         initMultiprocessDirinfo();
+        MultiprocessLogger.Log($" userprofile: {s_MultiprocessDirInfo.FullName} localipfile: {s_Localip_fileinfo}");
+        var rootdir_FileInfo = new FileInfo(Path.Combine(MultiprocessDirInfo.FullName, "rootdir"));
+        MultiprocessLogger.Log($"Checking for the existence of {rootdir_FileInfo.FullName}");
+        if (rootdir_FileInfo.Exists)
+        {
+            var rootDirText = (File.ReadAllText(rootdir_FileInfo.FullName)).Trim();
+            PathToDll = Path.Combine(rootDirText, "multiplayer-multiprocess-test-tools/BokkenForNetcode/ProvisionBokkenMachines/bin/Debug/netcoreapp3.1/osx-x64", "ProvisionBokkenMachines.dll");
+        }
+        else
+        {
+            MultiprocessLogger.Log("PathToDll cannot be set as rootDir doesn't exist");
+            PathToDll = "unknown";
+        }
     }
 
     /// <summary>
@@ -185,4 +203,77 @@ public static void ShutdownAllProcesses()
 
         s_Processes.Clear();
     }
+
+    public static bool IsRemoteOperationEnabled()
+    {
+        string encodedPlatformList = Environment.GetEnvironmentVariable("MP_PLATFORM_LIST");
+        if (encodedPlatformList != null && encodedPlatformList.Split(',').Length > 1)
+        {
+            return true;
+        }
+        return false;
+    }
+
+    public static string[] GetRemotePlatformList()
+    {
+        // "default-win:test-win,default-mac:test-mac"
+        if (!IsRemoteOperationEnabled())
+        {
+            return null;
+        }
+        string encodedPlatformList = Environment.GetEnvironmentVariable("MP_PLATFORM_LIST");
+        string[] separated = encodedPlatformList.Split(',');
+        return separated;
+    }
+
+    public static List<FileInfo> GetRemoteMachineList()
+    {
+        var machineJson = new List<FileInfo>();
+        foreach (var f in MultiprocessDirInfo.GetFiles("*.json"))
+        {
+            if (f.Name.Equals("remoteConfig.json"))
+            {
+                continue;
+            }
+            else
+            {
+                machineJson.Add(f);
+            }
+        }
+        return machineJson;
+    }
+
+    public static Process StartWorkersOnRemoteNodes(FileInfo machine)
+    {
+        string command = $" --command launch " +
+                $"--input-path {machine.FullName} ";
+
+        var workerProcess = new Process();
+
+        workerProcess.StartInfo.FileName = Path.Combine("dotnet");
+        workerProcess.StartInfo.UseShellExecute = false;
+        workerProcess.StartInfo.RedirectStandardError = true;
+        workerProcess.StartInfo.RedirectStandardOutput = true;
+        workerProcess.StartInfo.Arguments = $"{PathToDll} {command} ";
+        try
+        {
+            var newProcessStarted = workerProcess.Start();
+
+            if (!newProcessStarted)
+            {
+                throw new Exception("Failed to start worker process!");
+            }
+        }
+        catch (Win32Exception e)
+        {
+            MultiprocessLogger.LogError($"Error starting bokken process, {e.Message} {e.Data} {e.ErrorCode}");
+            throw;
+        }
+
+
+        ProcessList.Add(workerProcess);
+
+        MultiprocessLogger.Log($"Execute Command: {PathToDll} {command} End");
+        return workerProcess;
+    }
 }