Merge pull request #15156 from tamasvajk/standalone/temp-folder-structure

tamasvajk · web-flow · commit d3c685a41ac3 · 2023-12-20T11:57:42.000+01:00
C#: Fix working directory structures in standalone
diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
@@ -936,9 +936,8 @@ public void TestDotnetVersionNotInstalled()
         {
             actions.RunProcess["dotnet --list-sdks"] = 0;
             actions.RunProcessOut["dotnet --list-sdks"] = "2.1.2 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]";
-            actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
-            actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
-            actions.RunProcess[@"rm dotnet-install.sh"] = 0;
+            actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
+            actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
             actions.RunProcess[@"scratch/.dotnet/dotnet --info"] = 0;
             actions.RunProcess[@"scratch/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
             actions.RunProcess[@"scratch/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
@@ -960,10 +959,11 @@ public void TestDotnetVersionNotInstalled()
 
 </Project>");
             actions.LoadXml[@"C:\Project/test.csproj"] = xml;
-            actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "dotnet-install.sh"));
+            actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
+            actions.CreateDirectories.Add(@"scratch/.dotnet");
 
             var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
-            TestAutobuilderScript(autobuilder, 0, 8);
+            TestAutobuilderScript(autobuilder, 0, 7);
         }
 
         [Fact]
@@ -972,9 +972,8 @@ public void TestDotnetVersionAlreadyInstalled()
             actions.RunProcess["dotnet --list-sdks"] = 0;
             actions.RunProcessOut["dotnet --list-sdks"] = @"2.1.3 [C:\Program Files\dotnet\sdks]
 2.1.4 [C:\Program Files\dotnet\sdks]";
-            actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
-            actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
-            actions.RunProcess[@"rm dotnet-install.sh"] = 0;
+            actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
+            actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
             actions.RunProcess[@"scratch/.dotnet/dotnet --info"] = 0;
             actions.RunProcess[@"scratch/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
             actions.RunProcess[@"scratch/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
@@ -996,10 +995,11 @@ public void TestDotnetVersionAlreadyInstalled()
 
 </Project>");
             actions.LoadXml[@"C:\Project/test.csproj"] = xml;
-            actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "dotnet-install.sh"));
+            actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
+            actions.CreateDirectories.Add(@"scratch/.dotnet");
 
             var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
-            TestAutobuilderScript(autobuilder, 0, 8);
+            TestAutobuilderScript(autobuilder, 0, 7);
         }
 
         private void TestDotnetVersionWindows(Action action, int commandsRun)
diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs
@@ -190,30 +190,41 @@ BuildScript GetInstall(string pwsh) =>
                     }
                     else
                     {
+                        var dotnetInstallPath = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(
+                            builder.Actions.GetEnvironmentVariable,
+                            builder.Options.Language.UpperCaseName,
+                            out var shouldCleanUp), ".dotnet", "dotnet-install.sh");
+
                         var downloadDotNetInstallSh = BuildScript.DownloadFile(
                             "https://dot.net/v1/dotnet-install.sh",
-                            "dotnet-install.sh",
+                            dotnetInstallPath,
                             e => builder.Log(Severity.Warning, $"Failed to download 'dotnet-install.sh': {e.Message}"));
 
                         var chmod = new CommandBuilder(builder.Actions).
                             RunCommand("chmod").
                             Argument("u+x").
-                            Argument("dotnet-install.sh");
+                            Argument(dotnetInstallPath);
 
                         var install = new CommandBuilder(builder.Actions).
-                            RunCommand("./dotnet-install.sh").
+                            RunCommand(dotnetInstallPath).
                             Argument("--channel").
                             Argument("release").
                             Argument("--version").
                             Argument(version).
                             Argument("--install-dir").
                             Argument(path);
 
-                        var removeScript = new CommandBuilder(builder.Actions).
-                            RunCommand("rm").
-                            Argument("dotnet-install.sh");
+                        var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script;
+
+                        if (shouldCleanUp)
+                        {
+                            var removeScript = new CommandBuilder(builder.Actions).
+                                RunCommand("rm").
+                                Argument(dotnetInstallPath);
+                            buildScript &= removeScript.Script;
+                        }
 
-                        return downloadDotNetInstallSh & chmod.Script & install.Script & BuildScript.Try(removeScript.Script);
+                        return buildScript;
                     }
                 });
         }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -52,7 +52,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             this.progressMonitor = new ProgressMonitor(logger);
             this.sourceDir = new DirectoryInfo(srcDir);
 
-            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
+            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
             legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "legacypackages"));
             missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "missingpackages"));
 
@@ -467,15 +467,15 @@ private IEnumerable<FileInfo> GetAllFiles()
         /// with this source tree. Use a SHA1 of the directory name.
         /// </summary>
         /// <returns>The full path of the temp directory.</returns>
-        private static string ComputeTempDirectory(string srcDir, string packages = "packages")
+        private static string ComputeTempDirectory(string srcDir, string subfolderName)
         {
             var bytes = Encoding.Unicode.GetBytes(srcDir);
             var sha = SHA1.HashData(bytes);
             var sb = new StringBuilder();
             foreach (var b in sha.Take(8))
                 sb.AppendFormat("{0:x2}", b);
 
-            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out var _), "GitHub", packages, sb.ToString());
+            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out var _), sb.ToString(), subfolderName);
         }
 
         /// <summary>
@@ -723,7 +723,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
             Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
             {
                 progressMonitor.NugetInstall(package);
-                using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
+                using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
                 var success = dotnet.New(tempDir.DirInfo.FullName);
                 if (!success)
                 {
diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs
@@ -144,20 +144,26 @@ public static string NestPaths(ILogger logger, string? outerpath, string innerpa
             return nested;
         }
 
+        private static readonly Lazy<string> tempFolderPath = new Lazy<string>(() =>
+        {
+            var tempPath = Path.GetTempPath();
+            var name = Guid.NewGuid().ToString("N").ToUpper();
+            var tempFolder = Path.Combine(tempPath, "GitHub", name);
+            Directory.CreateDirectory(tempFolder);
+            return tempFolder;
+        });
+
         public static string GetTemporaryWorkingDirectory(Func<string, string?> getEnvironmentVariable, string lang, out bool shouldCleanUp)
         {
-            shouldCleanUp = false;
             var tempFolder = getEnvironmentVariable($"CODEQL_EXTRACTOR_{lang}_SCRATCH_DIR");
-
-            if (string.IsNullOrEmpty(tempFolder))
+            if (!string.IsNullOrEmpty(tempFolder))
             {
-                var tempPath = Path.GetTempPath();
-                var name = Guid.NewGuid().ToString("N").ToUpper();
-                tempFolder = Path.Combine(tempPath, "GitHub", name);
-                shouldCleanUp = true;
+                shouldCleanUp = false;
+                return tempFolder;
             }
 
-            return tempFolder;
+            shouldCleanUp = true;
+            return tempFolderPath.Value;
         }
 
         public static string GetTemporaryWorkingDirectory(out bool shouldCleanUp) =>
diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependencies_net48/Assemblies.ql b/csharp/ql/integration-tests/all-platforms/standalone_dependencies_net48/Assemblies.ql
@@ -4,10 +4,11 @@ private string getPath(Assembly a) {
   not a.getCompilation().getOutputAssembly() = a and
   exists(string s | s = a.getFile().getAbsolutePath() |
     result =
-      s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
+      s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
+          "/packages".length(), s.length())
     or
     result = s and
-    not exists(s.indexOf("GitHub/packages/"))
+    not exists(s.indexOf("test-db/working/"))
   )
 }
 
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies/Assemblies.ql
@@ -4,10 +4,11 @@ private string getPath(Assembly a) {
   not a.getCompilation().getOutputAssembly() = a and
   exists(string s | s = a.getFile().getAbsolutePath() |
     result =
-      s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
+      s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
+          "/packages".length(), s.length())
     or
     result = s and
-    not exists(s.indexOf("GitHub/packages/"))
+    not exists(s.indexOf("test-db/working/"))
   )
 }
 
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_multi_target/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_multi_target/Assemblies.ql
@@ -4,10 +4,11 @@ private string getPath(Assembly a) {
   not a.getCompilation().getOutputAssembly() = a and
   exists(string s | s = a.getFile().getAbsolutePath() |
     result =
-      s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
+      s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
+          "/packages".length(), s.length())
     or
     result = s and
-    not exists(s.indexOf("GitHub/packages/"))
+    not exists(s.indexOf("test-db/working/"))
   )
 }
 
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget/Assemblies.ql
@@ -4,19 +4,9 @@ private string getPath(Assembly a) {
   not a.getCompilation().getOutputAssembly() = a and
   exists(string s | s = a.getFile().getAbsolutePath() |
     result =
-      s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
-    or
-    result =
-      s.substring(s.indexOf("GitHub/legacypackages/") + "GitHub/legacypackages/".length() + 16,
-        s.length())
-    // TODO: excluding all other assemblies from the test result as mono installations seem problematic on ARM runners.
-    // or
-    // result = s.substring(s.indexOf("lib/mono/") + "lib/mono/".length(), s.length())
-    // or
-    // result = s and
-    // not exists(s.indexOf("GitHub/packages/")) and
-    // not exists(s.indexOf("GitHub/legacypackages/")) and
-    // not exists(s.indexOf("lib/mono/"))
+      s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
+          "/legacypackages".length(), s.length())
+    // TODO: include all other assemblies from the test results. Initially disable because mono installations were problematic on ARM runners.
   )
 }
 
diff --git a/csharp/ql/integration-tests/windows-only/standalone_dependencies/Assemblies.ql b/csharp/ql/integration-tests/windows-only/standalone_dependencies/Assemblies.ql
@@ -4,10 +4,11 @@ private string getPath(Assembly a) {
   not a.getCompilation().getOutputAssembly() = a and
   exists(string s | s = a.getFile().getAbsolutePath() |
     result =
-      s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
+      s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
+          "/packages".length(), s.length())
     or
     result = s and
-    not exists(s.indexOf("GitHub/packages/"))
+    not exists(s.indexOf("test-db/working/"))
   )
 }
 

Original file line number	Diff line number	Diff line change
`@@ -4,10 +4,11 @@ private string getPath(Assembly a) {`
`4`	`4`	`not a.getCompilation().getOutputAssembly() = a and`
`5`	`5`	`exists(string s \| s = a.getFile().getAbsolutePath() \|`
`6`	`6`	`result =`
`7`		`- s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())`
	`7`	`+ s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +`
	`8`	`+ "/packages".length(), s.length())`
`8`	`9`	`or`
`9`	`10`	`result = s and`
`10`		`- not exists(s.indexOf("GitHub/packages/"))`
	`11`	`+ not exists(s.indexOf("test-db/working/"))`
`11`	`12`	`)`
`12`	`13`	`}`
`13`	`14`