From 2bdf456d51cef94db8e9affc522f8f8fdd76ec63 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Thu, 13 Nov 2025 14:31:02 +0100 Subject: [PATCH 1/7] C#: Read from dependency directory from extractor option. --- .../DependencyDirectory.cs | 60 +++++++++++++++++++ .../NugetExeWrapper.cs | 6 +- .../NugetPackageRestorer.cs | 12 ++-- .../Semmle.Util/EnvironmentVariables.cs | 10 ++++ 4 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs new file mode 100644 index 000000000000..f19e4b8fd9c8 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs @@ -0,0 +1,60 @@ +using System; +using System.IO; +using Semmle.Util; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + /// + /// A directory used for storing fetched dependencies. + /// When a specific directory is set via the dependency directory extractor option, + /// we store dependencies in that directory for caching purposes. + /// Otherwise, we create a temporary directory that is deleted upon disposal. + /// + public sealed class DependencyDirectory : IDisposable + { + private readonly string userReportedDirectoryPurpose; + private readonly ILogger logger; + private readonly bool attemptCleanup; + + public DirectoryInfo DirInfo { get; } + + public DependencyDirectory(string subfolderName, string userReportedDirectoryPurpose, ILogger logger) + { + this.logger = logger; + this.userReportedDirectoryPurpose = userReportedDirectoryPurpose; + + string path; + if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir) + { + path = dir; + attemptCleanup = false; + } + else + { + path = FileUtils.GetTemporaryWorkingDirectory(out _); + attemptCleanup = true; + } + DirInfo = new DirectoryInfo(Path.Join(path, subfolderName)); + DirInfo.Create(); + } + + public void Dispose() + { + if (!attemptCleanup) + { + logger.LogInfo($"Keeping {userReportedDirectoryPurpose} directory {DirInfo.FullName} for possible caching purposes."); + return; + } + + try + { + DirInfo.Delete(true); + } + catch (Exception exc) + { + logger.LogInfo($"Couldn't delete {userReportedDirectoryPurpose} directory {exc.Message}"); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs index c77daa8899c8..10d89b1e009d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs @@ -25,15 +25,15 @@ internal class NugetExeWrapper : IDisposable /// /// The computed packages directory. - /// This will be in the Temp location + /// This will be in the Cached or Temp location /// so as to not trample the source tree. /// - private readonly TemporaryDirectory packageDirectory; + private readonly DependencyDirectory packageDirectory; /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger) + public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index e0e1bc649fa4..e2e548a46a96 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -24,12 +24,12 @@ internal sealed partial class NugetPackageRestorer : IDisposable private readonly IDotNet dotnet; private readonly DependabotProxy? dependabotProxy; private readonly IDiagnosticsWriter diagnosticsWriter; - private readonly TemporaryDirectory legacyPackageDirectory; - private readonly TemporaryDirectory missingPackageDirectory; + private readonly DependencyDirectory legacyPackageDirectory; + private readonly DependencyDirectory missingPackageDirectory; private readonly ILogger logger; private readonly ICompilationInfoContainer compilationInfoContainer; - public TemporaryDirectory PackageDirectory { get; } + public DependencyDirectory PackageDirectory { get; } public NugetPackageRestorer( FileProvider fileProvider, @@ -48,9 +48,9 @@ public NugetPackageRestorer( this.logger = logger; this.compilationInfoContainer = compilationInfoContainer; - PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("packages"), "package", logger); - legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger); - missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("missingpackages"), "missing package", logger); + PackageDirectory = new DependencyDirectory("packages", "package", logger); + legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger); + missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger); } public string? TryRestore(string package) diff --git a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs index edce64a53fe4..9f1519653de8 100644 --- a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs +++ b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs @@ -76,5 +76,15 @@ public static IEnumerable GetURLs(string name) { return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_BASE_METADATA_OUT"); } + + /// + /// If set, returns the directory where buildless dependencies should be stored. + /// This is needed for caching dependencies. + /// + /// + public static string? GetBuildlessDependencyDir() + { + return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR"); + } } } From 8d75a19e022e977fb6cd7ea3c33b80c3622437ba Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Thu, 13 Nov 2025 14:33:25 +0100 Subject: [PATCH 2/7] C#: Add extractor option for buildless dependency directory. --- csharp/codeql-extractor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/csharp/codeql-extractor.yml b/csharp/codeql-extractor.yml index da7d665f7a76..8cba8f18e47e 100644 --- a/csharp/codeql-extractor.yml +++ b/csharp/codeql-extractor.yml @@ -74,3 +74,8 @@ options: [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted. This option only works when `--build-mode none` is also specified. type: array + buildless_dependency_dir: + title: The path where buildless (standalone) extraction should keep dependencies. + description: > + If set, the buildless (standalone) extractor will store dependencies in this directory. + type: string From d274a4a654f411ae9407859b1d4189cebbdf0514 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 14 Nov 2025 10:24:15 +0100 Subject: [PATCH 3/7] C#: Try with Path.Combine instead. --- .../DependencyDirectory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs index f19e4b8fd9c8..05dff54d9290 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs @@ -35,7 +35,7 @@ public DependencyDirectory(string subfolderName, string userReportedDirectoryPur path = FileUtils.GetTemporaryWorkingDirectory(out _); attemptCleanup = true; } - DirInfo = new DirectoryInfo(Path.Join(path, subfolderName)); + DirInfo = new DirectoryInfo(Path.Combine(path, subfolderName)); DirInfo.Create(); } From 91267059e339aca2f6e8a07b3ca55f9f612dd915 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 14 Nov 2025 11:01:48 +0100 Subject: [PATCH 4/7] REMOVE AGAIN. --- .../DependencyDirectory.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs index 05dff54d9290..c038e0f54d33 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs @@ -25,16 +25,16 @@ public DependencyDirectory(string subfolderName, string userReportedDirectoryPur this.userReportedDirectoryPurpose = userReportedDirectoryPurpose; string path; - if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir) - { - path = dir; - attemptCleanup = false; - } - else - { - path = FileUtils.GetTemporaryWorkingDirectory(out _); - attemptCleanup = true; - } + // if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir) + // { + // path = dir; + // attemptCleanup = false; + // } + // else + // { + path = FileUtils.GetTemporaryWorkingDirectory(out _); + attemptCleanup = true; + //} DirInfo = new DirectoryInfo(Path.Combine(path, subfolderName)); DirInfo.Create(); } From 53be3bbffab8b1cf9ddc85def5baaede386818e5 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 14 Nov 2025 13:07:39 +0100 Subject: [PATCH 5/7] Rever to TempDir. --- .../NugetExeWrapper.cs | 4 ++-- .../NugetPackageRestorer.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs index 10d89b1e009d..cdf095a3dd52 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs @@ -28,12 +28,12 @@ internal class NugetExeWrapper : IDisposable /// This will be in the Cached or Temp location /// so as to not trample the source tree. /// - private readonly DependencyDirectory packageDirectory; + private readonly TemporaryDirectory packageDirectory; /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger) + public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index e2e548a46a96..b8afcd7825ea 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -24,7 +24,7 @@ internal sealed partial class NugetPackageRestorer : IDisposable private readonly IDotNet dotnet; private readonly DependabotProxy? dependabotProxy; private readonly IDiagnosticsWriter diagnosticsWriter; - private readonly DependencyDirectory legacyPackageDirectory; + private readonly TemporaryDirectory legacyPackageDirectory; private readonly DependencyDirectory missingPackageDirectory; private readonly ILogger logger; private readonly ICompilationInfoContainer compilationInfoContainer; @@ -49,7 +49,7 @@ public NugetPackageRestorer( this.compilationInfoContainer = compilationInfoContainer; PackageDirectory = new DependencyDirectory("packages", "package", logger); - legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger); + legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger); missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger); } From 6b62b9a95193372f5cae964852b58849ee6df775 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 14 Nov 2025 13:46:49 +0100 Subject: [PATCH 6/7] C#: try again. --- .../DependencyDirectory.cs | 12 +----------- .../NugetExeWrapper.cs | 4 ++-- .../NugetPackageRestorer.cs | 4 ++-- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs index c038e0f54d33..e3324a797971 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs @@ -24,18 +24,8 @@ public DependencyDirectory(string subfolderName, string userReportedDirectoryPur this.logger = logger; this.userReportedDirectoryPurpose = userReportedDirectoryPurpose; - string path; - // if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir) - // { - // path = dir; - // attemptCleanup = false; - // } - // else - // { - path = FileUtils.GetTemporaryWorkingDirectory(out _); attemptCleanup = true; - //} - DirInfo = new DirectoryInfo(Path.Combine(path, subfolderName)); + DirInfo = new DirectoryInfo(Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out _), subfolderName)); DirInfo.Create(); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs index cdf095a3dd52..10d89b1e009d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs @@ -28,12 +28,12 @@ internal class NugetExeWrapper : IDisposable /// This will be in the Cached or Temp location /// so as to not trample the source tree. /// - private readonly TemporaryDirectory packageDirectory; + private readonly DependencyDirectory packageDirectory; /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger) + public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index b8afcd7825ea..e2e548a46a96 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -24,7 +24,7 @@ internal sealed partial class NugetPackageRestorer : IDisposable private readonly IDotNet dotnet; private readonly DependabotProxy? dependabotProxy; private readonly IDiagnosticsWriter diagnosticsWriter; - private readonly TemporaryDirectory legacyPackageDirectory; + private readonly DependencyDirectory legacyPackageDirectory; private readonly DependencyDirectory missingPackageDirectory; private readonly ILogger logger; private readonly ICompilationInfoContainer compilationInfoContainer; @@ -49,7 +49,7 @@ public NugetPackageRestorer( this.compilationInfoContainer = compilationInfoContainer; PackageDirectory = new DependencyDirectory("packages", "package", logger); - legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger); + legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger); missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger); } From 9d62b3a747a578772b6d944a705a115eb6efa8e2 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 14 Nov 2025 14:17:54 +0100 Subject: [PATCH 7/7] C#: Add directory override. --- .../DependencyDirectory.cs | 2 ++ .../NugetPackageRestorer.cs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs index e3324a797971..ca44a92df3a8 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs @@ -46,5 +46,7 @@ public void Dispose() logger.LogInfo($"Couldn't delete {userReportedDirectoryPurpose} directory {exc.Message}"); } } + + public override string ToString() => DirInfo.FullName.ToString(); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index e2e548a46a96..a7258035484d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -49,7 +49,7 @@ public NugetPackageRestorer( this.compilationInfoContainer = compilationInfoContainer; PackageDirectory = new DependencyDirectory("packages", "package", logger); - legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger); + legacyPackageDirectory = new DependencyDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger); missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger); }