Swift: collect TRAP files related to an extractor run

In order to do this a mutable `SwiftExtractorState` is introduced.

Author: redsun82

swift/extractor/SwiftExtractor.cpp

@@ -109,7 +109,7 @@ static llvm::SmallVector<swift::Decl*> getTopLevelDecls(swift::ModuleDecl& modul
 }
 
 static std::unordered_set<swift::ModuleDecl*> extractDeclarations(
-    const SwiftExtractorConfiguration& config,
+    SwiftExtractorState& state,
     swift::CompilerInstance& compiler,
     swift::ModuleDecl& module,
     swift::SourceFile* primaryFile = nullptr) {
@@ -118,7 +118,7 @@ static std::unordered_set<swift::ModuleDecl*> extractDeclarations(
   // The extractor can be called several times from different processes with
   // the same input file(s). Using `TargetFile` the first process will win, and the following
   // will just skip the work
-  auto trap = createTargetTrapDomain(config, filename);
+  auto trap = createTargetTrapDomain(state, filename);
   if (!trap) {
     // another process arrived first, nothing to do for us
     return {};
@@ -170,8 +170,7 @@ static std::vector<swift::ModuleDecl*> collectLoadedModules(swift::CompilerInsta
   return ret;
 }
 
-void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
-                               swift::CompilerInstance& compiler) {
+void codeql::extractSwiftFiles(SwiftExtractorState& state, swift::CompilerInstance& compiler) {
   auto inputFiles = collectInputFilenames(compiler);
   std::vector<swift::ModuleDecl*> todo = collectLoadedModules(compiler);
   std::unordered_set<swift::ModuleDecl*> seen{todo.begin(), todo.end()};
@@ -190,11 +189,11 @@ void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
       if (inputFiles.count(sourceFile->getFilename().str()) == 0) {
         continue;
       }
-      archiveFile(config, *sourceFile);
-      encounteredModules = extractDeclarations(config, compiler, *module, sourceFile);
+      archiveFile(state.configuration, *sourceFile);
+      encounteredModules = extractDeclarations(state, compiler, *module, sourceFile);
     }
     if (!isFromSourceFile) {
-      encounteredModules = extractDeclarations(config, compiler, *module);
+      encounteredModules = extractDeclarations(state, compiler, *module);
     }
     for (auto encountered : encounteredModules) {
       if (seen.count(encountered) == 0) {

swift/extractor/SwiftExtractor.h

@@ -1,11 +1,10 @@
 #pragma once
 
-#include "swift/extractor/config/SwiftExtractorConfiguration.h"
+#include "swift/extractor/config/SwiftExtractorState.h"
 #include <swift/AST/SourceFile.h>
 #include <swift/Frontend/Frontend.h>
 #include <memory>
 
 namespace codeql {
-void extractSwiftFiles(const SwiftExtractorConfiguration& config,
-                       swift::CompilerInstance& compiler);
+void extractSwiftFiles(SwiftExtractorState& state, swift::CompilerInstance& compiler);
 }  // namespace codeql

swift/extractor/TargetTrapDomain.cpp

@@ -1,13 +1,15 @@
 #include "swift/extractor/TargetTrapDomain.h"
 #include <iomanip>
 namespace codeql {
-std::optional<TrapDomain> createTargetTrapDomain(const SwiftExtractorConfiguration& configuration,
+std::optional<TrapDomain> createTargetTrapDomain(SwiftExtractorState& state,
                                                  const std::filesystem::path& target) {
   auto trap = target;
   trap += ".trap";
-  if (auto ret = TargetFile::create(trap, configuration.trapDir, configuration.getTempTrapDir())) {
+  state.traps.push_back(trap.relative_path());
+  if (auto ret = TargetFile::create(trap, state.configuration.trapDir,
+                                    state.configuration.getTempTrapDir())) {
     *ret << "/* extractor-args:\n";
-    for (const auto& opt : configuration.frontendOptions) {
+    for (const auto& opt : state.configuration.frontendOptions) {
       *ret << "  " << std::quoted(opt) << " \\\n";
     }
     *ret << "\n*/\n";

swift/extractor/TargetTrapDomain.h

@@ -1,11 +1,11 @@
 #pragma once
 
 #include "swift/extractor/trap/TrapDomain.h"
-#include "swift/extractor/config/SwiftExtractorConfiguration.h"
+#include "swift/extractor/config/SwiftExtractorState.h"
 
 namespace codeql {
 
-std::optional<TrapDomain> createTargetTrapDomain(const SwiftExtractorConfiguration& configuration,
+std::optional<TrapDomain> createTargetTrapDomain(SwiftExtractorState& state,
                                                  const std::filesystem::path& target);
 
 }  // namespace codeql

swift/extractor/config/SwiftExtractorState.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <vector>
+#include <filesystem>
+
+#include <swift/AST/Decl.h>
+
+#include "swift/extractor/config/SwiftExtractorConfiguration.h"
+
+namespace codeql {
+struct SwiftExtractorState {
+  const SwiftExtractorConfiguration configuration;
+
+  // All the trap files related to this extraction. This may also include trap files generated in a
+  // previous run but that this run requested as well. Paths are relative to `configuration.trapDir`
+  std::vector<std::filesystem::path> traps;
+};
+
+}  // namespace codeql

swift/extractor/main.cpp

@@ -21,12 +21,12 @@
 using namespace std::string_literals;
 
 // must be called before processFrontendOptions modifies output paths
-static void lockOutputSwiftModuleTraps(const codeql::SwiftExtractorConfiguration& config,
+static void lockOutputSwiftModuleTraps(codeql::SwiftExtractorState& state,
                                        const swift::FrontendOptions& options) {
   for (const auto& input : options.InputsAndOutputs.getAllInputs()) {
     if (const auto& module = input.getPrimarySpecificPaths().SupplementaryOutputs.ModuleOutputPath;
         !module.empty()) {
-      if (auto target = codeql::createTargetTrapDomain(config, codeql::resolvePath(module))) {
+      if (auto target = codeql::createTargetTrapDomain(state, codeql::resolvePath(module))) {
         target->emit("// trap file deliberately empty\n"
                      "// this swiftmodule was created during the build, so its entities must have"
                      " been extracted directly from source files");
@@ -65,18 +65,18 @@ static void processFrontendOptions(swift::FrontendOptions& options) {
   }
 }
 
+codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state);
+
 // This is part of the swiftFrontendTool interface, we hook into the
 // compilation pipeline and extract files after the Swift frontend performed
 // semantic analysis
 class Observer : public swift::FrontendObserver {
  public:
-  explicit Observer(const codeql::SwiftExtractorConfiguration& config,
-                    codeql::SwiftDiagnosticsConsumer& diagConsumer)
-      : config{config}, diagConsumer{diagConsumer} {}
+  explicit Observer(const codeql::SwiftExtractorConfiguration& config) : state{config} {}
 
   void parsedArgs(swift::CompilerInvocation& invocation) override {
     auto& options = invocation.getFrontendOptions();
-    lockOutputSwiftModuleTraps(config, options);
+    lockOutputSwiftModuleTraps(state, options);
     processFrontendOptions(options);
   }
 
@@ -85,12 +85,13 @@ class Observer : public swift::FrontendObserver {
   }
 
   void performedSemanticAnalysis(swift::CompilerInstance& compiler) override {
-    codeql::extractSwiftFiles(config, compiler);
+    codeql::extractSwiftFiles(state, compiler);
   }
 
  private:
-  const codeql::SwiftExtractorConfiguration& config;
-  codeql::SwiftDiagnosticsConsumer& diagConsumer;
+  codeql::SwiftExtractorState state;
+  codeql::TrapDomain invocationTrap{invocationTrapDomain(state)};
+  codeql::SwiftDiagnosticsConsumer diagConsumer{invocationTrap};
 };
 
 static std::string getenv_or(const char* envvar, const std::string& def) {
@@ -145,11 +146,11 @@ static void checkWhetherToRunUnderTool(int argc, char* const* argv) {
 
 // Creates a target file that should store per-invocation info, e.g. compilation args,
 // compilations, diagnostics, etc.
-codeql::TrapDomain invocationTrapDomain(const codeql::SwiftExtractorConfiguration& configuration) {
+codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
   auto timestamp = std::chrono::system_clock::now().time_since_epoch().count();
   auto filename = std::to_string(timestamp) + '-' + std::to_string(getpid());
   auto target = std::filesystem::path("invocations") / std::filesystem::path(filename);
-  auto maybeDomain = codeql::createTargetTrapDomain(configuration, target);
+  auto maybeDomain = codeql::createTargetTrapDomain(state, target);
   if (!maybeDomain) {
     std::cerr << "Cannot create invocation trap file: " << target << "\n";
     abort();
@@ -183,9 +184,7 @@ int main(int argc, char** argv) {
 
   auto openInterception = codeql::setupFileInterception(configuration);
 
-  auto invocationDomain = invocationTrapDomain(configuration);
-  codeql::SwiftDiagnosticsConsumer diagConsumer(invocationDomain);
-  Observer observer(configuration, diagConsumer);
+  Observer observer(configuration);
   int frontend_rc = swift::performFrontend(configuration.frontendOptions, "swift-extractor",
                                            (void*)main, &observer);