JS: Overlay extraction support

Author: asgerf

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -465,10 +465,11 @@ public int run() throws IOException {
       try {
         CompletableFuture<?> sourceFuture = extractSource();
         sourceFuture.join(); // wait for source extraction to complete
-        if (hasSeenCode()) { // don't bother with the externs if no code was seen
+        if (hasSeenCode() && !isOverlayChangeMode()) { // don't bother with the externs if no code was seen
           extractExterns();
         }
         extractXml();
+        writeOverlayMetadata();
       } catch (OutOfMemoryError oom) {
         System.err.println("Out of memory while extracting the project.");
         return 137; // the CodeQL CLI will interpret this as an out-of-memory error
@@ -509,6 +510,21 @@ public int run() throws IOException {
     return 0;
   }
 
+  private void writeOverlayMetadata() {
+    String file = getEnvVar("CODEQL_EXTRACTOR_JAVASCRIPT_OVERLAY_BASE_METADATA_OUT");
+    if (file == null) {
+      // no overlay metadata file specified, so nothing to do
+      return;
+    }
+    // Write an empty string to the file as we currently have no metadata to emit.
+    // The file must be created for the database to recognized as an overlay base.
+    try {
+      Files.writeString(Paths.get(file), "", StandardCharsets.UTF_8);
+    } catch (IOException e) {
+      throw new ResourceError("Could not write overlay metadata to " + file, e);
+    }
+  }
+
   /**
    * A kind of error that can happen during extraction of JavaScript or TypeScript
    * code.
@@ -734,6 +750,17 @@ private CompletableFuture<?> extractSource() throws IOException {
     List<Path> tsconfigFiles = new ArrayList<>();
     findFilesToExtract(defaultExtractor, filesToExtract, tsconfigFiles);
 
+    OverlayChanges overlay = getOverlayChanges();
+    if (overlay != null) {
+      Set<Path> changedFiles = overlay.changes.stream()
+          .map(file -> Paths.get(file).toAbsolutePath())
+          .collect(Collectors.toSet());
+      int before = filesToExtract.size();
+      filesToExtract.retainAll(changedFiles);
+      int after = filesToExtract.size();
+      System.out.println("Overlay filter removed " + (before - after) + " out of " + before + " files from extraction.");
+    }
+
     tsconfigFiles = tsconfigFiles.stream()
          .sorted(PATH_ORDERING)
          .collect(Collectors.toList());
@@ -1338,6 +1365,18 @@ protected void extractXml() throws IOException {
     }
   }
 
+  private boolean isOverlayChangeMode() {
+    return getEnvVar("CODEQL_EXTRACTOR_JAVASCRIPT_OVERLAY_CHANGES") != null;
+  }
+
+  private OverlayChanges getOverlayChanges() throws IOException {
+    String jsonFile = getEnvVar("CODEQL_EXTRACTOR_JAVASCRIPT_OVERLAY_CHANGES");
+    if (jsonFile == null) {
+      return null;
+    }
+    return new Gson().fromJson(Files.newBufferedReader(Paths.get(jsonFile)), OverlayChanges.class);
+  }
+
   public static void main(String[] args) {
     try {
       System.exit(new AutoBuild().run());

javascript/extractor/src/com/semmle/js/extractor/OverlayChanges.java

@@ -0,0 +1,7 @@
+package com.semmle.js.extractor;
+
+import java.util.List;
+
+public class OverlayChanges {
+    public List<String> changes;
+}

javascript/resources/codeql-extractor.yml

@@ -6,6 +6,7 @@ display_name: "JavaScript/TypeScript"
 version: 1.22.1
 column_kind: "utf16"
 unicode_newlines: true
+overlay_support_version: 20250108
 build_modes:
   - none
 file_coverage_languages: