From 14f50880e973e5dc874e0fca81e1f1782575f44a Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 3 Jun 2025 09:44:22 +0000 Subject: [PATCH 1/7] JavaScript: Don't extract files in `tsconfig.json` `outDir` --- .../tsconfig/CompilerOptions.java | 13 +++++++++++ .../dependencies/tsconfig/TsConfigJson.java | 13 +++++++++++ .../com/semmle/js/extractor/AutoBuild.java | 22 +++++++++++++++++++ .../js/extractor/test/AutoBuildTests.java | 18 +++++++++++++++ 4 files changed, 66 insertions(+) create mode 100644 javascript/extractor/src/com/semmle/js/dependencies/tsconfig/CompilerOptions.java create mode 100644 javascript/extractor/src/com/semmle/js/dependencies/tsconfig/TsConfigJson.java diff --git a/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/CompilerOptions.java b/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/CompilerOptions.java new file mode 100644 index 000000000000..63bec753ac91 --- /dev/null +++ b/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/CompilerOptions.java @@ -0,0 +1,13 @@ +package com.semmle.js.dependencies.tsconfig; + +public class CompilerOptions { + private String outDir; + + public String getOutDir() { + return outDir; + } + + public void setOutDir(String outDir) { + this.outDir = outDir; + } +} diff --git a/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/TsConfigJson.java b/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/TsConfigJson.java new file mode 100644 index 000000000000..08dbe6631390 --- /dev/null +++ b/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/TsConfigJson.java @@ -0,0 +1,13 @@ +package com.semmle.js.dependencies.tsconfig; + +public class TsConfigJson { + private CompilerOptions compilerOptions; + + public CompilerOptions getCompilerOptions() { + return compilerOptions; + } + + public void setCompilerOptions(CompilerOptions compilerOptions) { + this.compilerOptions = compilerOptions; + } +} diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 22d008637c9a..44fa2f9209ac 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -39,6 +39,8 @@ import com.google.gson.Gson; import com.google.gson.JsonParseException; +import com.semmle.js.dependencies.tsconfig.TsConfigJson; +import com.semmle.js.dependencies.tsconfig.CompilerOptions; import com.semmle.js.dependencies.AsyncFetcher; import com.semmle.js.dependencies.DependencyResolver; import com.semmle.js.dependencies.packument.PackageJson; @@ -745,6 +747,26 @@ private CompletableFuture extractSource() throws IOException { .filter(p -> !isFileTooLarge(p)) .sorted(PATH_ORDERING) .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); + // exclude files in output directories as configured in tsconfig.json + final List outDirs = new ArrayList<>(); + for (Path cfg : tsconfigFiles) { + try { + String txt = new WholeIO().read(cfg); + TsConfigJson root = new Gson().fromJson(txt, TsConfigJson.class); + if (root != null && root.getCompilerOptions() != null) { + if (root.getCompilerOptions().getOutDir() == null) { + // no outDir specified, so skip this tsconfig.json + continue; + } + Path odir = cfg.getParent().resolve(root.getCompilerOptions().getOutDir()).toAbsolutePath().normalize(); + outDirs.add(odir); + } + } catch (Exception e) { + // ignore malformed tsconfig or missing fields + } + } + // exclude files in output directories as configured in tsconfig.json + filesToExtract.removeIf(f -> outDirs.stream().anyMatch(od -> f.startsWith(od))); DependencyInstallationResult dependencyInstallationResult = DependencyInstallationResult.empty; if (!tsconfigFiles.isEmpty()) { diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java index 0a924d54319a..532e29e6e312 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java @@ -203,6 +203,24 @@ public void typescriptWrongConfig() throws IOException { runTest(); } + @Test + public void skipFilesInTsconfigOutDir() throws IOException { + envVars.put("LGTM_INDEX_TYPESCRIPT", "basic"); + // Files under outDir in tsconfig.json should be excluded + // Create tsconfig.json with outDir set to "dist" + addFile(true, LGTM_SRC, "tsconfig.json"); + Path config = Paths.get(LGTM_SRC.toString(), "tsconfig.json"); + Files.write(config, + "{\"compilerOptions\":{\"outDir\":\"dist\"}}".getBytes(StandardCharsets.UTF_8)); + // Add files outside outDir (should be extracted) + addFile(true, LGTM_SRC, "src", "app.ts"); + addFile(true, LGTM_SRC, "main.js"); + // Add files under dist/outDir (should be skipped) + addFile(false, LGTM_SRC, "dist", "generated.js"); + addFile(false, LGTM_SRC, "dist", "sub", "x.js"); + runTest(); + } + @Test public void includeFile() throws IOException { envVars.put("LGTM_INDEX_INCLUDE", "tst.js"); From 8829f7820a0fd075492cbcf189aa5357f84c4dfe Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 3 Jun 2025 09:44:52 +0000 Subject: [PATCH 2/7] JavaScript: Don't extract files with TypeScript progenitors --- .../com/semmle/js/extractor/AutoBuild.java | 14 ++++++++++++-- .../js/extractor/test/AutoBuildTests.java | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 44fa2f9209ac..ca69bf06791f 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -818,9 +818,19 @@ private CompletableFuture extractFiles( */ private boolean isFileDerivedFromTypeScriptFile(Path path, Set extractedFiles) { String name = path.getFileName().toString(); - if (!name.endsWith(".js")) + // only skip JS variants when a corresponding TS/TSX file was already extracted + if (!(name.endsWith(".js") + || name.endsWith(".cjs") + || name.endsWith(".mjs") + || name.endsWith(".jsx") + || name.endsWith(".cjsx") + || name.endsWith(".mjsx"))) { return false; - String stem = name.substring(0, name.length() - ".js".length()); + } + // strip off extension + int dot = name.lastIndexOf('.'); + String stem = dot != -1 ? name.substring(0, dot) : name; + // if a TS/TSX file with same base name was extracted, skip this file for (String ext : FileType.TYPESCRIPT.getExtensions()) { if (extractedFiles.contains(path.getParent().resolve(stem + ext))) { return true; diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java index 532e29e6e312..4a93a97cc58c 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java @@ -203,6 +203,25 @@ public void typescriptWrongConfig() throws IOException { runTest(); } + @Test + public void skipJsFilesDerivedFromTypeScriptFiles() throws IOException { + // JS-derived files (.js, .cjs, .mjs, .jsx, .cjsx, .mjsx) should be skipped when TS indexing + envVars.put("LGTM_INDEX_TYPESCRIPT", "basic"); + // Add TypeScript sources + addFile(true, LGTM_SRC, "foo.ts"); + addFile(true, LGTM_SRC, "bar.tsx"); + // Add derived JS variants (should be skipped) + addFile(false, LGTM_SRC, "foo.js"); + addFile(false, LGTM_SRC, "bar.jsx"); + addFile(false, LGTM_SRC, "foo.cjs"); + addFile(false, LGTM_SRC, "foo.mjs"); + addFile(false, LGTM_SRC, "bar.cjsx"); + addFile(false, LGTM_SRC, "bar.mjsx"); + // A normal JS file without TS counterpart should be extracted + addFile(true, LGTM_SRC, "normal.js"); + runTest(); + } + @Test public void skipFilesInTsconfigOutDir() throws IOException { envVars.put("LGTM_INDEX_TYPESCRIPT", "basic"); From 619256e03716dabcdead1321ddf9d491cbfc918a Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 5 Jun 2025 14:59:40 +0000 Subject: [PATCH 3/7] JavaScript: Fix existing tests and test runner Fixes two things: - The basic test should no longer extract `tst.js` (as `tst.ts` is present) - The `AutoBuild` mock did not populate `extractedFiles` correctly, which broke the logic that looks for TypeScript files with the same basename. --- .../test/com/semmle/js/extractor/test/AutoBuildTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java index 4a93a97cc58c..28c8e593dcd1 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java @@ -135,6 +135,7 @@ public void extractTypeScriptFiles( FileExtractors extractors) { for (Path f : files) { actual.add(f.toString()); + extractedFiles.add(f); } } @@ -175,7 +176,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) @Test public void basicTest() throws IOException { - addFile(true, LGTM_SRC, "tst.js"); + addFile(false, LGTM_SRC, "tst.js"); addFile(true, LGTM_SRC, "tst.ts"); addFile(true, LGTM_SRC, "tst.html"); addFile(true, LGTM_SRC, "tst.xsjs"); From 281ccf7c11f5ee495a97408b292490695ef5dad8 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 5 Jun 2025 15:01:05 +0000 Subject: [PATCH 4/7] JavaScript: Extract `tsconfig.json` also in `basic` mode This is needed for the logic that skips files inside the directory specified in the `tsconfig.json` `outDir` compiler option. --- javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index ca69bf06791f..b8105db6b392 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -1186,7 +1186,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) } // extract TypeScript projects from 'tsconfig.json' - if (typeScriptMode == TypeScriptMode.FULL + if (typeScriptMode != TypeScriptMode.NONE && treatAsTSConfig(file.getFileName().toString()) && !excludes.contains(file) && isFileIncluded(file)) { From b8772bc73603b6dfa1da6dd80f67e659694888db Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 5 Jun 2025 15:06:40 +0000 Subject: [PATCH 5/7] JavaScript: Add change note --- .../change-notes/2025-06-05-skip-obviously-generated-files.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 javascript/ql/lib/change-notes/2025-06-05-skip-obviously-generated-files.md diff --git a/javascript/ql/lib/change-notes/2025-06-05-skip-obviously-generated-files.md b/javascript/ql/lib/change-notes/2025-06-05-skip-obviously-generated-files.md new file mode 100644 index 000000000000..16d81cb4cc30 --- /dev/null +++ b/javascript/ql/lib/change-notes/2025-06-05-skip-obviously-generated-files.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* The JavaScript extractor now skips generated JavaScript files if the original TypeScript files are already present. It also skips any files in the output directory specified in the `compilerOptions` part of the `tsconfig.json` file. From f08c2fa3875a0898e875dbeac327db690934f07a Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 10 Jun 2025 12:58:16 +0000 Subject: [PATCH 6/7] JavaScript: Move tsconfig files into `extractor.tsconfig` package Also make the indentation in `CompilerOptions.java` more consistent. --- .../js/dependencies/tsconfig/CompilerOptions.java | 13 ------------- .../src/com/semmle/js/extractor/AutoBuild.java | 4 ++-- .../js/extractor/tsconfig/CompilerOptions.java | 13 +++++++++++++ .../tsconfig/TsConfigJson.java | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) delete mode 100644 javascript/extractor/src/com/semmle/js/dependencies/tsconfig/CompilerOptions.java create mode 100644 javascript/extractor/src/com/semmle/js/extractor/tsconfig/CompilerOptions.java rename javascript/extractor/src/com/semmle/js/{dependencies => extractor}/tsconfig/TsConfigJson.java (85%) diff --git a/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/CompilerOptions.java b/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/CompilerOptions.java deleted file mode 100644 index 63bec753ac91..000000000000 --- a/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/CompilerOptions.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.semmle.js.dependencies.tsconfig; - -public class CompilerOptions { - private String outDir; - - public String getOutDir() { - return outDir; - } - - public void setOutDir(String outDir) { - this.outDir = outDir; - } -} diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index b8105db6b392..661f315bf4a7 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -39,8 +39,8 @@ import com.google.gson.Gson; import com.google.gson.JsonParseException; -import com.semmle.js.dependencies.tsconfig.TsConfigJson; -import com.semmle.js.dependencies.tsconfig.CompilerOptions; +import com.semmle.js.extractor.tsconfig.TsConfigJson; +import com.semmle.js.extractor.tsconfig.CompilerOptions; import com.semmle.js.dependencies.AsyncFetcher; import com.semmle.js.dependencies.DependencyResolver; import com.semmle.js.dependencies.packument.PackageJson; diff --git a/javascript/extractor/src/com/semmle/js/extractor/tsconfig/CompilerOptions.java b/javascript/extractor/src/com/semmle/js/extractor/tsconfig/CompilerOptions.java new file mode 100644 index 000000000000..fa7b664f2eb0 --- /dev/null +++ b/javascript/extractor/src/com/semmle/js/extractor/tsconfig/CompilerOptions.java @@ -0,0 +1,13 @@ +package com.semmle.js.extractor.tsconfig; + +public class CompilerOptions { + private String outDir; + + public String getOutDir() { + return outDir; + } + + public void setOutDir(String outDir) { + this.outDir = outDir; + } +} diff --git a/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/TsConfigJson.java b/javascript/extractor/src/com/semmle/js/extractor/tsconfig/TsConfigJson.java similarity index 85% rename from javascript/extractor/src/com/semmle/js/dependencies/tsconfig/TsConfigJson.java rename to javascript/extractor/src/com/semmle/js/extractor/tsconfig/TsConfigJson.java index 08dbe6631390..9e12d5cc0aa9 100644 --- a/javascript/extractor/src/com/semmle/js/dependencies/tsconfig/TsConfigJson.java +++ b/javascript/extractor/src/com/semmle/js/extractor/tsconfig/TsConfigJson.java @@ -1,4 +1,4 @@ -package com.semmle.js.dependencies.tsconfig; +package com.semmle.js.extractor.tsconfig; public class TsConfigJson { private CompilerOptions compilerOptions; From e3d9d92f25d23f5d102e7234be09e1c8cf30b9bd Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 10 Jun 2025 12:59:03 +0000 Subject: [PATCH 7/7] JavaScript: Fix duplicate comment --- javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java index 661f315bf4a7..f96211bd5c41 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java +++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java @@ -747,7 +747,7 @@ private CompletableFuture extractSource() throws IOException { .filter(p -> !isFileTooLarge(p)) .sorted(PATH_ORDERING) .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); - // exclude files in output directories as configured in tsconfig.json + // gather all output directories specified in tsconfig.json files final List outDirs = new ArrayList<>(); for (Path cfg : tsconfigFiles) { try {