Merge pull request #510 from xiemaisi/js/exclude-minified

Approved by asger-semmle

Author: semmle-qlci

change-notes/1.19/extractor-javascript.md

@@ -16,8 +16,16 @@
 
 ## General improvements
 
-> Changes that affect alerts in many files or from many queries
-> For example, changes to file classification
+* On LGTM, files whose name ends in `.min.js` or `-min.js` are no longer extracted by default, since they most likely contain minified code and results in these files would be hidden by default anyway. To extract such files anyway, you can add the following filters to your `lgtm.yml` file (or add them to existing filters):
+
+```yaml
+extraction:
+  javascript:
+    index:
+      filters:
+        - include: "**/*.min.js"
+        - include: "**/*-min.js"
+```
 
 ## Changes to code extraction
 

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -111,7 +111,7 @@
  *
  * <p>
  * The filtering phase is parameterised by a list of include/exclude patterns in the style of
- * {@link ProjectLayout} specifications. There are some built-in include patterns discussed
+ * {@link ProjectLayout} specifications. There are some built-in include/exclude patterns discussed
  * below. Additionally, the environment variable <code>LGTM_INDEX_FILTERS</code> is interpreted
  * as a newline-separated list of patterns to append to that list (hence taking precedence over
  * the built-in patterns). Unlike for {@link ProjectLayout}, patterns in
@@ -140,6 +140,15 @@
  * </p>
  *
  * <p>
+ * The default exclusion patterns cause the following files to be excluded:
+ * </p>
+ * <ul>
+ * <li>All JavaScript files whose name ends with <code>-min.js</code> or <code>.min.js</code>.
+ *     Such files typically contain minified code. Since LGTM by default does not show results
+ *     in minified files, it is not usually worth extracting them in the first place.</li>
+ * </ul>
+ *
+ * <p>
  * JavaScript files are normally extracted with {@link SourceType#AUTO}, but an explicit
  * source type can be specified in the environment variable <code>LGTM_INDEX_SOURCE_TYPE</code>.
  * </p>
@@ -317,6 +326,10 @@ private void setupFilters() {
 		patterns.add("**/.eslintrc*");
 		patterns.add("**/package.json");
 
+		// exclude files whose name strongly suggests they are minified
+		patterns.add("-**/*.min.js");
+		patterns.add("-**/*-min.js");
+
 		String base = LGTM_SRC.toString().replace('\\', '/');
 		// process `$LGTM_INDEX_FILTERS`
 		for (String pattern : Main.NEWLINE.split(getEnvVar("LGTM_INDEX_FILTERS", ""))) {

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -433,4 +433,23 @@ public void excludeNonExistentFile() throws IOException {
 		addFile(true, LGTM_SRC, "tst.js");
 		runTest();
 	}
+
+	@Test
+	public void minifiedFilesAreExcluded() throws IOException {
+		addFile(true, LGTM_SRC, "admin.js");
+		addFile(false, LGTM_SRC, "jquery.min.js");
+		addFile(false, LGTM_SRC, "lib", "lodash-min.js");
+		addFile(true, LGTM_SRC, "compute_min.js");
+		runTest();
+	}
+
+	@Test
+	public void minifiedFilesCanBeReIncluded() throws IOException {
+		envVars.put("LGTM_INDEX_FILTERS", "include:**/*.min.js\ninclude:**/*-min.js");
+		addFile(true, LGTM_SRC, "admin.js");
+		addFile(true, LGTM_SRC, "jquery.min.js");
+		addFile(true, LGTM_SRC, "lib", "lodash-min.js");
+		addFile(true, LGTM_SRC, "compute_min.js");
+		runTest();
+	}
 }