JS: Add environment variable to opt out of the behaviour if needed

Author: asgerf

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -408,8 +408,10 @@ private void setupFilters() {
     for (String extension : fileTypes.keySet()) patterns.add("**/*" + extension);
 
     // exclude files whose name strongly suggests they are minified
-    patterns.add("-**/*.min.js");
-    patterns.add("-**/*-min.js");
+    if (!EnvironmentVariables.allowMinifiedFiles()) {
+      patterns.add("-**/*.min.js");
+      patterns.add("-**/*-min.js");
+    }
 
     // exclude `node_modules` and `bower_components`
     patterns.add("-**/node_modules");
@@ -1074,6 +1076,7 @@ private ExtractorConfig mkExtractorConfig() {
     config = config.withSourceType(getSourceType());
     config = config.withVirtualSourceRoot(virtualSourceRoot);
     if (defaultEncoding != null) config = config.withDefaultEncoding(defaultEncoding);
+    config = config.withAllowMinified(EnvironmentVariables.allowMinifiedFiles());
     return config;
   }
 

javascript/extractor/src/com/semmle/js/extractor/EnvironmentVariables.java

@@ -101,4 +101,12 @@ public static String getWipDatabase() {
   public static boolean isActionsExtractor() {
     return Env.systemEnv().getNonEmpty(CODEQL_EXTRACTOR_ACTIONS_WIP_DATABASE_ENV_VAR) != null;
   }
+
+  public static boolean allowMinifiedFiles() {
+    String env = Env.systemEnv().getNonEmpty("CODEQL_EXTRACTOR_JAVASCRIPT_ALLOW_MINIFIED_FILES");
+    if (env == null) {
+      return false; // default is to not allow minified files
+    }
+    return Boolean.parseBoolean(env);
+  }
 }

javascript/extractor/src/com/semmle/js/extractor/ExtractorConfig.java

@@ -205,6 +205,9 @@ public Set<String> getPredefinedGlobals() {
   /** Should parse errors be reported as violations instead of aborting extraction? */
   private boolean tolerateParseErrors;
 
+  /** Should minified files be allowed? */
+  private boolean allowMinified;
+
   /** How should HTML files be extracted? */
   private HtmlPopulator.Config htmlHandling;
 
@@ -236,6 +239,7 @@ public ExtractorConfig(boolean experimental) {
     this.sourceType = SourceType.AUTO;
     this.htmlHandling = HtmlPopulator.Config.ELEMENTS;
     this.tolerateParseErrors = true;
+    this.allowMinified = false;
     if (experimental) {
       this.mozExtensions = true;
       this.jscript = true;
@@ -258,6 +262,7 @@ public ExtractorConfig(ExtractorConfig that) {
     this.v8Extensions = that.v8Extensions;
     this.e4x = that.e4x;
     this.tolerateParseErrors = that.tolerateParseErrors;
+    this.allowMinified = that.allowMinified;
     this.fileType = that.fileType;
     this.sourceType = that.sourceType;
     this.htmlHandling = that.htmlHandling;
@@ -357,6 +362,16 @@ public ExtractorConfig withTolerateParseErrors(boolean tolerateParseErrors) {
     return res;
   }
 
+  public boolean isAllowMinified() {
+    return allowMinified;
+  }
+
+  public ExtractorConfig withAllowMinified(boolean allowMinified) {
+    ExtractorConfig res = new ExtractorConfig(this);
+    res.allowMinified = allowMinified;
+    return res;
+  }
+
   public boolean hasFileType() {
     return fileType != null;
   }
@@ -467,6 +482,8 @@ public String toString() {
         + e4x
         + ", tolerateParseErrors="
         + tolerateParseErrors
+        + ", allowMinified="
+        + allowMinified
         + ", htmlHandling="
         + htmlHandling
         + ", fileType="

javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java

@@ -62,7 +62,7 @@ public ParseResultInfo extract(TextualExtractor textualExtractor) {
     LocationManager locationManager = textualExtractor.getLocationManager();
     String source = textualExtractor.getSource();
 
-    if (isMinified(source)) {
+    if (!config.isAllowMinified() && isMinified(source)) {
       return ParseResultInfo.skipped("File appears to be minified.");
     }