Update and fix search expressions for inclusion and exclusion of folders and files

Author: malte-hansen

README.md

@@ -13,7 +13,7 @@ The property `explorviz.gitanalysis.run-mode` determines if the code-agent runs
     ./gradlew quarkusDev
     ```
 2. Visit [http://localhost:8078/](http://localhost:8078/). The new single-page UI is served directly from `src/main/resources/META-INF/resources/index.html`.
-3. Fill in either a local repository path or a remote URL (plus any optional parameters such as branch, source directories, credentials, metrics toggles, etc.).
+3. Fill in either a local repository path or a remote URL (plus any optional parameters such as branch,filters, credentials, metrics toggles, etc.).
 4. Hit **Run Analysis**. The page calls the existing REST endpoint at `/api/analysis/trigger` and streams the textual response back into the UI.
 
 The form mirrors the fields of `AnalysisRequest`, so anything you can configure via JSON can now be triggered from the browser.
@@ -121,21 +121,19 @@ Type: Boolean or Empty (defaults to false)
 If a remote storage is used and this is set to true, the analysis data will be sent to the remote endpoint, if set to false, the analysis data will be stored as json on disc.
 The storage location will be printed on startup and is relative to the java working directory.
 
-### explorviz.gitanalysis.source-directory
+### explorviz.gitanalysis.include-in-analysis-expressions
 
 Type: String or empty
 
-To detect java types successfully, the source directory should be specified.
-Source files are expected to be somewhere inside the given folders.
-Provide one or multiple [search expressions](#search-expressions).
+Only files contained in the folders matching one of the search expressions are analyzed.
+Provide one or multiple search expressions.
 
-### explorviz.gitanalysis.restrict-analysis-to-folders
+### explorviz.gitanalysis.exclude-from-analysis-expressions
 
 Type: String or empty
 
-Only java files contained in the reachable folders from this search expression are analyzed.
-Type detection is not affected by this setting, any files reachable by the [source directory setting](#explorvizgitanalysissource-directory) are used to detect the correct type.
-Provide one or multiple [search expressions](#search-expressions).
+Only files with a filename matching one of the search expressions are analyzed.
+Provide one or multiple search expressions.
 
 ### explorviz.gitanalysis.start-commit-sha1
 
@@ -182,29 +180,10 @@ If the folder location changes over the course of the development, it is possibl
 
 Leading Wildcard:
 
-E.g. the folder location `*/src/main/java` is searched in the repository everywhere, until some folder hierarchy matches `src/main/java` for example the path `project/javasources/src/main/java`.
+E.g. the folder location `**/src/main/java/**` is searched in the repository everywhere, until some folder hierarchy matches `src/main/java` for example the path `project/javasources/src/main/java`.
 The first match is used, if there are multiple matches, try to specify the path even more.
 Keep in mind to not use a line separator in front of or after the wildcard.
 
 Infix Wildcards:
 
-E.g. the folder location `/src/*/java` is searched in the repository starting with `src` and must end with `java`.
-The path `src/main/java` would match as well as `/src/some/deep/hierarchy/to/search/java`.
-If both folder structures would exist, one of these could be returned as the folders are not searched in a specific order.
-Make sure the folder is unambiguous.
-
-Single wildcards do not guarantee a deeper folder level, the search string `/some/*/path` will match the folder `some/path` even if `some/other/path/` exists and might be the wanted.
-
-Multiple consecutive wildcards e.g. `/some/*/*/path` enforce a depth of _number of wildcards - 1_.
-This search string matches `/some/other/path/`, `/some/deeper/other/path/` or even deeper but not `/some/path/`.
-
-**WARNING: paths ending with wildcards are not allowed!**
-
-<!-- The following can produce syntax errors
-#### Optional Paths
-
-If a folder, like the test folder, is not present from the first commit onwards, it is possible to define these type of
-folders as optional. Optional folders are considered if they are found and ignored if not present.
-
-To define a search expression as optional, simply put it in brackets: ``[\src\test\java]``. Wildcards are supported in
-optional search expressions. -->
+Multiple consecutive wildcards e.g. `/some/*/*/path` enforce a certain depth of the directories.

build.gradle

@@ -113,20 +113,3 @@ test {
     useJUnitPlatform()
     testLogging.showStandardStreams = true
 }
-
-def registerGitHook = { taskName, hookFile, targetHook ->
-  tasks.register(taskName, Copy) {
-    from("code-analysis/${hookFile}")
-    into(".git/hooks")
-    rename { targetHook }
-  }
-}
-
-registerGitHook("registerPreCommitHook", "pre-commit", "pre-commit")
-registerGitHook("registerPreMergeCommitHook", "pre-commit", "pre-merge-commit")
-
-tasks.named("quarkusGenerateCode") {
-  dependsOn("registerPreCommitHook")
-  dependsOn("registerPreMergeCommitHook")
-}
-

src/main/java/net/explorviz/code/analysis/GitAnalysis.java

@@ -49,11 +49,11 @@ public class GitAnalysis { // NOPMD
   @ConfigProperty(name = "explorviz.gitanalysis.branch")
   /* default */ Optional<String> repositoryBranchProperty; // NOCS
 
-  @ConfigProperty(name = "explorviz.gitanalysis.source-directory")
-  /* default */ Optional<String> sourceDirectoryProperty; // NOCS
+  @ConfigProperty(name = "explorviz.gitanalysis.include-in-analysis-expressions")
+  /* default */ Optional<String> includeInAnalysisExpressionsProperty; // NOCS NOPMD
 
-  @ConfigProperty(name = "explorviz.gitanalysis.restrict-analysis-to-folders")
-  /* default */ Optional<String> restrictAnalysisToFoldersProperty; // NOCS NOPMD
+  @ConfigProperty(name = "explorviz.gitanalysis.exclude-from-analysis-expressions")
+  /* default */ Optional<String> excludeFromAnalysisExpressionsProperty; // NOCS NOPMD
 
   @ConfigProperty(name = "explorviz.gitanalysis.application-root")
   /* default */ Optional<String> applicationRootProperty; // NOCS
@@ -79,9 +79,6 @@ public class GitAnalysis { // NOPMD
   @ConfigProperty(name = "explorviz.gitanalysis.application-name")
   /* default */ String applicationNameProperty; // NOCS
 
-  @ConfigProperty(name = "explorviz.gitanalysis.code-analysis-excluded-file-extensions", defaultValue = "")
-  /* default */ Optional<String> codeAnalysisExcludedFileExtensionsProperty; // NOCS
-
   @Inject
   /* package */ GrpcExporter grpcExporter; // NOCS
 
@@ -100,16 +97,15 @@ private AnalysisConfig createConfig() {
         .gitUsername(usernameProperty)
         .gitPassword(passwordProperty)
         .branch(repositoryBranchProperty)
-        .sourceDirectory(sourceDirectoryProperty)
-        .restrictAnalysisToFolders(restrictAnalysisToFoldersProperty)
+        .includeInAnalysisExpressions(includeInAnalysisExpressionsProperty)
+        .excludeFromAnalysisExpressions(excludeFromAnalysisExpressionsProperty)
         .applicationRoot(applicationRootProperty)
         .calculateMetrics(calculateMetricsProperty)
         .startCommit(startCommitProperty)
         .endCommit(endCommitProperty)
         .commitAnalysisLimit(commitAnalysisLimitProperty)
         .landscapeToken(landscapeTokenProperty)
         .applicationName(applicationNameProperty)
-        .codeAnalysisExcludedFileExtensions(codeAnalysisExcludedFileExtensionsProperty.orElse(""))
         .build();
   }
 

src/main/java/net/explorviz/code/analysis/api/AnalysisRequest.java

@@ -14,8 +14,8 @@ public class AnalysisRequest {
   private String username;
   private String password;
   private String branch;
-  private String sourceDirectory;
-  private String restrictAnalysisToFolders;
+  private String includeInAnalysisExpressions;
+  private String excludeFromAnalysisExpressions;
 
   private boolean sendToRemote = true;
   private boolean calculateMetrics = true;
@@ -25,7 +25,6 @@ public class AnalysisRequest {
   private String landscapeToken = "mytokenvalue";
   private String applicationName = "";
   private String applicationRoot;
-  private String codeAnalysisExcludedFileExtensions;
 
   public AnalysisRequest() {
   }
@@ -70,20 +69,20 @@ public void setBranch(final String branch) {
     this.branch = branch;
   }
 
-  public String getSourceDirectory() {
-    return sourceDirectory;
+  public String getIncludeInAnalysisExpressions() {
+    return includeInAnalysisExpressions;
   }
 
-  public void setSourceDirectory(final String sourceDirectory) {
-    this.sourceDirectory = sourceDirectory;
+  public void setIncludeInAnalysisExpressions(final String includeInAnalysisExpressions) {
+    this.includeInAnalysisExpressions = includeInAnalysisExpressions;
   }
 
-  public String getRestrictAnalysisToFolders() {
-    return restrictAnalysisToFolders;
+  public String getExcludeFromAnalysisExpressions() {
+    return excludeFromAnalysisExpressions;
   }
 
-  public void setRestrictAnalysisToFolders(final String restrictAnalysisToFolders) {
-    this.restrictAnalysisToFolders = restrictAnalysisToFolders;
+  public void setExcludeFromAnalysisExpressions(final String excludeFromAnalysisExpressions) {
+    this.excludeFromAnalysisExpressions = excludeFromAnalysisExpressions;
   }
 
   public String getApplicationRoot() {
@@ -142,7 +141,6 @@ public void setApplicationName(final String applicationName) {
     this.applicationName = applicationName;
   }
 
-
   public Integer getCommitAnalysisLimit() {
     return commitAnalysisLimit;
   }
@@ -151,14 +149,6 @@ public void setCommitAnalysisLimit(final Integer commitAnalysisLimit) {
     this.commitAnalysisLimit = commitAnalysisLimit;
   }
 
-  public String getCodeAnalysisExcludedFileExtensions() {
-    return codeAnalysisExcludedFileExtensions;
-  }
-
-  public void setCodeAnalysisExcludedFileExtensions(final String codeAnalysisExcludedFileExtensions) {
-    this.codeAnalysisExcludedFileExtensions = codeAnalysisExcludedFileExtensions;
-  }
-
   /**
    * Converts this request to an AnalysisConfig.
    *
@@ -171,16 +161,15 @@ public AnalysisConfig toConfig() {
         .gitUsername(Optional.ofNullable(username))
         .gitPassword(Optional.ofNullable(password))
         .branch(Optional.ofNullable(branch))
-        .sourceDirectory(Optional.ofNullable(sourceDirectory))
-        .restrictAnalysisToFolders(Optional.ofNullable(restrictAnalysisToFolders))
+        .includeInAnalysisExpressions(Optional.ofNullable(includeInAnalysisExpressions))
+        .excludeFromAnalysisExpressions(Optional.ofNullable(excludeFromAnalysisExpressions))
         .calculateMetrics(calculateMetrics)
         .startCommit(Optional.ofNullable(startCommit))
         .endCommit(Optional.ofNullable(endCommit))
         .commitAnalysisLimit(Optional.ofNullable(commitAnalysisLimit))
         .landscapeToken((landscapeToken != null && !landscapeToken.isBlank()) ? landscapeToken : "mytokenvalue")
         .applicationName(applicationName != null ? applicationName : "")
         .applicationRoot(Optional.ofNullable(applicationRoot))
-        .codeAnalysisExcludedFileExtensions(codeAnalysisExcludedFileExtensions)
         .build();
   }
 }

src/main/java/net/explorviz/code/analysis/git/DirectoryFinder.java

@@ -1,21 +1,16 @@
 package net.explorviz.code.analysis.git;
 
 import java.io.IOException;
-import java.nio.file.FileSystems;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.PathMatcher;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import net.explorviz.code.analysis.exceptions.MalformedPathException;
 import net.explorviz.code.analysis.exceptions.NotFoundException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,26 +21,8 @@
 public final class DirectoryFinder {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DirectoryFinder.class);
-  private static final Map<String, List<String>> PATHS = new HashMap<>();
 
   private DirectoryFinder() {
-
-  }
-
-  /**
-   * Resets the given path entry, the saved value will be removed.
-   *
-   * @param path the search string for the path used to create it
-   */
-  public static void resetDirectory(final String path) {
-    PATHS.remove(path);
-  }
-
-  /**
-   * Resets the internal path storage.
-   */
-  public static void reset() {
-    PATHS.clear();
   }
 
   /**
@@ -86,10 +63,15 @@ public static List<String> getDirectories(String root, List<String> searchPaths)
         public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
             throws IOException {
           for (String searchPath : searchPaths) {
-            PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + searchPath);
+            String glob = searchPath;
+            if (!glob.startsWith("glob:") && !glob.startsWith("regex:")) {
+              glob = "glob:" + glob;
+            }
+            java.nio.file.PathMatcher matcher = java.nio.file.FileSystems.getDefault().getPathMatcher(glob);
             if (matcher.matches(startPath.relativize(dir))) {
               pathSet.add(dir.toAbsolutePath().normalize()
                   .toString());  // Normalize the path and add to the set
+              LOGGER.atTrace().addArgument(dir).log("Directory matched glob and was added to set: {}");
             }
           }
           return FileVisitResult.CONTINUE;

src/main/java/net/explorviz/code/analysis/git/GitRepositoryHandler.java

@@ -536,6 +536,7 @@ private List<FileDescriptor> listFilesInCommit(final Repository repository, // N
         objectIdList.add(new FileDescriptor(treeWalk.getObjectId(0), treeWalk.getNameString(),
             treeWalk.getPathString()));
       }
+      LOGGER.atDebug().addArgument(objectIdList.size()).log("Discovered {} files in commit tree");
     }
     return objectIdList;
 
@@ -545,8 +546,10 @@ private TreeFilter getSourceFileTreeFilter(final List<String> pathRestrictions)
       throws NotFoundException {
     if (pathRestrictions.isEmpty() || pathRestrictions.size() == 1 && pathRestrictions.get(0)
         .isBlank()) {
+      LOGGER.atInfo().log("No path restrictions provided. Analyzing all files.");
       return TreeFilter.ALL;
     } else {
+      LOGGER.atInfo().addArgument(pathRestrictions).log("Applying path restrictions: {}");
       final List<String> pathList = DirectoryFinder.getRelativeDirectory(pathRestrictions,
           getCurrentRepositoryPath());
       final List<String> newPathList = new ArrayList<>();