C#: Only use small files during file content reference analysis.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -60,7 +60,8 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
             var allFiles = GetFiles("*.*").ToList();
-            this.fileContent = new FileContent(progressMonitor, GetFileNames(allFiles));
+            var smallFiles = GetSmallFiles(allFiles);
+            this.fileContent = new FileContent(progressMonitor, GetFileNames(smallFiles));
             this.allSources = GetFileNames(allFiles, ".cs").ToList();
             var allProjects = GetFileNames(allFiles, ".csproj");
             var solutions = options.SolutionFile is not null
@@ -194,6 +195,20 @@ private IEnumerable<FileInfo> GetFiles(string pattern, bool recurseSubdirectorie
         private static IEnumerable<string> GetFileNames(IEnumerable<FileInfo> files, params string[] extensions) =>
             files.Where(fi => !extensions.Any() || extensions.Contains(fi.Extension)).Select(fi => fi.FullName);
 
+        private IEnumerable<FileInfo> GetSmallFiles(IEnumerable<FileInfo> files)
+        {
+            const int oneMb = 1_048_576;
+            return files.Where(file =>
+            {
+                if (file.Length > oneMb)
+                {
+                    progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
+                    return false;
+                }
+                return true;
+            });
+        }
+
         /// <summary>
         /// Computes a unique temp directory for the packages associated
         /// with this source tree. Use a SHA1 of the directory name.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs

@@ -18,7 +18,7 @@ public void Log(Severity severity, string message) =>
         public void LogInfo(string message) =>
             logger.Log(Severity.Info, message);
 
-        private void LogDebug(string message) =>
+        public void LogDebug(string message) =>
             logger.Log(Severity.Debug, message);
 
         private void LogError(string message) =>