From e3a04757d7cd8f059a9cd814ee914f82e79682bd Mon Sep 17 00:00:00 2001
From: Dilan Bhalla <dilanbhalla@microsoft.com>
Date: Fri, 22 Nov 2024 14:11:02 -0800
Subject: [PATCH 1/2] msft extractor queries

---
 cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql  | 14 +++++++++
 .../ql/src/Diagnostics/ExtractorErrorMsft.ql  | 16 ++++++++++
 go/ql/src/Diagnostics/ExtractorErrorMsft.ql   | 21 ++++++++++++++
 .../ql/src/Diagnostics/ExtractionErrorMsft.ql | 29 +++++++++++++++++++
 .../ql/src/Diagnostics/ExtractionErrorMsft.ql | 15 ++++++++++
 .../ql/src/Diagnostics/ExtractorErrorMsft.ql  | 15 ++++++++++
 .../queries/diagnostics/ExtractorErrorMsft.ql | 16 ++++++++++
 .../queries/diagnostics/ExtractorErrorMsft.ql | 17 +++++++++++
 .../ql/src/diagnostics/ExtractorErrorMsft.ql  | 12 ++++++++
 9 files changed, 155 insertions(+)
 create mode 100644 cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql
 create mode 100644 csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql
 create mode 100644 go/ql/src/Diagnostics/ExtractorErrorMsft.ql
 create mode 100644 java/ql/src/Diagnostics/ExtractionErrorMsft.ql
 create mode 100644 javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql
 create mode 100644 python/ql/src/Diagnostics/ExtractorErrorMsft.ql
 create mode 100644 ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
 create mode 100644 rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
 create mode 100644 swift/ql/src/diagnostics/ExtractorErrorMsft.ql

diff --git a/cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql b/cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql
new file mode 100644
index 000000000000..970de8cc55db
--- /dev/null
+++ b/cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql
@@ -0,0 +1,14 @@
+/**
+ * @name Extraction errors msft
+ * @description List all extraction errors for files in the source code directory.
+ * @kind diagnostic
+ * @id cpp/diagnostics/extraction-errors
+ */
+
+ import cpp
+ import ExtractionErrors
+ 
+ from ExtractionError error
+ select error.getFile(), error.getErrorMessage()
+
+ 
\ No newline at end of file
diff --git a/csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql b/csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql
new file mode 100644
index 000000000000..ba91a6081cc6
--- /dev/null
+++ b/csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql
@@ -0,0 +1,16 @@
+/**
+ * @name Extraction error msft
+ * @description An error message reported by the extractor, limited to those files where there are no
+ *              compilation errors. This indicates a bug or limitation in the extractor, and could lead
+ *              to inaccurate results.
+ * @kind diagnostic
+ * @id cs/extraction-error-msft
+ * @tags security
+ */
+
+ import csharp
+ import semmle.code.csharp.commons.Diagnostics
+ 
+ from ExtractorError error
+ select error.getLocation().getFile(), error.getText()
+ 
\ No newline at end of file
diff --git a/go/ql/src/Diagnostics/ExtractorErrorMsft.ql b/go/ql/src/Diagnostics/ExtractorErrorMsft.ql
new file mode 100644
index 000000000000..53a1091c76a7
--- /dev/null
+++ b/go/ql/src/Diagnostics/ExtractorErrorMsft.ql
@@ -0,0 +1,21 @@
+/**
+ * @id go/diagnostics/extraction-errors-msft
+ * @name Extraction errors msft
+ * @description List all extraction errors for files in the source code directory.
+ * @kind diagnostic
+ */
+
+import go
+import semmle.go.DiagnosticsReporting
+
+// Go does not have warnings, so all errors have error severity
+predicate reportableDiagnosticsMsft(Diagnostic d, File f, string msg) {
+    // Only report errors for files that would have been extracted
+    f = d.getFile() and
+    exists(f.getAChild()) and
+    msg = removeAbsolutePaths(d.getMessage())
+  }
+  
+from Diagnostic d, File f, string msg
+where reportableDiagnostics(d, f, msg)
+select f, msg
diff --git a/java/ql/src/Diagnostics/ExtractionErrorMsft.ql b/java/ql/src/Diagnostics/ExtractionErrorMsft.ql
new file mode 100644
index 000000000000..a606007f1a33
--- /dev/null
+++ b/java/ql/src/Diagnostics/ExtractionErrorMsft.ql
@@ -0,0 +1,29 @@
+/**
+ * @name Extraction errors msft
+ * @description A list of extraction errors for files in the source code directory.
+ * @kind diagnostic
+ * @id java/diagnostics/extraction-errors-msft
+ */
+
+import java
+import DiagnosticsReporting
+
+private predicate knownErrorsMsft(Diagnostic d, File f, string msg) {
+      d.getSeverity() = [6, 7, 8] and
+      f = d.getLocation().getFile()
+      msg = d.getMessage()
+  }
+  
+  private predicate unknownErrorsMsft(Diagnostic d, File f, string msg) {
+    not knownErrors(d, _, _) and
+    d.getSeverity() > 3 and
+    d.getLocation().getFile() = f and
+    exists(f.getRelativePath()) and
+    msg = "Unknown error"
+  }
+
+from Diagnostic d, File f, string msg
+where
+    knownErrorsMsft(Diagnostic d, File f, string msg) or
+    unknownErrorsMsft(Diagnostic d, File f, string msg)
+select f, msg
diff --git a/javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql b/javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql
new file mode 100644
index 000000000000..26fdf1947581
--- /dev/null
+++ b/javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql
@@ -0,0 +1,15 @@
+/**
+ * @name Extraction errors msft
+ * @description List all extraction errors for files in the source code directory.
+ * @kind diagnostic
+ * @id js/diagnostics/extraction-errors-msft
+ */
+
+ import javascript
+ 
+ from Error error
+ where
+   exists(error.getFile().getRelativePath()) and
+   error.isFatal()
+ select error.getFile(), error.getMessage()
+ 
\ No newline at end of file
diff --git a/python/ql/src/Diagnostics/ExtractorErrorMsft.ql b/python/ql/src/Diagnostics/ExtractorErrorMsft.ql
new file mode 100644
index 000000000000..adaca8370513
--- /dev/null
+++ b/python/ql/src/Diagnostics/ExtractorErrorMsft.ql
@@ -0,0 +1,15 @@
+/**
+ * @name Python extraction warnings msft
+ * @description List all extraction warnings for Python files in the source code directory.
+ * @kind diagnostic
+ * @id py/diagnostics/extraction-warnings-msft
+ */
+
+ import python
+
+ from SyntaxError error, File file
+ where
+   file = error.getFile() and
+   exists(file.getRelativePath())
+ select file, error.getMessage()
+ 
\ No newline at end of file
diff --git a/ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql b/ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
new file mode 100644
index 000000000000..b5ae6689e794
--- /dev/null
+++ b/ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
@@ -0,0 +1,16 @@
+/**
+ * @name Extraction errors msft
+ * @description List all extraction errors for files in the source code directory.
+ * @kind diagnostic
+ * @id rb/diagnostics/extraction-errors-msft
+ */
+
+ import codeql.ruby.AST
+ import codeql.ruby.Diagnostics
+ 
+ from ExtractionError error, File f
+ where
+   f = error.getLocation().getFile() and
+   exists(f.getRelativePath())
+ select f, error.getMessage()
+ 
\ No newline at end of file
diff --git a/rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql b/rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
new file mode 100644
index 000000000000..5442b22268e0
--- /dev/null
+++ b/rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
@@ -0,0 +1,17 @@
+/**
+ * @name Extraction errors msft
+ * @description List all extraction errors for files in the source code directory.
+ * @kind diagnostic
+ * @id rust/diagnostics/extraction-errors-msft
+ */
+
+ import codeql.rust.Diagnostics
+ import codeql.files.FileSystem
+ 
+ from ExtractionError error, File f
+ where
+   f = error.getLocation().getFile() and
+   exists(f.getRelativePath())
+ select f, error.getMessage()
+
+ 
\ No newline at end of file
diff --git a/swift/ql/src/diagnostics/ExtractorErrorMsft.ql b/swift/ql/src/diagnostics/ExtractorErrorMsft.ql
new file mode 100644
index 000000000000..9b819adc85c7
--- /dev/null
+++ b/swift/ql/src/diagnostics/ExtractorErrorMsft.ql
@@ -0,0 +1,12 @@
+/**
+ * @name Compiler errors msft
+ * @description List all compiler errors for files in the source code directory.
+ * @kind diagnostic
+ * @id swift/diagnostics/extraction-errors-msft
+ */
+
+ import swift
+
+ from CompilerError error
+ select error.getFile(), error.getText()
+ 
\ No newline at end of file

From eb56cb94b0ac00077555d9cdd0acbdae14f3cbfd Mon Sep 17 00:00:00 2001
From: Dilan Bhalla <dilanbhalla@microsoft.com>
Date: Fri, 22 Nov 2024 14:29:43 -0800
Subject: [PATCH 2/2] metadata fixes

---
 cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql          | 6 ++++--
 csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql       | 5 +++--
 go/ql/src/Diagnostics/ExtractorErrorMsft.ql           | 6 ++++--
 java/ql/src/Diagnostics/ExtractionErrorMsft.ql        | 6 ++++--
 javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql  | 6 ++++--
 python/ql/src/Diagnostics/ExtractorErrorMsft.ql       | 6 ++++--
 ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql | 6 ++++--
 rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql | 6 ++++--
 swift/ql/src/diagnostics/ExtractorErrorMsft.ql        | 6 ++++--
 9 files changed, 35 insertions(+), 18 deletions(-)

diff --git a/cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql b/cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql
index 970de8cc55db..072047ec2798 100644
--- a/cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql
+++ b/cpp/ql/src/Diagnostics/ExtractorErrorMsft.ql
@@ -1,8 +1,10 @@
 /**
  * @name Extraction errors msft
  * @description List all extraction errors for files in the source code directory.
- * @kind diagnostic
- * @id cpp/diagnostics/extraction-errors
+ * @id cpp/extractor-error-msft
+ * @kind problem
+ * @tags security
+ *       extraction
  */
 
  import cpp
diff --git a/csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql b/csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql
index ba91a6081cc6..948da7f885f3 100644
--- a/csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql
+++ b/csharp/ql/src/Diagnostics/ExtractorErrorMsft.ql
@@ -3,9 +3,10 @@
  * @description An error message reported by the extractor, limited to those files where there are no
  *              compilation errors. This indicates a bug or limitation in the extractor, and could lead
  *              to inaccurate results.
- * @kind diagnostic
- * @id cs/extraction-error-msft
+ * @id cs/extractor-error-msft
+ * @kind problem
  * @tags security
+ *       extraction
  */
 
  import csharp
diff --git a/go/ql/src/Diagnostics/ExtractorErrorMsft.ql b/go/ql/src/Diagnostics/ExtractorErrorMsft.ql
index 53a1091c76a7..244018669e9e 100644
--- a/go/ql/src/Diagnostics/ExtractorErrorMsft.ql
+++ b/go/ql/src/Diagnostics/ExtractorErrorMsft.ql
@@ -1,8 +1,10 @@
 /**
- * @id go/diagnostics/extraction-errors-msft
  * @name Extraction errors msft
  * @description List all extraction errors for files in the source code directory.
- * @kind diagnostic
+ * @id go/extractor-error-msft
+ * @kind problem
+ * @tags security
+ *       extraction
  */
 
 import go
diff --git a/java/ql/src/Diagnostics/ExtractionErrorMsft.ql b/java/ql/src/Diagnostics/ExtractionErrorMsft.ql
index a606007f1a33..b90c39210f1d 100644
--- a/java/ql/src/Diagnostics/ExtractionErrorMsft.ql
+++ b/java/ql/src/Diagnostics/ExtractionErrorMsft.ql
@@ -1,8 +1,10 @@
 /**
  * @name Extraction errors msft
  * @description A list of extraction errors for files in the source code directory.
- * @kind diagnostic
- * @id java/diagnostics/extraction-errors-msft
+ * @id java/extractor-error-msft
+ * @kind problem
+ * @tags security
+ *       extraction
  */
 
 import java
diff --git a/javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql b/javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql
index 26fdf1947581..ca7b9760ff4f 100644
--- a/javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql
+++ b/javascript/ql/src/Diagnostics/ExtractionErrorMsft.ql
@@ -1,8 +1,10 @@
 /**
  * @name Extraction errors msft
  * @description List all extraction errors for files in the source code directory.
- * @kind diagnostic
- * @id js/diagnostics/extraction-errors-msft
+ * @id js/extractor-error-msft
+ * @kind problem
+ * @tags security
+ *       extraction
  */
 
  import javascript
diff --git a/python/ql/src/Diagnostics/ExtractorErrorMsft.ql b/python/ql/src/Diagnostics/ExtractorErrorMsft.ql
index adaca8370513..49b4da524076 100644
--- a/python/ql/src/Diagnostics/ExtractorErrorMsft.ql
+++ b/python/ql/src/Diagnostics/ExtractorErrorMsft.ql
@@ -1,8 +1,10 @@
 /**
  * @name Python extraction warnings msft
  * @description List all extraction warnings for Python files in the source code directory.
- * @kind diagnostic
- * @id py/diagnostics/extraction-warnings-msft
+ * @id py/extractor-error-msft
+ * @kind problem
+ * @tags security
+ *       extraction
  */
 
  import python
diff --git a/ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql b/ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
index b5ae6689e794..edcdf010d10f 100644
--- a/ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
+++ b/ruby/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
@@ -1,8 +1,10 @@
 /**
  * @name Extraction errors msft
  * @description List all extraction errors for files in the source code directory.
- * @kind diagnostic
- * @id rb/diagnostics/extraction-errors-msft
+ * @id rb/extractor-error-msft
+ * @kind problem
+ * @tags security
+ *       extraction
  */
 
  import codeql.ruby.AST
diff --git a/rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql b/rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
index 5442b22268e0..3d17ffbbc806 100644
--- a/rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
+++ b/rust/ql/src/queries/diagnostics/ExtractorErrorMsft.ql
@@ -1,8 +1,10 @@
 /**
  * @name Extraction errors msft
  * @description List all extraction errors for files in the source code directory.
- * @kind diagnostic
- * @id rust/diagnostics/extraction-errors-msft
+ * @id rust/extractor-error-msft
+ * @kind problem
+ * @tags security
+ *       extraction
  */
 
  import codeql.rust.Diagnostics
diff --git a/swift/ql/src/diagnostics/ExtractorErrorMsft.ql b/swift/ql/src/diagnostics/ExtractorErrorMsft.ql
index 9b819adc85c7..a897d52744d7 100644
--- a/swift/ql/src/diagnostics/ExtractorErrorMsft.ql
+++ b/swift/ql/src/diagnostics/ExtractorErrorMsft.ql
@@ -1,8 +1,10 @@
 /**
  * @name Compiler errors msft
  * @description List all compiler errors for files in the source code directory.
- * @kind diagnostic
- * @id swift/diagnostics/extraction-errors-msft
+ * @id swift/extractor-error-msft
+ * @kind problem
+ * @tags security
+ *       extraction
  */
 
  import swift