Merge pull request #13071 from github/redsun82/swift-xcode-failure-diag

Swift: add autobuild failure diagnostics

Author: redsun82

swift/README.md

@@ -52,7 +52,7 @@ A log file is produced for each run under `CODEQL_EXTRACTOR_SWIFT_LOG_DIR` (the
 You can use the environment variable `CODEQL_EXTRACTOR_SWIFT_LOG_LEVELS` to configure levels for
 loggers and outputs. This must have the form of a comma separated `spec:min_level` list, where
 `spec` is either a glob pattern (made up of alphanumeric, `/`, `*` and `.` characters) for
-matching logger names or one of `out:bin`, `out:text` or `out:console`, and `min_level` is one
+matching logger names or one of `out:binary`, `out:text`, `out:console` or `out:diagnostics`, and `min_level` is one
 of `trace`, `debug`, `info`, `warning`, `error`, `critical` or `no_logs` to turn logs completely off.
 
 Current output default levels are no binary logs, `info` logs or higher in the text file and `warning` logs or higher on

swift/extractor/infra/SwiftDispatcher.h

@@ -5,6 +5,7 @@
 #include <swift/AST/SourceFile.h>
 #include <swift/Basic/SourceManager.h>
 #include <swift/Parse/Token.h>
+#include "absl/strings/str_cat.h"
 
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/SwiftTagTraits.h"
@@ -83,7 +84,7 @@ class SwiftDispatcher {
           valid = false;
         }
         LOG_ERROR("{} has undefined field {}{}, {}", entry.NAME, field,
-                  index >= 0 ? ('[' + std::to_string(index) + ']') : "", action);
+                  index >= 0 ? absl::StrCat("[", index, "]") : "", action);
       }
     });
     if (valid) {

swift/extractor/infra/SwiftMangledName.cpp

@@ -1,4 +1,5 @@
 #include "swift/extractor/infra/SwiftMangledName.h"
+#include "absl/strings/str_cat.h"
 
 namespace codeql {
 
@@ -8,13 +9,11 @@ void appendPart(std::string& out, const std::string& prefix) {
 }
 
 void appendPart(std::string& out, UntypedTrapLabel label) {
-  out += '{';
-  out += label.str();
-  out += '}';
+  absl::StrAppend(&out, "{", label.str(), "}");
 }
 
 void appendPart(std::string& out, unsigned index) {
-  out += std::to_string(index);
+  absl::StrAppend(&out, index);
 }
 
 }  // namespace

swift/extractor/main.cpp

@@ -10,6 +10,7 @@
 #include <swift/Basic/InitializeSwiftModules.h>
 
 #include "absl/strings/str_join.h"
+#include "absl/strings/str_cat.h"
 
 #include "swift/extractor/SwiftExtractor.h"
 #include "swift/extractor/infra/TargetDomains.h"
@@ -166,7 +167,7 @@ static void checkWhetherToRunUnderTool(int argc, char* const* argv) {
 // compilations, diagnostics, etc.
 codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
   auto timestamp = std::chrono::system_clock::now().time_since_epoch().count();
-  auto filename = std::to_string(timestamp) + '-' + std::to_string(getpid());
+  auto filename = absl::StrCat(timestamp, "-", getpid());
   auto target = std::filesystem::path("invocations") / std::filesystem::path(filename);
   auto maybeDomain = codeql::createTargetTrapDomain(state, target, codeql::TrapType::invocation);
   CODEQL_ASSERT(maybeDomain, "Cannot create invocation trap file for {}", target);

swift/extractor/trap/TrapDomain.h

@@ -2,6 +2,7 @@
 
 #include <memory>
 #include <sstream>
+#include "absl/strings/str_cat.h"
 
 #include "swift/extractor/trap/TrapLabel.h"
 #include "swift/extractor/infra/file/TargetFile.h"
@@ -27,7 +28,7 @@ class TrapDomain {
       e.forEachLabel([&e, this](const char* field, int index, auto& label) {
         if (!label.valid()) {
           LOG_ERROR("{} has undefined field {}{}", e.NAME, field,
-                    index >= 0 ? ('[' + std::to_string(index) + ']') : "");
+                    index >= 0 ? absl::StrCat("[", index, "]") : "");
         }
       });
     }

swift/integration-tests/create_database_utils.py

@@ -1,15 +1,34 @@
 """
-recreation of internal `create_database_utils.py` to run the tests locally, with minimal
-and swift-specialized functionality
+Simplified version of internal `create_database_utils.py` used to run the tests locally, with
+minimal and swift-specialized functionality
+TODO unify integration testing code across the public and private repositories
 """
 import subprocess
 import pathlib
 import sys
+import shutil
 
 
-def run_codeql_database_create(cmds, lang, keep_trap=True):
+def runSuccessfully(cmd):
+    res = subprocess.run(cmd)
+    if res.returncode:
+        print("FAILED", file=sys.stderr)
+        print(" ", *cmd, f"(exit code {res.returncode})", file=sys.stderr)
+        sys.exit(res.returncode)
+
+def runUnsuccessfully(cmd):
+    res = subprocess.run(cmd)
+    if res.returncode == 0:
+        print("FAILED", file=sys.stderr)
+        print(" ", *cmd, f"(exit code 0, expected to fail)", file=sys.stderr)
+        sys.exit(1)
+
+
+def run_codeql_database_create(cmds, lang, keep_trap=True, db=None, runFunction=runSuccessfully):
+    """ db parameter is here solely for compatibility with the internal test runner """
     assert lang == 'swift'
     codeql_root = pathlib.Path(__file__).parents[2]
+    shutil.rmtree("db", ignore_errors=True)
     cmd = [
         "codeql", "database", "create",
         "-s", ".", "-l", "swift", "--internal-use-lua-tracing", f"--search-path={codeql_root}", "--no-cleanup",
@@ -19,8 +38,4 @@ def run_codeql_database_create(cmds, lang, keep_trap=True):
     for c in cmds:
         cmd += ["-c", c]
     cmd.append("db")
-    res = subprocess.run(cmd)
-    if res.returncode:
-        print("FAILED", file=sys.stderr)
-        print(" ", *cmd, file=sys.stderr)
-        sys.exit(res.returncode)
+    runFunction(cmd)

swift/integration-tests/diagnostics_test_utils.py

@@ -0,0 +1,64 @@
+"""
+Simplified POSIX only version of internal `diagnostics_test_utils.py` used to run the tests locally
+TODO unify integration testing code across the public and private repositories
+"""
+
+import json
+import pathlib
+import subprocess
+import os
+import difflib
+import sys
+
+
+def _normalize_actual(test_dir, database_dir):
+    proc = subprocess.run(['codeql', 'database', 'export-diagnostics', '--format', 'raw', '--', database_dir],
+                          stdout=subprocess.PIPE, universal_newlines=True, check=True, text=True)
+    data = proc.stdout.replace(str(test_dir.absolute()), "<test-root-directory>")
+    data = json.loads(data)
+    data[:] = [e for e in data if not e["source"]["id"].startswith("cli/")]
+    for e in data:
+        e.pop("timestamp")
+    return _normalize_json(data)
+
+
+def _normalize_expected(test_dir):
+    with open(test_dir / "diagnostics.expected") as expected:
+        text = expected.read()
+    return _normalize_json(_load_concatenated_json(text))
+
+
+def _load_concatenated_json(text):
+    text = text.lstrip()
+    entries = []
+    decoder = json.JSONDecoder()
+    while text:
+        obj, index = decoder.raw_decode(text)
+        entries.append(obj)
+        text = text[index:].lstrip()
+    return entries
+
+
+def _normalize_json(data):
+    entries = [json.dumps(e, sort_keys=True, indent=2) for e in data]
+    entries.sort()
+    entries.append("")
+    return "\n".join(entries)
+
+
+def check_diagnostics(test_dir=".", test_db="db"):
+    test_dir = pathlib.Path(test_dir)
+    test_db = pathlib.Path(test_db)
+    actual = _normalize_actual(test_dir, test_db)
+    if os.environ.get("CODEQL_INTEGRATION_TEST_LEARN") == "true":
+        with open(test_dir / "diagnostics.expected", "w") as expected:
+            expected.write(actual)
+        return
+    expected = _normalize_expected(test_dir)
+    if actual != expected:
+        with open(test_dir / "diagnostics.actual", "w") as actual_out:
+            actual_out.write(actual)
+        actual = actual.splitlines(keepends=True)
+        expected = expected.splitlines(keepends=True)
+        print("".join(difflib.unified_diff(actual, expected, fromfile="diagnostics.actual", tofile="diagnostics.expected")), file=sys.stderr)
+        sys.exit(1)

swift/integration-tests/osx-only/autobuilder/failure/.gitignore

@@ -0,0 +1 @@
+/build

swift/integration-tests/osx-only/autobuilder/failure/diagnostics.expected

@@ -0,0 +1,18 @@
+{
+  "helpLinks": [
+    "https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/customizing-code-scanning",
+    "https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language"
+  ],
+  "plaintextMessage": "The detected build command failed (tried /usr/bin/xcodebuild build -project <test-root-directory>/hello-failure.xcodeproj -target hello-failure CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO).\n\nSet up a manual build command.",
+  "severity": "error",
+  "source": {
+    "extractorName": "swift",
+    "id": "swift/autobuilder/build-command-failed",
+    "name": "Detected build command failed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}