Sanitizing ClickHouse errors from TSQL

Author: matt-aitken

internal-packages/clickhouse/src/client/tsql.ts

@@ -9,6 +9,7 @@ import type { ClickHouseSettings } from "@clickhouse/client";
 import { z } from "zod";
 import {
   compileTSQL,
+  sanitizeErrorMessage,
   transformResults,
   type TableSchema,
   type QuerySettings,
@@ -137,7 +138,9 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
     const [error, result] = await queryFn(params);
 
     if (error) {
-      return [error, null];
+      // Sanitize error message to show TSQL names instead of ClickHouse internals
+      const sanitizedMessage = sanitizeErrorMessage(error.message, options.tableSchema);
+      return [new QueryError(sanitizedMessage, { query: options.query }), null];
     }
 
     const { rows, stats } = result;
@@ -156,7 +159,7 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
   } catch (error) {
     const errorMessage = error instanceof Error ? error.message : "Unknown error";
 
-    // Log TSQL compilation or unexpected errors
+    // Log TSQL compilation or unexpected errors (with original message for debugging)
     logger.error("[TSQL] Query error", {
       name: options.name,
       error: errorMessage,
@@ -165,8 +168,11 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
       generatedParams: generatedParams ?? {},
     });
 
+    // Sanitize error message to show TSQL names instead of ClickHouse internals
+    const sanitizedMessage = sanitizeErrorMessage(errorMessage, options.tableSchema);
+
     if (error instanceof Error) {
-      return [new QueryError(error.message, { query: options.query }), null];
+      return [new QueryError(sanitizedMessage, { query: options.query }), null];
     }
     return [new QueryError("Unknown error executing TSQL query", { query: options.query }), null];
   }

internal-packages/tsql/src/index.ts

@@ -76,6 +76,8 @@ export {
   isValidUserValue,
   // Virtual column utilities
   isVirtualColumn,
+  // Error message sanitization
+  sanitizeErrorMessage,
   validateFilterColumn,
   validateGroupColumn,
   validateSelectColumn,

internal-packages/tsql/src/query/schema.test.ts

@@ -11,8 +11,10 @@ import {
   getExternalValue,
   getInternalValueFromMapping,
   getInternalValueFromMappingCaseInsensitive,
+  sanitizeErrorMessage,
   type ColumnSchema,
   type FieldMappings,
+  type TableSchema,
 } from "./schema.js";
 
 describe("Value mapping helper functions", () => {
@@ -363,7 +365,9 @@ describe("Field mapping helper functions (runtime dynamic mappings)", () => {
     });
 
     it("should return null if mapping name does not exist", () => {
-      expect(getInternalValueFromMapping(fieldMappings, "nonexistent", "my-project-ref")).toBeNull();
+      expect(
+        getInternalValueFromMapping(fieldMappings, "nonexistent", "my-project-ref")
+      ).toBeNull();
     });
 
     it("should return null for empty mappings", () => {
@@ -454,3 +458,161 @@ describe("Field mapping helper functions (runtime dynamic mappings)", () => {
   });
 });
 
+describe("Error message sanitization", () => {
+  // Test schema mimicking the real runs schema
+  const runsSchema: TableSchema = {
+    name: "runs",
+    clickhouseName: "trigger_dev.task_runs_v2",
+    description: "Task runs table",
+    tenantColumns: {
+      organizationId: "organization_id",
+      projectId: "project_id",
+      environmentId: "environment_id",
+    },
+    columns: {
+      run_id: {
+        name: "run_id",
+        clickhouseName: "friendly_id",
+        ...column("String"),
+      },
+      triggered_at: {
+        name: "triggered_at",
+        clickhouseName: "created_at",
+        ...column("DateTime64"),
+      },
+      machine: {
+        name: "machine",
+        clickhouseName: "machine_preset",
+        ...column("String"),
+      },
+      status: {
+        name: "status",
+        // No clickhouseName - same as name
+        ...column("String"),
+      },
+      task_identifier: {
+        name: "task_identifier",
+        // No clickhouseName - same as name
+        ...column("String"),
+      },
+    },
+  };
+
+  describe("sanitizeErrorMessage", () => {
+    it("should replace fully qualified table.column references", () => {
+      const error = "Missing column trigger_dev.task_runs_v2.friendly_id in query";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Missing column runs.run_id in query");
+    });
+
+    it("should replace standalone table names", () => {
+      const error = "Table trigger_dev.task_runs_v2 does not exist";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Table runs does not exist");
+    });
+
+    it("should replace standalone column names with different clickhouseName", () => {
+      const error = "Unknown identifier: friendly_id";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Unknown identifier: run_id");
+    });
+
+    it("should replace multiple occurrences in the same message", () => {
+      const error =
+        "Cannot compare friendly_id with created_at: incompatible types in trigger_dev.task_runs_v2";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Cannot compare run_id with triggered_at: incompatible types in runs");
+    });
+
+    it("should not replace column names that have no clickhouseName mapping", () => {
+      const error = "Invalid value for column status";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Invalid value for column status");
+    });
+
+    it("should handle error messages with quoted identifiers", () => {
+      const error = "Column 'machine_preset' is not of type String";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Column 'machine' is not of type String");
+    });
+
+    it("should handle error messages with backtick identifiers", () => {
+      const error = "Unknown column `friendly_id` in table";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Unknown column `run_id` in table");
+    });
+
+    it("should not replace partial matches within larger identifiers", () => {
+      const error = "Column my_friendly_id_column not found";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      // Should not replace "friendly_id" within "my_friendly_id_column"
+      expect(sanitized).toBe("Column my_friendly_id_column not found");
+    });
+
+    it("should return original message if no schemas provided", () => {
+      const error = "Some error with trigger_dev.task_runs_v2";
+      const sanitized = sanitizeErrorMessage(error, []);
+      expect(sanitized).toBe("Some error with trigger_dev.task_runs_v2");
+    });
+
+    it("should return original message if no matches found", () => {
+      const error = "Generic database error occurred";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Generic database error occurred");
+    });
+
+    it("should handle multiple tables", () => {
+      const eventsSchema: TableSchema = {
+        name: "events",
+        clickhouseName: "trigger_dev.task_events",
+        description: "Task events table",
+        tenantColumns: {
+          organizationId: "organization_id",
+          projectId: "project_id",
+          environmentId: "environment_id",
+        },
+        columns: {
+          event_id: {
+            name: "event_id",
+            clickhouseName: "internal_event_id",
+            ...column("String"),
+          },
+        },
+      };
+
+      const error =
+        "Cannot join trigger_dev.task_runs_v2 with trigger_dev.task_events on internal_event_id";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema, eventsSchema]);
+      expect(sanitized).toBe("Cannot join runs with events on event_id");
+    });
+
+    it("should handle real ClickHouse error format", () => {
+      const error =
+        "Unable to query clickhouse: Code: 47. DB::Exception: Missing columns: 'friendly_id' while processing query";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe(
+        "Unable to query clickhouse: Code: 47. DB::Exception: Missing columns: 'run_id' while processing query"
+      );
+    });
+
+    it("should handle error with column in parentheses", () => {
+      const error = "Function count(friendly_id) expects different arguments";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Function count(run_id) expects different arguments");
+    });
+
+    it("should handle error with column after comma", () => {
+      const error = "SELECT friendly_id, created_at FROM table";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("SELECT run_id, triggered_at FROM table");
+    });
+
+    it("should prioritize longer matches (table.column before standalone column)", () => {
+      // This tests that we replace "trigger_dev.task_runs_v2.friendly_id" as a unit,
+      // not "trigger_dev.task_runs_v2" and then "friendly_id" separately
+      const error = "Error in trigger_dev.task_runs_v2.friendly_id";
+      const sanitized = sanitizeErrorMessage(error, [runsSchema]);
+      expect(sanitized).toBe("Error in runs.run_id");
+    });
+  });
+});

internal-packages/tsql/src/query/schema.ts

@@ -682,3 +682,106 @@ export function getTableColumnNames(schema: SchemaRegistry, tableName: string):
 export function getAllTableNames(schema: SchemaRegistry): string[] {
   return Object.keys(schema.tables);
 }
+
+// ============================================================
+// Error Message Sanitization
+// ============================================================
+
+/**
+ * Sanitize a ClickHouse error message by replacing internal ClickHouse names
+ * with their user-facing TSQL equivalents.
+ *
+ * This function handles:
+ * - Fully qualified table names (e.g., `trigger_dev.task_runs_v2` → `runs`)
+ * - Column names with table prefix (e.g., `trigger_dev.task_runs_v2.friendly_id` → `runs.run_id`)
+ * - Standalone column names (e.g., `friendly_id` → `run_id`)
+ *
+ * @param message - The error message from ClickHouse
+ * @param schemas - The table schemas defining name mappings
+ * @returns The sanitized error message with TSQL names
+ *
+ * @example
+ * ```typescript
+ * const sanitized = sanitizeErrorMessage(
+ *   "Missing column trigger_dev.task_runs_v2.friendly_id",
+ *   [runsSchema]
+ * );
+ * // Returns: "Missing column runs.run_id"
+ * ```
+ */
+export function sanitizeErrorMessage(message: string, schemas: TableSchema[]): string {
+  // Build reverse lookup maps
+  const tableNameMap = new Map<string, string>(); // clickhouseName -> tsqlName
+  const columnNameMap = new Map<string, { tsqlName: string; tableTsqlName: string }>(); // clickhouseName -> { tsqlName, tableTsqlName }
+
+  for (const table of schemas) {
+    // Map table names
+    tableNameMap.set(table.clickhouseName, table.name);
+
+    // Map column names
+    for (const col of Object.values(table.columns)) {
+      const clickhouseColName = col.clickhouseName ?? col.name;
+      if (clickhouseColName !== col.name) {
+        // Only add to map if there's actually a different ClickHouse name
+        columnNameMap.set(clickhouseColName, {
+          tsqlName: col.name,
+          tableTsqlName: table.name,
+        });
+      }
+    }
+  }
+
+  let result = message;
+
+  // Replace fully qualified column references first (table.column)
+  // This handles patterns like: trigger_dev.task_runs_v2.friendly_id
+  for (const table of schemas) {
+    for (const col of Object.values(table.columns)) {
+      const clickhouseColName = col.clickhouseName ?? col.name;
+      const fullyQualified = `${table.clickhouseName}.${clickhouseColName}`;
+      const tsqlQualified = `${table.name}.${col.name}`;
+
+      if (fullyQualified !== tsqlQualified) {
+        // Use word boundary-aware replacement
+        result = replaceAllOccurrences(result, fullyQualified, tsqlQualified);
+      }
+    }
+  }
+
+  // Replace standalone table names (after column references to avoid partial matches)
+  // Sort by length descending to replace longer names first
+  const sortedTableNames = [...tableNameMap.entries()].sort((a, b) => b[0].length - a[0].length);
+  for (const [clickhouseName, tsqlName] of sortedTableNames) {
+    if (clickhouseName !== tsqlName) {
+      result = replaceAllOccurrences(result, clickhouseName, tsqlName);
+    }
+  }
+
+  // Replace standalone column names (for unqualified references)
+  // Sort by length descending to replace longer names first
+  const sortedColumnNames = [...columnNameMap.entries()].sort((a, b) => b[0].length - a[0].length);
+  for (const [clickhouseName, { tsqlName }] of sortedColumnNames) {
+    result = replaceAllOccurrences(result, clickhouseName, tsqlName);
+  }
+
+  return result;
+}
+
+/**
+ * Replace all occurrences of a string, respecting word boundaries where sensible.
+ * Uses word-boundary matching to avoid replacing substrings within larger identifiers.
+ */
+function replaceAllOccurrences(text: string, search: string, replacement: string): string {
+  // Escape special regex characters in the search string
+  const escaped = search.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+  // Use word boundary matching - identifiers are typically surrounded by
+  // non-identifier characters (spaces, quotes, parentheses, operators, etc.)
+  // We use a pattern that matches the identifier when it's not part of a larger identifier
+  const pattern = new RegExp(
+    `(?<![a-zA-Z0-9_])${escaped}(?![a-zA-Z0-9_])`,
+    "g"
+  );
+
+  return text.replace(pattern, replacement);
+}