diff --git a/src/commands/issue/explain.ts b/src/commands/issue/explain.ts index 498ba4ee..c364555a 100644 --- a/src/commands/issue/explain.ts +++ b/src/commands/issue/explain.ts @@ -10,15 +10,13 @@ import { getAutofixState, triggerRootCauseAnalysis, } from "../../lib/api-client.js"; -import { ApiError } from "../../lib/errors.js"; import { writeFooter, writeJson } from "../../lib/formatters/index.js"; -import { - formatRootCauseList, - handleSeerApiError, -} from "../../lib/formatters/seer.js"; +import { formatRootCauseList } from "../../lib/formatters/seer.js"; +import { trackSeerOutcome } from "../../lib/telemetry.js"; import { extractRootCauses } from "../../types/seer.js"; import { buildCommandHint, + handleSeerCommandError, type IssueIdFlags, issueIdFlags, issueIdPositional, @@ -120,12 +118,16 @@ export const explainCommand = buildCommand({ // 4. Extract root causes from steps const causes = extractRootCauses(state); if (causes.length === 0) { + trackSeerOutcome("explain", "no_solution"); throw new Error( "Analysis completed but no root causes found. " + "The issue may not have enough context for root cause analysis." ); } + // Track successful outcome + trackSeerOutcome("explain", "success"); + // 5. Output results if (flags.json) { writeJson(stdout, causes); @@ -140,11 +142,7 @@ export const explainCommand = buildCommand({ `To create a plan, run: sentry issue plan ${issueId}` ); } catch (error) { - // Handle API errors with friendly messages - if (error instanceof ApiError) { - throw handleSeerApiError(error.status, error.detail, resolvedOrg); - } - throw error; + throw handleSeerCommandError(error, "explain", resolvedOrg); } }, }); diff --git a/src/commands/issue/plan.ts b/src/commands/issue/plan.ts index 3a30b0a6..a4628f51 100644 --- a/src/commands/issue/plan.ts +++ b/src/commands/issue/plan.ts @@ -11,13 +11,11 @@ import { getAutofixState, triggerSolutionPlanning, } from "../../lib/api-client.js"; -import { ApiError, ValidationError } from "../../lib/errors.js"; +import { ValidationError } from "../../lib/errors.js"; import { muted } from "../../lib/formatters/colors.js"; import { writeJson } from "../../lib/formatters/index.js"; -import { - formatSolution, - handleSeerApiError, -} from "../../lib/formatters/seer.js"; +import { formatSolution } from "../../lib/formatters/seer.js"; +import { trackSeerOutcome } from "../../lib/telemetry.js"; import { type AutofixState, extractRootCauses, @@ -26,6 +24,7 @@ import { } from "../../types/seer.js"; import { buildCommandHint, + handleSeerCommandError, type IssueIdFlags, issueIdFlags, issueIdPositional, @@ -219,12 +218,14 @@ export const planCommand = buildCommand({ // Handle errors if (finalState.status === "ERROR") { + trackSeerOutcome("plan", "failed"); throw new Error( "Plan creation failed. Check the Sentry web UI for details." ); } if (finalState.status === "CANCELLED") { + trackSeerOutcome("plan", "failed"); throw new Error("Plan creation was cancelled."); } @@ -233,6 +234,8 @@ export const planCommand = buildCommand({ // Output results if (flags.json) { + // Track outcome based on whether solution was found + trackSeerOutcome("plan", solution ? "success" : "no_solution"); writeJson(stdout, { run_id: finalState.run_id, status: finalState.status, @@ -243,19 +246,17 @@ export const planCommand = buildCommand({ // Human-readable output if (solution) { + trackSeerOutcome("plan", "success"); const lines = formatSolution(solution); stdout.write(`${lines.join("\n")}\n`); } else { + trackSeerOutcome("plan", "no_solution"); stderr.write( "No solution found. Check the Sentry web UI for details.\n" ); } } catch (error) { - // Handle API errors with friendly messages - if (error instanceof ApiError) { - throw handleSeerApiError(error.status, error.detail, resolvedOrg); - } - throw error; + throw handleSeerCommandError(error, "plan", resolvedOrg); } }, }); diff --git a/src/commands/issue/utils.ts b/src/commands/issue/utils.ts index fc7e3262..b9204c2b 100644 --- a/src/commands/issue/utils.ts +++ b/src/commands/issue/utils.ts @@ -12,8 +12,16 @@ import { } from "../../lib/api-client.js"; import { getProjectByAlias } from "../../lib/db/project-aliases.js"; import { createDsnFingerprint, detectAllDsns } from "../../lib/dsn/index.js"; -import { ApiError, CliError, ContextError } from "../../lib/errors.js"; -import { getProgressMessage } from "../../lib/formatters/seer.js"; +import { + ApiError, + CliError, + ContextError, + SeerError, +} from "../../lib/errors.js"; +import { + getProgressMessage, + handleSeerApiError, +} from "../../lib/formatters/seer.js"; import { expandToFullShortId, isShortId, @@ -22,6 +30,7 @@ import { } from "../../lib/issue-id.js"; import { poll } from "../../lib/polling.js"; import { resolveOrg, resolveOrgAndProject } from "../../lib/resolve-target.js"; +import { type SeerOutcome, trackSeerOutcome } from "../../lib/telemetry.js"; import type { SentryIssue, Writer } from "../../types/index.js"; import { type AutofixState, isTerminalStatus } from "../../types/seer.js"; @@ -293,6 +302,46 @@ export async function resolveOrgAndIssueId( return { org: result.org, issueId: result.issue.id }; } +/** + * Handle and track errors from Seer commands. + * + * Converts ApiErrors to Seer-specific errors when applicable and + * tracks the appropriate telemetry outcome. For timeout errors, + * tracks the timeout outcome. + * + * @param error - The caught error + * @param command - The Seer command for telemetry + * @param orgSlug - Organization slug for error context + * @returns The error to throw (transformed if needed) + */ +export function handleSeerCommandError( + error: unknown, + command: "explain" | "plan", + orgSlug: string | undefined +): Error { + // Handle API errors with friendly messages + if (error instanceof ApiError) { + const seerError = handleSeerApiError(error.status, error.detail, orgSlug); + // Track Seer-specific errors vs generic failures + const outcome: SeerOutcome = + seerError instanceof SeerError ? seerError.reason : "failed"; + trackSeerOutcome(command, outcome); + return seerError; + } + + // Track timeout errors from polling + if (error instanceof Error && error.message.includes("timed out")) { + trackSeerOutcome(command, "timeout"); + return error; + } + + // Re-throw unknown errors without tracking + if (error instanceof Error) { + return error; + } + return new Error(String(error)); +} + type PollAutofixOptions = { /** Organization slug */ orgSlug: string; diff --git a/src/lib/telemetry.ts b/src/lib/telemetry.ts index 8b58e420..fb547f2e 100644 --- a/src/lib/telemetry.ts +++ b/src/lib/telemetry.ts @@ -289,3 +289,34 @@ export function withSerializeSpan(operation: string, fn: () => T): T { fn ); } + +/** Possible outcomes for Seer commands */ +export type SeerOutcome = + | "success" + | "timeout" + | "failed" + | "no_budget" + | "no_solution" + | "not_enabled" + | "ai_disabled"; + +/** + * Track Seer command outcome metrics. + * + * Increments a counter metric for Seer command outcomes to understand + * success/failure patterns across explain and plan commands. + * + * @param command - The Seer command (explain or plan) + * @param outcome - The outcome type (success, timeout, failed, etc.) + */ +export function trackSeerOutcome( + command: "explain" | "plan", + outcome: SeerOutcome +): void { + Sentry.metrics.count("seer.command.outcome", 1, { + attributes: { + command, + outcome, + }, + }); +}