diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index b071e80a..f8ad856d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -223,7 +223,7 @@ jobs: # Tear down the preview deployment echo "Tearing down smoke test deployment..." envsubst < cluster/preview/deployment.yaml | kubectl delete -f - --ignore-not-found - kubectl delete pvc -l app=mod-bot-pr-${PR_NUMBER} --ignore-not-found + kubectl delete pvc -l preview=pr-${PR_NUMBER} --ignore-not-found echo "Smoke test complete" diff --git a/Dockerfile b/Dockerfile index 64823303..fb69ed4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,10 @@ WORKDIR /app ENV NODE_ENV=production -COPY --from=build /app/node_modules /app/node_modules -ADD package.json package-lock.json ./ -RUN npm prune --production +COPY package.json package-lock.json ./ +RUN apk add --no-cache --virtual .build-deps python3 make g++ && \ + npm ci --omit=dev && \ + apk del .build-deps COPY --from=build /app/build ./build ADD index.prod.js ./ diff --git a/app/commands/report/constructLog.ts b/app/commands/report/constructLog.ts index fd309286..3ddb91c8 100644 --- a/app/commands/report/constructLog.ts +++ b/app/commands/report/constructLog.ts @@ -69,7 +69,7 @@ ${preface} -# ${extra}${formatDistanceToNowStrict(lastReport.message.createdAt)} ago · `).trim(), allowedMentions: { roles: [moderator] }, } satisfies MessageCreateOptions; - }); + }).pipe(Effect.withSpan("constructLog")); export const isForwardedMessage = (message: Message): boolean => { return message.reference?.type === MessageReferenceType.Forward; diff --git a/app/commands/report/userLog.ts b/app/commands/report/userLog.ts index d26199e1..622449a0 100644 --- a/app/commands/report/userLog.ts +++ b/app/commands/report/userLog.ts @@ -8,7 +8,12 @@ import { Effect } from "effect"; import { runEffect } from "#~/AppRuntime"; import { type DatabaseService, type SqlError } from "#~/Database"; -import { forwardMessageSafe, sendMessage } from "#~/effects/discordSdk.ts"; +import { + fetchMessage, + forwardMessageSafe, + messageReply, + sendMessage, +} from "#~/effects/discordSdk.ts"; import { DiscordApiError, type NotFoundError } from "#~/effects/errors"; import { logEffect } from "#~/effects/observability"; import { @@ -106,26 +111,20 @@ export function logUserMessage({ if (alreadyReported && reason !== ReportReasons.modResolution) { // Message already reported with this reason, just add to thread - const latestReport = yield* Effect.tryPromise({ - try: async () => { - try { - const reportContents = `${staff ? ` ${staff.username} ` : ""}${ReadableReasons[reason]}`; - const priorLogMessage = await thread.messages.fetch( - alreadyReported.log_message_id, - ); - return priorLogMessage - .reply(reportContents) - .catch(() => priorLogMessage.channel.send(reportContents)); - } catch (_) { - return thread.send(logBody); - } - }, - catch: (error) => - new DiscordApiError({ - operation: "logUserMessage existing", - cause: error, - }), - }); + const reportContents = `${staff ? ` ${staff.username} ` : ""}${ReadableReasons[reason]}`; + const latestReport = (yield* fetchMessage( + thread, + alreadyReported.log_message_id, + ).pipe( + Effect.flatMap((priorLogMessage) => + messageReply(priorLogMessage, reportContents).pipe( + Effect.catchAll(() => + sendMessage(priorLogMessage.channel, reportContents), + ), + ), + ), + Effect.catchAll(() => sendMessage(thread, logBody)), + )) as Message; yield* logEffect( "info", diff --git a/app/discord/auditLog.ts b/app/discord/auditLog.ts index 5ad39ebf..f07c5ace 100644 --- a/app/discord/auditLog.ts +++ b/app/discord/auditLog.ts @@ -31,6 +31,10 @@ export const fetchAuditLogEntry = ( const auditLogs = yield* Effect.promise(() => guild.fetchAuditLogs({ type: auditLogType, limit: 5 }), + ).pipe( + Effect.withSpan("discord.fetchAuditLogs", { + attributes: { attempt: attempt + 1, guildId: guild.id }, + }), ); const entry = findEntry(auditLogs.entries); @@ -38,9 +42,17 @@ export const fetchAuditLogEntry = ( yield* logEffect("debug", "AuditLog", "Record found", { attempt: attempt + 1, }); + yield* Effect.annotateCurrentSpan({ + "auditLog.found": true, + "auditLog.attempts": attempt + 1, + }); return entry; } } + yield* Effect.annotateCurrentSpan({ + "auditLog.found": false, + "auditLog.attempts": 3, + }); return undefined; }).pipe( Effect.withSpan("fetchAuditLogEntry", { diff --git a/app/discord/gateway.ts b/app/discord/gateway.ts index 837085fd..87daedce 100644 --- a/app/discord/gateway.ts +++ b/app/discord/gateway.ts @@ -76,25 +76,40 @@ export const initDiscordBot: Effect.Effect = Effect.gen(function* () { customId: interaction.customId, }); let config: AnyCommand | undefined = undefined; + let commandName: string | undefined = undefined; switch (interaction.type) { case InteractionType.ApplicationCommand: { - config = matchCommand(interaction.commandName); + commandName = interaction.commandName; + config = matchCommand(commandName); break; } case InteractionType.MessageComponent: case InteractionType.ModalSubmit: { - config = matchCommand(interaction.customId); + commandName = interaction.customId; + config = matchCommand(commandName); break; } } - if (!config) { + if (!config || !commandName) { log("debug", "deployCommands", "no matching command found"); return; } log("debug", "deployCommands", "found matching command", { config }); - void runEffect(config.handler(interaction as never)); + void runEffect( + config.handler(interaction as never).pipe( + Effect.withSpan(`command.${commandName}`, { + attributes: { + "command.name": commandName, + "command.type": interaction.type, + "interaction.id": interaction.id, + guildId: interaction.guildId, + userId: interaction.user.id, + }, + }), + ), + ); }); const errorHandler = (error: unknown) => { diff --git a/app/effects/devSpanExporter.ts b/app/effects/devSpanExporter.ts new file mode 100644 index 00000000..e2bcb6bb --- /dev/null +++ b/app/effects/devSpanExporter.ts @@ -0,0 +1,136 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { ExportResultCode, type ExportResult } from "@opentelemetry/core"; +import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; + +const ORPHAN_TTL_MS = 30_000; + +function hrTimeToMs(hrTime: [number, number]): number { + return hrTime[0] * 1000 + hrTime[1] / 1_000_000; +} + +/** + * Dev-mode span exporter that prints a human-readable timing tree to console. + * + * Accumulates spans by trace ID and prints the full tree when the root span + * (no parent) completes. Use with SimpleSpanProcessor for immediate output. + * + * Example output: + * + * --- Trace: DeletionLogger.messageDelete (1823.4ms) --- + * discord.fetchGuild 45.2ms + * fetchAuditLogEntry 1612.8ms + * discord.fetchAuditLogs 89.3ms + * discord.fetchAuditLogs 76.1ms + * getOrCreateDeletionLogThread 89.4ms + * sql.execute 2.1ms + * discord.sendMessage 48.2ms + */ +export class DevTreeSpanExporter implements SpanExporter { + private spansByTrace = new Map< + string, + { spans: ReadableSpan[]; firstSeen: number } + >(); + + export( + spans: ReadableSpan[], + resultCallback: (result: ExportResult) => void, + ): void { + for (const span of spans) { + const traceId = span.spanContext().traceId; + + let entry = this.spansByTrace.get(traceId); + if (!entry) { + entry = { spans: [], firstSeen: Date.now() }; + this.spansByTrace.set(traceId, entry); + } + entry.spans.push(span); + + // Root span (no parent) — print the tree + const parentId = span.parentSpanContext?.spanId; + if (!parentId) { + this.printTree(traceId); + this.spansByTrace.delete(traceId); + } + } + + // Evict orphaned traces older than TTL + const now = Date.now(); + for (const [traceId, entry] of this.spansByTrace) { + if (now - entry.firstSeen > ORPHAN_TTL_MS) { + this.spansByTrace.delete(traceId); + } + } + + resultCallback({ code: ExportResultCode.SUCCESS }); + } + + private printTree(traceId: string): void { + const entry = this.spansByTrace.get(traceId); + if (!entry) return; + const { spans } = entry; + + // Build parent→children map + const childMap = new Map(); + let root: ReadableSpan | undefined; + + for (const span of spans) { + const parentId = span.parentSpanContext?.spanId; + if (!parentId) { + root = span; + } else { + let children = childMap.get(parentId); + if (!children) { + children = []; + childMap.set(parentId, children); + } + children.push(span); + } + } + + if (!root) return; + + const rootDuration = hrTimeToMs(root.duration); + const lines: string[] = []; + this.renderChildren(root, childMap, 1, lines); + + if (lines.length === 0) { + // Single span, no children — print on one line + console.log(`[trace] ${root.name} ${rootDuration.toFixed(1)}ms`); + } else { + console.log(`--- Trace: ${root.name} (${rootDuration.toFixed(1)}ms) ---`); + for (const line of lines) console.log(line); + } + } + + private renderChildren( + parent: ReadableSpan, + childMap: Map, + depth: number, + lines: string[], + ): void { + const children = childMap.get(parent.spanContext().spanId) ?? []; + + // Sort by start time + children.sort((a, b) => { + const diff = a.startTime[0] - b.startTime[0]; + return diff !== 0 ? diff : a.startTime[1] - b.startTime[1]; + }); + + for (const child of children) { + const durationMs = hrTimeToMs(child.duration); + const status = child.status.code === SpanStatusCode.ERROR ? " ERROR" : ""; + const indent = " ".repeat(depth); + lines.push(`${indent}${child.name} ${durationMs.toFixed(1)}ms${status}`); + + this.renderChildren(child, childMap, depth + 1, lines); + } + } + + async shutdown(): Promise { + this.spansByTrace.clear(); + } + + async forceFlush(): Promise { + // No-op: SimpleSpanProcessor exports immediately, nothing to flush + } +} diff --git a/app/effects/discordSdk.ts b/app/effects/discordSdk.ts index 24037512..5e39be64 100644 --- a/app/effects/discordSdk.ts +++ b/app/effects/discordSdk.ts @@ -3,6 +3,9 @@ * * These helpers provide consistent error handling and reduce boilerplate * when calling Discord.js APIs from Effect-based code. + * + * All wrappers include `Effect.withSpan("discord.")` for + * performance tracing. Span names use a `discord.` prefix consistently. */ import type { ChatInputCommandInteraction, @@ -29,14 +32,16 @@ export const fetchGuild = (client: Client, guildId: string) => try: () => client.guilds.fetch(guildId), catch: (error) => new DiscordApiError({ operation: "fetchGuild", cause: error }), - }); + }).pipe(Effect.withSpan("discord.fetchGuild", { attributes: { guildId } })); export const fetchChannel = (guild: Guild, channelId: string) => Effect.tryPromise({ try: () => guild.channels.fetch(channelId), catch: (error) => new DiscordApiError({ operation: "fetchChannel", cause: error }), - }); + }).pipe( + Effect.withSpan("discord.fetchChannel", { attributes: { channelId } }), + ); export const fetchChannelFromClient = ( client: Client, @@ -46,14 +51,18 @@ export const fetchChannelFromClient = ( try: () => client.channels.fetch(channelId) as Promise, catch: (error) => new DiscordApiError({ operation: "fetchChannel", cause: error }), - }); + }).pipe( + Effect.withSpan("discord.fetchChannel", { + attributes: { channelId, variant: "fromClient" }, + }), + ); export const fetchMember = (guild: Guild, userId: string) => Effect.tryPromise({ try: () => guild.members.fetch(userId), catch: (error) => new DiscordApiError({ operation: "fetchMember", cause: error }), - }); + }).pipe(Effect.withSpan("discord.fetchMember", { attributes: { userId } })); export const fetchMemberOrNull = ( guild: Guild, @@ -62,14 +71,22 @@ export const fetchMemberOrNull = ( Effect.tryPromise({ try: () => guild.members.fetch(userId), catch: () => null, - }).pipe(Effect.catchAll(() => Effect.succeed(null))); + }).pipe( + Effect.catchAll(() => Effect.succeed(null)), + Effect.tap((result) => + Effect.annotateCurrentSpan({ found: result !== null }), + ), + Effect.withSpan("discord.fetchMember", { + attributes: { userId, variant: "orNull" }, + }), + ); export const fetchUser = (client: Client, userId: string) => Effect.tryPromise({ try: () => client.users.fetch(userId), catch: (error) => new DiscordApiError({ operation: "fetchUser", cause: error }), - }); + }).pipe(Effect.withSpan("discord.fetchUser", { attributes: { userId } })); export const fetchUserOrNull = ( client: Client, @@ -78,7 +95,15 @@ export const fetchUserOrNull = ( Effect.tryPromise({ try: () => client.users.fetch(userId), catch: () => null, - }).pipe(Effect.catchAll(() => Effect.succeed(null))); + }).pipe( + Effect.catchAll(() => Effect.succeed(null)), + Effect.tap((result) => + Effect.annotateCurrentSpan({ found: result !== null }), + ), + Effect.withSpan("discord.fetchUser", { + attributes: { userId, variant: "orNull" }, + }), + ); export const fetchMessage = ( channel: GuildTextBasedChannel | ThreadChannel, @@ -88,14 +113,22 @@ export const fetchMessage = ( try: () => channel.messages.fetch(messageId), catch: (error) => new DiscordApiError({ operation: "fetchMessage", cause: error }), - }); + }).pipe( + Effect.withSpan("discord.fetchMessage", { + attributes: { messageId, channelId: channel.id }, + }), + ); export const deleteMessage = (message: Message | PartialMessage) => Effect.tryPromise({ try: () => message.delete(), catch: (error) => new DiscordApiError({ operation: "deleteMessage", cause: error }), - }); + }).pipe( + Effect.withSpan("discord.deleteMessage", { + attributes: { messageId: message.id }, + }), + ); export const sendMessage = ( channel: GuildTextBasedChannel | ThreadChannel, @@ -105,7 +138,11 @@ export const sendMessage = ( try: () => channel.send(options), catch: (error) => new DiscordApiError({ operation: "sendMessage", cause: error }), - }); + }).pipe( + Effect.withSpan("discord.sendMessage", { + attributes: { channelId: channel.id }, + }), + ); export const editMessage = ( message: Message, @@ -115,7 +152,11 @@ export const editMessage = ( try: () => message.edit(options), catch: (error) => new DiscordApiError({ operation: "editMessage", cause: error }), - }); + }).pipe( + Effect.withSpan("discord.editMessage", { + attributes: { messageId: message.id }, + }), + ); export const forwardMessageSafe = (message: Message, targetChannelId: string) => Effect.tryPromise({ @@ -129,6 +170,9 @@ export const forwardMessageSafe = (message: Message, targetChannelId: string) => targetChannelId, }), ), + Effect.withSpan("discord.forwardMessage", { + attributes: { messageId: message.id, targetChannelId, variant: "safe" }, + }), ); export const messageReply = ( @@ -139,7 +183,11 @@ export const messageReply = ( try: () => message.reply(options), catch: (error) => new DiscordApiError({ operation: "messageReply", cause: error }), - }).pipe(Effect.withSpan("messageReply")); + }).pipe( + Effect.withSpan("discord.messageReply", { + attributes: { messageId: message.id }, + }), + ); export const replyAndForwardSafe = ( message: Message, @@ -161,6 +209,13 @@ export const replyAndForwardSafe = ( forwardToChannelId, }), ), + Effect.withSpan("discord.replyAndForward", { + attributes: { + messageId: message.id, + forwardToChannelId, + variant: "safe", + }, + }), ); /** @@ -171,7 +226,7 @@ export const replyAndForwardSafe = ( export const resolveMessagePartial = ( msg: Message | PartialMessage, ): Effect.Effect => - msg.partial + (msg.partial ? Effect.tryPromise({ try: () => msg.fetch(), catch: (error) => @@ -180,7 +235,12 @@ export const resolveMessagePartial = ( cause: error, }), }) - : Effect.succeed(msg); + : Effect.succeed(msg) + ).pipe( + Effect.withSpan("discord.resolveMessagePartial", { + attributes: { wasPartial: msg.partial }, + }), + ); export const interactionReply = ( interaction: @@ -195,7 +255,8 @@ export const interactionReply = ( try: () => interaction.reply(options), catch: (error) => new DiscordApiError({ operation: "interactionReply", cause: error }), - }); + }).pipe(Effect.withSpan("discord.interactionReply")); + export const interactionDeferReply = ( interaction: | MessageComponentInteraction @@ -208,7 +269,8 @@ export const interactionDeferReply = ( try: () => interaction.deferReply(options), catch: (error) => new DiscordApiError({ operation: "interactionDeferReply", cause: error }), - }); + }).pipe(Effect.withSpan("discord.interactionDeferReply")); + export const interactionEditReply = ( interaction: | MessageComponentInteraction @@ -221,7 +283,8 @@ export const interactionEditReply = ( try: () => interaction.editReply(options), catch: (error) => new DiscordApiError({ operation: "interactionEditReply", cause: error }), - }); + }).pipe(Effect.withSpan("discord.interactionEditReply")); + export const interactionFollowUp = ( interaction: | MessageComponentInteraction @@ -234,7 +297,7 @@ export const interactionFollowUp = ( try: () => interaction.followUp(options), catch: (error) => new DiscordApiError({ operation: "interactionFollowUp", cause: error }), - }); + }).pipe(Effect.withSpan("discord.interactionFollowUp")); export const interactionUpdate = ( interaction: MessageComponentInteraction, @@ -244,4 +307,4 @@ export const interactionUpdate = ( try: () => interaction.update(options), catch: (error) => new DiscordApiError({ operation: "interactionUpdate", cause: error }), - }); + }).pipe(Effect.withSpan("discord.interactionUpdate")); diff --git a/app/effects/tracing.ts b/app/effects/tracing.ts index 4410b1aa..57fc019c 100644 --- a/app/effects/tracing.ts +++ b/app/effects/tracing.ts @@ -1,14 +1,12 @@ import { NodeSdk } from "@effect/opentelemetry"; -import { - BatchSpanProcessor, - ConsoleSpanExporter, -} from "@opentelemetry/sdk-trace-base"; +import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; import { SentryPropagator, SentrySampler, SentrySpanProcessor, } from "@sentry/opentelemetry"; +import { DevTreeSpanExporter } from "#~/effects/devSpanExporter.js"; import Sentry, { isValidDsn } from "#~/helpers/sentry.server.js"; const sentryClient = Sentry.getClient(); @@ -24,6 +22,9 @@ const sentryClient = Sentry.getClient(); * - SentrySpanProcessor: Exports spans to Sentry (it IS a SpanProcessor, not an exporter) * - SentrySampler: Respects Sentry's tracesSampleRate * - SentryPropagator: Enables distributed tracing + * + * In dev mode (no Sentry DSN), spans are printed as a human-readable timing + * tree via DevTreeSpanExporter. */ export const TracingLive = NodeSdk.layer(() => ({ resource: { serviceName: "mod-bot" }, @@ -31,7 +32,7 @@ export const TracingLive = NodeSdk.layer(() => ({ // SentrySpanProcessor is already a SpanProcessor, don't wrap in BatchSpanProcessor spanProcessor: isValidDsn ? new SentrySpanProcessor() - : new BatchSpanProcessor(new ConsoleSpanExporter()), + : new SimpleSpanProcessor(new DevTreeSpanExporter()), sampler: isValidDsn && sentryClient ? new SentrySampler(sentryClient) : undefined, propagator: isValidDsn ? new SentryPropagator() : undefined, diff --git a/app/helpers/sentry.server.ts b/app/helpers/sentry.server.ts index ff0536c6..c3132a96 100644 --- a/app/helpers/sentry.server.ts +++ b/app/helpers/sentry.server.ts @@ -12,9 +12,16 @@ if (isValidDsn) { // Skip Sentry's auto OpenTelemetry setup - we'll use Effect's OpenTelemetry // and provide the SentrySpanProcessor to it skipOpenTelemetrySetup: true, - // Set tracesSampleRate to 1.0 to capture 100% - // of transactions for performance monitoring. - tracesSampleRate: isProd() ? 0.2 : 1, + // Configurable via SENTRY_TRACES_SAMPLE_RATE env var for diagnosis periods. + // Defaults: 0.2 in prod, 1.0 in dev. + tracesSampleRate: process.env.SENTRY_TRACES_SAMPLE_RATE + ? Math.min( + 1, + Math.max(0, parseFloat(process.env.SENTRY_TRACES_SAMPLE_RATE)), + ) + : isProd() + ? 0.2 + : 1, sendDefaultPii: true, }; diff --git a/app/models/guilds.server.ts b/app/models/guilds.server.ts index 1bfc646f..263fa72c 100644 --- a/app/models/guilds.server.ts +++ b/app/models/guilds.server.ts @@ -128,4 +128,8 @@ export const fetchSettingsEffect = ( ); } return Object.fromEntries(result) as Pick; - }); + }).pipe( + Effect.withSpan("fetchSettingsEffect", { + attributes: { guildId, keys: keys.join(",") }, + }), + ); diff --git a/app/server.ts b/app/server.ts index acdeadc0..f6d98e0d 100644 --- a/app/server.ts +++ b/app/server.ts @@ -126,7 +126,7 @@ const startup = Effect.gen(function* () { yield* initializeGroups(discordClient.guilds.cache); yield* logEffect("debug", "Server", "scheduling integrity check"); - yield* runtime.runFork(runIntegrityCheck); + runtime.runFork(runIntegrityCheck); // Graceful shutdown handler to checkpoint WAL and dispose the runtime // (tears down PostHog finalizer, feature flag interval, and SQLite connection) diff --git a/cluster/pdb.yaml b/cluster/pdb.yaml index 4fa4743e..0fc7e06f 100644 --- a/cluster/pdb.yaml +++ b/cluster/pdb.yaml @@ -3,7 +3,7 @@ kind: PodDisruptionBudget metadata: name: mod-bot-pdb spec: - maxUnavailable: 0 + maxUnavailable: 1 selector: matchLabels: app: mod-bot diff --git a/cluster/preview/certificate.yaml b/cluster/preview/certificate.yaml new file mode 100644 index 00000000..21ad78fb --- /dev/null +++ b/cluster/preview/certificate.yaml @@ -0,0 +1,12 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: wildcard-staging-cert + namespace: staging +spec: + secretName: wildcard-staging-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - "*.euno-staging.reactiflux.com" diff --git a/cluster/preview/deployment.yaml b/cluster/preview/deployment.yaml index 272ffd97..0e2abd73 100644 --- a/cluster/preview/deployment.yaml +++ b/cluster/preview/deployment.yaml @@ -94,7 +94,6 @@ metadata: annotations: nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - cert-manager.io/cluster-issuer: letsencrypt-prod spec: ingressClassName: nginx rules: @@ -111,4 +110,4 @@ spec: tls: - hosts: - ${PR_NUMBER}.euno-staging.reactiflux.com - secretName: mod-bot-pr-${PR_NUMBER}-tls + secretName: wildcard-staging-tls