DX improvements: tracing instrumentation and k8s fixes

.github/workflows/cd.yml

@@ -223,7 +223,7 @@ jobs:
           # Tear down the preview deployment
           echo "Tearing down smoke test deployment..."
           envsubst < cluster/preview/deployment.yaml | kubectl delete -f - --ignore-not-found
-          kubectl delete pvc -l app=mod-bot-pr-${PR_NUMBER} --ignore-not-found
+          kubectl delete pvc -l preview=pr-${PR_NUMBER} --ignore-not-found
 
           echo "Smoke test complete"
 

Dockerfile

@@ -18,9 +18,10 @@ WORKDIR /app
 
 ENV NODE_ENV=production
 
-COPY --from=build /app/node_modules /app/node_modules
-ADD package.json package-lock.json ./
-RUN npm prune --production
+COPY package.json package-lock.json ./
+RUN apk add --no-cache --virtual .build-deps python3 make g++ && \
+    npm ci --omit=dev && \
+    apk del .build-deps
 
 COPY --from=build /app/build ./build
 ADD index.prod.js ./

app/commands/report/constructLog.ts

@@ -69,7 +69,7 @@ ${preface}
 -# ${extra}${formatDistanceToNowStrict(lastReport.message.createdAt)} ago · <t:${Math.floor(lastReport.message.createdTimestamp / 1000)}:R>`).trim(),
       allowedMentions: { roles: [moderator] },
     } satisfies MessageCreateOptions;
-  });
+  }).pipe(Effect.withSpan("constructLog"));
 
 export const isForwardedMessage = (message: Message): boolean => {
   return message.reference?.type === MessageReferenceType.Forward;

app/commands/report/userLog.ts

@@ -8,7 +8,12 @@ import { Effect } from "effect";
 
 import { runEffect } from "#~/AppRuntime";
 import { type DatabaseService, type SqlError } from "#~/Database";
-import { forwardMessageSafe, sendMessage } from "#~/effects/discordSdk.ts";
+import {
+  fetchMessage,
+  forwardMessageSafe,
+  messageReply,
+  sendMessage,
+} from "#~/effects/discordSdk.ts";
 import { DiscordApiError, type NotFoundError } from "#~/effects/errors";
 import { logEffect } from "#~/effects/observability";
 import {
@@ -106,26 +111,20 @@ export function logUserMessage({
 
     if (alreadyReported && reason !== ReportReasons.modResolution) {
       // Message already reported with this reason, just add to thread
-      const latestReport = yield* Effect.tryPromise({
-        try: async () => {
-          try {
-            const reportContents = `${staff ? ` ${staff.username} ` : ""}${ReadableReasons[reason]}`;
-            const priorLogMessage = await thread.messages.fetch(
-              alreadyReported.log_message_id,
-            );
-            return priorLogMessage
-              .reply(reportContents)
-              .catch(() => priorLogMessage.channel.send(reportContents));
-          } catch (_) {
-            return thread.send(logBody);
-          }
-        },
-        catch: (error) =>
-          new DiscordApiError({
-            operation: "logUserMessage existing",
-            cause: error,
-          }),
-      });
+      const reportContents = `${staff ? ` ${staff.username} ` : ""}${ReadableReasons[reason]}`;
+      const latestReport = (yield* fetchMessage(
+        thread,
+        alreadyReported.log_message_id,
+      ).pipe(
+        Effect.flatMap((priorLogMessage) =>
+          messageReply(priorLogMessage, reportContents).pipe(
+            Effect.catchAll(() =>
+              sendMessage(priorLogMessage.channel, reportContents),
+            ),
+          ),
+        ),
+        Effect.catchAll(() => sendMessage(thread, logBody)),
+      )) as Message<true>;
 
       yield* logEffect(
         "info",

app/discord/auditLog.ts

@@ -31,16 +31,28 @@ export const fetchAuditLogEntry = (
 
       const auditLogs = yield* Effect.promise(() =>
         guild.fetchAuditLogs({ type: auditLogType, limit: 5 }),
+      ).pipe(
+        Effect.withSpan("discord.fetchAuditLogs", {
+          attributes: { attempt: attempt + 1, guildId: guild.id },
+        }),
       );
 
       const entry = findEntry(auditLogs.entries);
       if (entry?.executor) {
         yield* logEffect("debug", "AuditLog", "Record found", {
           attempt: attempt + 1,
         });
+        yield* Effect.annotateCurrentSpan({
+          "auditLog.found": true,
+          "auditLog.attempts": attempt + 1,
+        });
         return entry;
       }
     }
+    yield* Effect.annotateCurrentSpan({
+      "auditLog.found": false,
+      "auditLog.attempts": 3,
+    });
     return undefined;
   }).pipe(
     Effect.withSpan("fetchAuditLogEntry", {

app/discord/gateway.ts

@@ -76,25 +76,40 @@ export const initDiscordBot: Effect.Effect<Client> = Effect.gen(function* () {
       customId: interaction.customId,
     });
     let config: AnyCommand | undefined = undefined;
+    let commandName: string | undefined = undefined;
     switch (interaction.type) {
       case InteractionType.ApplicationCommand: {
-        config = matchCommand(interaction.commandName);
+        commandName = interaction.commandName;
+        config = matchCommand(commandName);
         break;
       }
       case InteractionType.MessageComponent:
       case InteractionType.ModalSubmit: {
-        config = matchCommand(interaction.customId);
+        commandName = interaction.customId;
+        config = matchCommand(commandName);
         break;
       }
     }
 
-    if (!config) {
+    if (!config || !commandName) {
       log("debug", "deployCommands", "no matching command found");
       return;
     }
     log("debug", "deployCommands", "found matching command", { config });
 
-    void runEffect(config.handler(interaction as never));
+    void runEffect(
+      config.handler(interaction as never).pipe(
+        Effect.withSpan(`command.${commandName}`, {
+          attributes: {
+            "command.name": commandName,
+            "command.type": interaction.type,
+            "interaction.id": interaction.id,
+            guildId: interaction.guildId,
+            userId: interaction.user.id,
+          },
+        }),
+      ),
+    );
   });
 
   const errorHandler = (error: unknown) => {

app/effects/devSpanExporter.ts

@@ -0,0 +1,136 @@
+import { SpanStatusCode } from "@opentelemetry/api";
+import { ExportResultCode, type ExportResult } from "@opentelemetry/core";
+import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base";
+
+const ORPHAN_TTL_MS = 30_000;
+
+function hrTimeToMs(hrTime: [number, number]): number {
+  return hrTime[0] * 1000 + hrTime[1] / 1_000_000;
+}
+
+/**
+ * Dev-mode span exporter that prints a human-readable timing tree to console.
+ *
+ * Accumulates spans by trace ID and prints the full tree when the root span
+ * (no parent) completes. Use with SimpleSpanProcessor for immediate output.
+ *
+ * Example output:
+ *
+ *   --- Trace: DeletionLogger.messageDelete (1823.4ms) ---
+ *     discord.fetchGuild 45.2ms
+ *     fetchAuditLogEntry 1612.8ms
+ *       discord.fetchAuditLogs 89.3ms
+ *       discord.fetchAuditLogs 76.1ms
+ *     getOrCreateDeletionLogThread 89.4ms
+ *       sql.execute 2.1ms
+ *     discord.sendMessage 48.2ms
+ */
+export class DevTreeSpanExporter implements SpanExporter {
+  private spansByTrace = new Map<
+    string,
+    { spans: ReadableSpan[]; firstSeen: number }
+  >();
+
+  export(
+    spans: ReadableSpan[],
+    resultCallback: (result: ExportResult) => void,
+  ): void {
+    for (const span of spans) {
+      const traceId = span.spanContext().traceId;
+
+      let entry = this.spansByTrace.get(traceId);
+      if (!entry) {
+        entry = { spans: [], firstSeen: Date.now() };
+        this.spansByTrace.set(traceId, entry);
+      }
+      entry.spans.push(span);
+
+      // Root span (no parent) — print the tree
+      const parentId = span.parentSpanContext?.spanId;
+      if (!parentId) {
+        this.printTree(traceId);
+        this.spansByTrace.delete(traceId);
+      }
+    }
+
+    // Evict orphaned traces older than TTL
+    const now = Date.now();
+    for (const [traceId, entry] of this.spansByTrace) {
+      if (now - entry.firstSeen > ORPHAN_TTL_MS) {
+        this.spansByTrace.delete(traceId);
+      }
+    }
+
+    resultCallback({ code: ExportResultCode.SUCCESS });
+  }
+
+  private printTree(traceId: string): void {
+    const entry = this.spansByTrace.get(traceId);
+    if (!entry) return;
+    const { spans } = entry;
+
+    // Build parent→children map
+    const childMap = new Map<string, ReadableSpan[]>();
+    let root: ReadableSpan | undefined;
+
+    for (const span of spans) {
+      const parentId = span.parentSpanContext?.spanId;
+      if (!parentId) {
+        root = span;
+      } else {
+        let children = childMap.get(parentId);
+        if (!children) {
+          children = [];
+          childMap.set(parentId, children);
+        }
+        children.push(span);
+      }
+    }
+
+    if (!root) return;
+
+    const rootDuration = hrTimeToMs(root.duration);
+    const lines: string[] = [];
+    this.renderChildren(root, childMap, 1, lines);
+
+    if (lines.length === 0) {
+      // Single span, no children — print on one line
+      console.log(`[trace] ${root.name} ${rootDuration.toFixed(1)}ms`);
+    } else {
+      console.log(`--- Trace: ${root.name} (${rootDuration.toFixed(1)}ms) ---`);
+      for (const line of lines) console.log(line);
+    }
+  }
+
+  private renderChildren(
+    parent: ReadableSpan,
+    childMap: Map<string, ReadableSpan[]>,
+    depth: number,
+    lines: string[],
+  ): void {
+    const children = childMap.get(parent.spanContext().spanId) ?? [];
+
+    // Sort by start time
+    children.sort((a, b) => {
+      const diff = a.startTime[0] - b.startTime[0];
+      return diff !== 0 ? diff : a.startTime[1] - b.startTime[1];
+    });
+
+    for (const child of children) {
+      const durationMs = hrTimeToMs(child.duration);
+      const status = child.status.code === SpanStatusCode.ERROR ? " ERROR" : "";
+      const indent = "  ".repeat(depth);
+      lines.push(`${indent}${child.name} ${durationMs.toFixed(1)}ms${status}`);
+
+      this.renderChildren(child, childMap, depth + 1, lines);
+    }
+  }
+
+  async shutdown(): Promise<void> {
+    this.spansByTrace.clear();
+  }
+
+  async forceFlush(): Promise<void> {
+    // No-op: SimpleSpanProcessor exports immediately, nothing to flush
+  }
+}