From b9719333085d39ab179634092ef2b00d8ea7db28 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 4 Jun 2026 18:51:17 -0400 Subject: [PATCH 1/2] feat: expose preview evidence refs --- packages/runtime-core/src/index.ts | 1 + .../runtime-core/src/runtime-contracts.ts | 58 +++++++ .../src/artifact-bundle-builder.ts | 158 ++++++++++++++++++ packages/runtime-playground/src/artifacts.ts | 3 + scripts/artifact-contract-smoke.ts | 31 ++++ .../artifact-package-provenance-shape.json | 6 +- 6 files changed, 254 insertions(+), 3 deletions(-) diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts index f3418156..6866bc56 100644 --- a/packages/runtime-core/src/index.ts +++ b/packages/runtime-core/src/index.ts @@ -82,6 +82,7 @@ export interface ArtifactReview { runtimeEpisodeTrace?: string runtimeReferenceManifest?: string runtimeReplayReferenceIndex?: string + previewEvidence?: string agentResult?: string transcript?: string } diff --git a/packages/runtime-core/src/runtime-contracts.ts b/packages/runtime-core/src/runtime-contracts.ts index 31620050..ac099d5f 100644 --- a/packages/runtime-core/src/runtime-contracts.ts +++ b/packages/runtime-core/src/runtime-contracts.ts @@ -513,6 +513,63 @@ export interface ArtifactPreview { holdSeconds?: number } +export interface ArtifactPreviewUrlRef { + kind: "preview-url" + availability: "reviewer-safe" | "local-only" | "unavailable" + reviewerSafe: boolean + url?: string + reason?: string +} + +export interface ArtifactPreviewEvidence { + schema: "wp-codebox/preview-evidence/v1" + createdAt: string + session: { + kind: "browser-playground-session" + id: string + runtimeId: string + backend: RuntimeBackendKind + environment: { + kind: EnvironmentSpec["kind"] + name: string + version: string + } + } + run: RuntimeEpisodeTraceRef + preview: { + status: ArtifactPreview["status"] | "unavailable" + lifecycle: ArtifactPreview["lifecycle"] | "not-started" + source?: ArtifactPreview["source"] + createdAt?: string + expiresAt?: string + holdSeconds?: number + url: ArtifactPreviewUrlRef + publicUrl?: ArtifactPreviewUrlRef + localUrl?: ArtifactPreviewUrlRef + siteUrl?: ArtifactPreviewUrlRef + } + readiness: { + ready: boolean + status: BrowserStartupProgressEvent["status"] | "not-started" + phase?: BrowserStartupProgressEvent["phase"] + events: Array<{ + id: string + phase: BrowserStartupProgressEvent["phase"] + status: BrowserStartupProgressEvent["status"] + label?: string + elapsed_ms?: number + timestamp: string + }> + } + components: { + packages?: ArtifactPackageProvenance + runtime: { + backend: RuntimeBackendKind + wordpressVersion?: string + } + } +} + export interface ArtifactBundle { id: string directory: string @@ -542,6 +599,7 @@ export interface ArtifactBundle { runtimeReferenceManifestPath?: string runtimeReferenceIndexPath?: string runtimeReplayReferenceIndexPath?: string + previewEvidencePath?: string preview?: ArtifactPreview contentDigest: string createdAt: string diff --git a/packages/runtime-playground/src/artifact-bundle-builder.ts b/packages/runtime-playground/src/artifact-bundle-builder.ts index 7a038c02..977054e9 100644 --- a/packages/runtime-playground/src/artifact-bundle-builder.ts +++ b/packages/runtime-playground/src/artifact-bundle-builder.ts @@ -11,13 +11,16 @@ import { type ArtifactManifest, type ArtifactManifestFile, type ArtifactPreview, + type ArtifactPreviewEvidence, type ArtifactReviewBrowserSummary, type ArtifactSpec, + type BrowserStartupProgressEvent, type ExecutionResult, type LifecycleEvent, type MountSpec, type ObservationResult, type RuntimeCreateSpec, + type RuntimeEpisodeTraceRef, type RuntimeInfo, type Snapshot, } from "@automattic/wp-codebox-core" @@ -101,6 +104,7 @@ export class ArtifactBundleBuilder { const runtimeReferenceManifestPath = join(filesDirectory, "runtime-reference-manifest.json") const runtimeReferenceIndexPath = join(filesDirectory, "runtime-reference-index.json") const runtimeReplayReferenceIndexPath = join(filesDirectory, "runtime-replay-index.json") + const previewEvidencePath = join(filesDirectory, "preview-evidence.json") const redactor = new ArtifactRedactor(source.spec.secretEnv) await source.redactBrowserArtifacts(redactor) @@ -132,6 +136,20 @@ export class ArtifactBundleBuilder { context: source.spec.metadata ?? {}, mounts: source.mounts, }) + const artifactBundleRef: RuntimeEpisodeTraceRef = { + kind: "artifact-bundle", + id: bundleId, + digest: { algorithm: "sha256", value: contentDigest }, + path: "manifest.json", + } + const previewEvidence = buildPreviewEvidence({ + createdAt, + runtime, + preview, + events: source.events, + packages: provenance.packages, + artifactBundleRef, + }) const metadata: Record = { id: bundleId, contentDigest: contentDigestMetadata, @@ -163,6 +181,7 @@ export class ArtifactBundleBuilder { runtimeCreatedAt: source.runtimeCreatedAt, mounts: source.mounts, preview, + previewEvidencePath: "files/preview-evidence.json", browser, diagnosticsPath: "files/diagnostics.json", }) @@ -184,6 +203,7 @@ export class ArtifactBundleBuilder { runtimeReferenceManifest: relative(source.artifactRoot, runtimeReferenceManifestPath), runtimeReferenceIndex: relative(source.artifactRoot, runtimeReferenceIndexPath), runtimeReplayReferenceIndex: relative(source.artifactRoot, runtimeReplayReferenceIndexPath), + previewEvidence: relative(source.artifactRoot, previewEvidencePath), mountDiffs: relative(source.artifactRoot, diffsPath), ...(runtimeSnapshotFiles.length > 0 ? { runtimeSnapshots: runtimeSnapshotFiles.map((file) => relative(source.artifactRoot, file.path)) } : {}), ...(browser ? { browser: browser.probes.find((probe) => probe.summaryFile)?.summaryFile ?? "files/browser/summary.json" } : {}), @@ -225,6 +245,7 @@ export class ArtifactBundleBuilder { artifactManifestFile(runtimeReferenceManifestPath, "runtime-reference-manifest", "application/json"), artifactManifestFile(runtimeReferenceIndexPath, "runtime-reference-index", "application/json"), artifactManifestFile(runtimeReplayReferenceIndexPath, "runtime-replay-index", "application/json"), + artifactManifestFile(previewEvidencePath, "preview-evidence", "application/json"), ...source.browserManifestFiles(), ...source.observationManifestFiles(), ...source.pluginCheckManifestFiles(), @@ -253,6 +274,7 @@ export class ArtifactBundleBuilder { await writeFile(patchPath, redactedPatch) await writeRedactedArtifact(redactor, diagnosticsPath, source.artifactRoot, `${JSON.stringify(diagnostics, null, 2)}\n`) await writeRedactedArtifact(redactor, testResultsPath, source.artifactRoot, `${JSON.stringify(testResults, null, 2)}\n`) + await writeRedactedArtifact(redactor, previewEvidencePath, source.artifactRoot, `${JSON.stringify(previewEvidence, null, 2)}\n`) const redaction = redactor.summary() if (redaction.total > 0) { review.redaction = redaction @@ -351,6 +373,7 @@ export class ArtifactBundleBuilder { runtimeReferenceManifestPath, runtimeReferenceIndexPath, runtimeReplayReferenceIndexPath, + previewEvidencePath, ...(preview ? { preview } : {}), contentDigest, createdAt, @@ -358,6 +381,141 @@ export class ArtifactBundleBuilder { } } +function buildPreviewEvidence({ + artifactBundleRef, + createdAt, + events, + packages, + preview, + runtime, +}: { + artifactBundleRef: RuntimeEpisodeTraceRef + createdAt: string + events: LifecycleEvent[] + packages?: ArtifactPreviewEvidence["components"]["packages"] + preview?: ArtifactPreview + runtime: RuntimeInfo +}): ArtifactPreviewEvidence { + const progressEvents = events.flatMap((event) => { + if (event.type !== "runtime.browser-startup-progress") { + return [] + } + + const progress = event.data?.event + if (!isBrowserStartupProgressEvent(progress)) { + return [] + } + + return [{ + id: event.id, + phase: progress.phase, + status: progress.status, + label: progress.label, + elapsed_ms: progress.elapsed_ms, + timestamp: event.timestamp, + }] + }) + const lastProgress = progressEvents.at(-1) + const ready = progressEvents.some((event) => event.phase === "preview:ready" && event.status === "complete") + + return { + schema: "wp-codebox/preview-evidence/v1", + createdAt, + session: { + kind: "browser-playground-session", + id: `browser-playground-session-${runtime.id}`, + runtimeId: runtime.id, + backend: runtime.backend, + environment: { + kind: runtime.environment.kind, + name: runtime.environment.name ?? runtime.environment.kind, + version: runtime.environment.version ?? "unknown", + }, + }, + run: artifactBundleRef, + preview: { + status: preview?.status ?? "unavailable", + lifecycle: preview?.lifecycle ?? "not-started", + source: preview?.source, + createdAt: preview?.createdAt, + expiresAt: preview?.expiresAt, + holdSeconds: preview?.holdSeconds, + url: safePreviewUrlRef(preview?.url), + ...(preview?.publicUrl ? { publicUrl: safePreviewUrlRef(preview.publicUrl) } : {}), + ...(preview?.localUrl ? { localUrl: safePreviewUrlRef(preview.localUrl) } : {}), + ...(preview?.siteUrl ? { siteUrl: safePreviewUrlRef(preview.siteUrl) } : {}), + }, + readiness: { + ready, + status: lastProgress?.status ?? "not-started", + phase: lastProgress?.phase, + events: progressEvents, + }, + components: { + packages, + runtime: { + backend: runtime.backend, + wordpressVersion: runtime.environment.version, + }, + }, + } +} + +function isBrowserStartupProgressEvent(value: unknown): value is BrowserStartupProgressEvent { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false + } + + const candidate = value as Partial + return typeof candidate.phase === "string" && typeof candidate.status === "string" +} + +function safePreviewUrlRef(url: string | undefined): ArtifactPreviewEvidence["preview"]["url"] { + if (!url) { + return { + kind: "preview-url", + availability: "unavailable", + reviewerSafe: false, + reason: "preview-url-unavailable", + } + } + + try { + const parsed = new URL(url) + if (isLocalPreviewHost(parsed.hostname)) { + return { + kind: "preview-url", + availability: "local-only", + reviewerSafe: false, + reason: "loopback-url-omitted", + } + } + } catch { + return { + kind: "preview-url", + availability: "unavailable", + reviewerSafe: false, + reason: "invalid-url-omitted", + } + } + + return { + kind: "preview-url", + availability: "reviewer-safe", + reviewerSafe: true, + url, + } +} + +function isLocalPreviewHost(hostname: string): boolean { + const normalized = hostname.toLowerCase().replace(/^\[|\]$/g, "") + return normalized === "localhost" + || normalized === "0.0.0.0" + || normalized === "127.0.0.1" + || normalized === "::1" + || normalized.startsWith("127.") +} + async function writeJsonLines(path: string, records: unknown[], redactor: ArtifactRedactor, artifactRoot: string): Promise { await writeRedactedArtifact(redactor, path, artifactRoot, records.length > 0 ? `${records.map((record) => JSON.stringify(record)).join("\n")}\n` : "") } diff --git a/packages/runtime-playground/src/artifacts.ts b/packages/runtime-playground/src/artifacts.ts index 587d92a2..445e359f 100644 --- a/packages/runtime-playground/src/artifacts.ts +++ b/packages/runtime-playground/src/artifacts.ts @@ -271,6 +271,7 @@ export function buildArtifactReview({ runtimeCreatedAt, mounts, preview, + previewEvidencePath, browser, diagnosticsPath, }: { @@ -283,6 +284,7 @@ export function buildArtifactReview({ runtimeCreatedAt: string mounts: MountSpec[] preview?: ArtifactPreview + previewEvidencePath?: string browser?: ArtifactReviewBrowserSummary diagnosticsPath?: string }): ArtifactReview { @@ -366,6 +368,7 @@ export function buildArtifactReview({ testResults: "files/test-results.json", runtimeReferenceManifest: "files/runtime-reference-manifest.json", runtimeReplayReferenceIndex: "files/runtime-replay-index.json", + ...(previewEvidencePath ? { previewEvidence: previewEvidencePath } : {}), }, ...(browser ? { browser } : {}), riskFlags: suspiciousFullFileRewriteRiskFlags(patch), diff --git a/scripts/artifact-contract-smoke.ts b/scripts/artifact-contract-smoke.ts index e738611a..7d21ec8c 100644 --- a/scripts/artifact-contract-smoke.ts +++ b/scripts/artifact-contract-smoke.ts @@ -48,6 +48,7 @@ try { assert.ok(artifacts.diagnosticsPath, "artifact bundle should expose diagnosticsPath") assert.ok(artifacts.testResultsPath, "artifact bundle should expose testResultsPath") assert.ok(artifacts.reviewPath, "artifact bundle should expose reviewPath") + assert.ok(artifacts.previewEvidencePath, "artifact bundle should expose previewEvidencePath") assert.equal(artifacts.preview.status, "available") assert.equal(artifacts.preview.lifecycle, "held-after-run") assert.equal(artifacts.preview.holdSeconds, 1) @@ -66,6 +67,7 @@ try { const diagnostics = JSON.parse(await readFile(artifacts.diagnosticsPath, "utf8")) const testResults = JSON.parse(await readFile(artifacts.testResultsPath, "utf8")) const review = JSON.parse(await readFile(artifacts.reviewPath, "utf8")) + const previewEvidence = JSON.parse(await readFile(artifacts.previewEvidencePath, "utf8")) const runtimeReferenceManifest = JSON.parse(await readFile(artifacts.runtimeReferenceManifestPath, "utf8")) const runtimeReferenceIndex = JSON.parse(await readFile(artifacts.runtimeReferenceIndexPath, "utf8")) const runtimeReplayReferenceIndex = JSON.parse(await readFile(artifacts.runtimeReplayReferenceIndexPath, "utf8")) @@ -96,6 +98,7 @@ try { assert.ok(manifest.files.some((file: { path: string; kind: string }) => file.path === "files/runtime-reference-manifest.json" && file.kind === "runtime-reference-manifest")) assert.ok(manifest.files.some((file: { path: string; kind: string }) => file.path === "files/runtime-reference-index.json" && file.kind === "runtime-reference-index")) assert.ok(manifest.files.some((file: { path: string; kind: string }) => file.path === "files/runtime-replay-index.json" && file.kind === "runtime-replay-index")) + assert.ok(manifest.files.some((file: { path: string; kind: string }) => file.path === "files/preview-evidence.json" && file.kind === "preview-evidence")) assert.ok(manifest.files.some((file: { path: string; kind: string }) => file.path === "files/runtime-evidence/run-attestation.json" && file.kind === "run-attestation")) const runtimeEvidence = metadata.artifacts.runtimeEvidence assert.match(runtimeEvidence["run-attestation"].sha256, /^[a-f0-9]{64}$/) @@ -109,6 +112,7 @@ try { runtimeReferenceManifest: "files/runtime-reference-manifest.json", runtimeReferenceIndex: "files/runtime-reference-index.json", runtimeReplayReferenceIndex: "files/runtime-replay-index.json", + previewEvidence: "files/preview-evidence.json", mountDiffs: "files/diffs.json", runtimeEvidence, }) @@ -185,6 +189,24 @@ try { assert.equal(review.evidence.diagnostics, "files/diagnostics.json") assert.equal(review.evidence.testResults, "files/test-results.json") assert.equal(review.evidence.runtimeReferenceManifest, "files/runtime-reference-manifest.json") + assert.equal(review.evidence.previewEvidence, "files/preview-evidence.json") + assert.equal(previewEvidence.schema, "wp-codebox/preview-evidence/v1") + assert.equal(previewEvidence.session.kind, "browser-playground-session") + assert.equal(previewEvidence.session.runtimeId, output.runtime.id) + assert.equal(previewEvidence.session.backend, "wordpress-playground") + assert.equal(previewEvidence.run.kind, "artifact-bundle") + assert.equal(previewEvidence.run.id, artifacts.id) + assert.equal(previewEvidence.run.path, "manifest.json") + assert.equal(previewEvidence.run.digest.value, artifacts.contentDigest) + assert.equal(previewEvidence.preview.url.availability, "reviewer-safe") + assert.equal(previewEvidence.preview.url.url, "https://preview.example.test/codebox/") + assert.equal(previewEvidence.preview.publicUrl.url, "https://preview.example.test/codebox/") + assert.equal(previewEvidence.preview.localUrl.availability, "local-only") + assert.equal(Object.hasOwn(previewEvidence.preview.localUrl, "url"), false) + assert.equal(JSON.stringify(previewEvidence).includes("127.0.0.1"), false) + assert.equal(previewEvidence.readiness.ready, true) + assert.ok(previewEvidence.readiness.events.some((event: { phase: string; status: string }) => event.phase === "preview:ready" && event.status === "complete")) + assert.deepEqual(previewEvidence.components.packages, metadata.provenance.packages) assert.equal(runtimeReferenceIndex.schema, "wp-codebox/runtime-reference-index/v1") assert.equal(runtimeReferenceIndex.summary.references, runtimeReferenceIndex.references.length) assert.equal(runtimeReferenceIndex.summary.present, runtimeReferenceIndex.present.length) @@ -270,6 +292,10 @@ try { assert.equal(runOutput.artifacts.preview.url, "https://run-preview.example.test/") assert.equal(runOutput.artifacts.preview.publicUrl, "https://run-preview.example.test/") assert.match(runOutput.artifacts.preview.localUrl, /^http:\/\/127\.0\.0\.1:/) + const runPreviewEvidence = JSON.parse(await readFile(runOutput.artifacts.previewEvidencePath, "utf8")) + assert.equal(runPreviewEvidence.preview.url.url, "https://run-preview.example.test/") + assert.equal(runPreviewEvidence.preview.localUrl.availability, "local-only") + assert.equal(JSON.stringify(runPreviewEvidence).includes("127.0.0.1"), false) const agentMount = join(artifactsDirectory, "agent-mounted-component") await mkdir(agentMount, { recursive: true }) @@ -336,9 +362,14 @@ try { const agentArtifacts = await runtime.collectArtifacts({ includeLogs: true }) const agentMetadata = JSON.parse(await readFile(agentArtifacts.metadataPath, "utf8")) const agentReview = JSON.parse(await readFile(agentArtifacts.reviewPath, "utf8")) + const agentPreviewEvidence = JSON.parse(await readFile(agentArtifacts.previewEvidencePath ?? "", "utf8")) assert.equal(agentArtifacts.preview?.status, "expired-on-completion") assert.equal(agentReview.preview.status, "expired-on-completion") assert.equal(agentReview.preview.lifecycle, "destroyed-on-completion") + assert.equal(agentReview.evidence.previewEvidence, "files/preview-evidence.json") + assert.equal(agentPreviewEvidence.preview.url.availability, "local-only") + assert.equal(Object.hasOwn(agentPreviewEvidence.preview.url, "url"), false) + assert.equal(JSON.stringify(agentPreviewEvidence).includes("127.0.0.1"), false) assert.equal(agentMetadata.provenance.task.input, "Cook provenance smoke") assert.deepEqual(agentMetadata.provenance.agent, { agent: "sandbox-agent", provider: "openai", model: "gpt-5.5" }) assert.equal(agentReview.provenance.agent.model, "gpt-5.5") diff --git a/tests/fixtures/artifact-package-provenance-shape.json b/tests/fixtures/artifact-package-provenance-shape.json index a5784adc..2133faf5 100644 --- a/tests/fixtures/artifact-package-provenance-shape.json +++ b/tests/fixtures/artifact-package-provenance-shape.json @@ -2,15 +2,15 @@ "schema": "wp-codebox/package-provenance/v1", "wpCodebox": { "name": "wp-codebox-workspace", - "version": "0.4.0" + "version": "0.5.2" }, "runtimeCore": { "name": "@automattic/wp-codebox-core", - "version": "0.4.0" + "version": "0.5.2" }, "runtimePlayground": { "name": "@automattic/wp-codebox-playground", - "version": "0.4.0" + "version": "0.5.2" }, "playground": { "cli": { From e66fd412bac3d1c44151fee576ae73965aaf9568 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 4 Jun 2026 18:56:38 -0400 Subject: [PATCH 2/2] test: sync package provenance fixture --- tests/fixtures/artifact-package-provenance-shape.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/artifact-package-provenance-shape.json b/tests/fixtures/artifact-package-provenance-shape.json index 2133faf5..9140cf7a 100644 --- a/tests/fixtures/artifact-package-provenance-shape.json +++ b/tests/fixtures/artifact-package-provenance-shape.json @@ -2,15 +2,15 @@ "schema": "wp-codebox/package-provenance/v1", "wpCodebox": { "name": "wp-codebox-workspace", - "version": "0.5.2" + "version": "0.6.0" }, "runtimeCore": { "name": "@automattic/wp-codebox-core", - "version": "0.5.2" + "version": "0.6.0" }, "runtimePlayground": { "name": "@automattic/wp-codebox-playground", - "version": "0.5.2" + "version": "0.6.0" }, "playground": { "cli": {