feat: ingestion, chunk, embeddings

Author: caio-pizzol

.env.example

@@ -0,0 +1,17 @@
+# Database connection
+# For local development with Docker:
+DATABASE_URL=postgresql://postgres:postgres@localhost:5432/ecma_spec
+
+# For production with NeonDB:
+# DATABASE_URL=postgresql://user:password@ep-xxx.us-east-1.aws.neon.tech/ecma_spec?sslmode=require
+
+# Embedding API keys (only one required, based on chosen provider)
+OPENAI_API_KEY=sk-...
+# GOOGLE_API_KEY=...
+# VOYAGE_API_KEY=...
+# COHERE_API_KEY=...
+
+# Embedding configuration
+EMBEDDING_PROVIDER=openai
+EMBEDDING_MODEL=text-embedding-3-large
+EMBEDDING_DIMENSIONS=3072

bun.lock

@@ -0,0 +1,23 @@
+-- ECMA-376 Spec Vector Database Schema
+-- Simple single-table design - evolve as needed
+
+CREATE EXTENSION IF NOT EXISTS vector;
+
+-- Single table for all spec content
+CREATE TABLE spec_content (
+  id SERIAL PRIMARY KEY,
+  part_number INT NOT NULL,
+  section_id TEXT,
+  title TEXT,
+  content TEXT NOT NULL,
+  content_type TEXT DEFAULT 'text',
+  embedding vector(1024),
+  created_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- Vector similarity search
+CREATE INDEX idx_content_embedding ON spec_content USING hnsw (embedding vector_cosine_ops);
+
+-- Filtering indexes
+CREATE INDEX idx_content_part ON spec_content(part_number);
+CREATE INDEX idx_content_section ON spec_content(section_id);

db/schema.sql

@@ -0,0 +1,21 @@
+services:
+  db:
+    image: pgvector/pgvector:pg16
+    container_name: ecma-spec-db
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: ecma_spec
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+      - ./db/schema.sql:/docker-entrypoint-initdb.d/schema.sql
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  pgdata:

docker-compose.yml

@@ -14,7 +14,16 @@
     "typecheck": "bun run --cwd apps/web typecheck && bun run --cwd apps/mcp-server typecheck && bun run --cwd packages/shared typecheck",
     "release": "semantic-release",
     "deploy": "bun run format && bun run lint && bun run --cwd apps/web deploy",
-    "prepare": "husky"
+    "prepare": "husky",
+    "db:up": "docker compose up -d",
+    "db:down": "docker compose down",
+    "db:reset": "docker compose down -v && docker compose up -d",
+    "db:shell": "docker compose exec db psql -U postgres -d ecma_spec",
+    "ingest": "bun scripts/ingest/pipeline.ts",
+    "ingest:chunk": "bun scripts/ingest/chunk.ts",
+    "ingest:embed": "bun scripts/ingest/embed.ts",
+    "ingest:upload": "bun scripts/ingest/upload.ts",
+    "ingest:setup": "pip install -r scripts/requirements.txt"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.3.11",

package.json

@@ -14,6 +14,9 @@
 	"scripts": {
 		"typecheck": "tsc --noEmit"
 	},
+	"dependencies": {
+		"postgres": "^3.4.5"
+	},
 	"devDependencies": {
 		"typescript": "~5.9.3"
 	}

packages/shared/package.json

@@ -1,6 +1,148 @@
-// Database client for NeonDB
-// TODO: Implement NeonDB connection
+import postgres from "postgres";
+import type { SearchResult, SpecContent } from "../types";
 
-export function createDbClient(_connectionString: string) {
-	throw new Error("Not implemented yet");
+export type DbClient = ReturnType<typeof createDbClient>;
+
+export function createDbClient(connectionString: string) {
+	const sql = postgres(connectionString);
+
+	return {
+		sql,
+
+		async close() {
+			await sql.end();
+		},
+
+		// Insert content
+		async insert(content: Omit<SpecContent, "id">) {
+			const [result] = await sql<[{ id: number }]>`
+				INSERT INTO spec_content (part_number, section_id, title, content, content_type, embedding)
+				VALUES (
+					${content.partNumber},
+					${content.sectionId},
+					${content.title},
+					${content.content},
+					${content.contentType},
+					${content.embedding ? `[${content.embedding.join(",")}]` : null}
+				)
+				RETURNING id
+			`;
+			return result.id;
+		},
+
+		// Insert multiple (batch)
+		async insertBatch(items: Omit<SpecContent, "id">[]) {
+			const values = items.map((item) => ({
+				part_number: item.partNumber,
+				section_id: item.sectionId,
+				title: item.title,
+				content: item.content,
+				content_type: item.contentType,
+				embedding: item.embedding ? `[${item.embedding.join(",")}]` : null,
+			}));
+
+			const result = await sql`
+				INSERT INTO spec_content ${sql(values)}
+				RETURNING id
+			`;
+			return result.map((r) => r.id as number);
+		},
+
+		// Update embedding
+		async updateEmbedding(id: number, embedding: number[]) {
+			await sql`
+				UPDATE spec_content
+				SET embedding = ${`[${embedding.join(",")}]`}
+				WHERE id = ${id}
+			`;
+		},
+
+		// Semantic search
+		async search(
+			queryEmbedding: number[],
+			options: { limit?: number; partNumber?: number; contentType?: string } = {},
+		): Promise<SearchResult[]> {
+			const { limit = 5, partNumber, contentType } = options;
+			const embeddingStr = `[${queryEmbedding.join(",")}]`;
+
+			const results = await sql<
+				Array<{
+					id: number;
+					part_number: number;
+					section_id: string | null;
+					title: string | null;
+					content: string;
+					content_type: string;
+					score: number;
+				}>
+			>`
+				SELECT
+					id, part_number, section_id, title, content, content_type,
+					1 - (embedding <=> ${embeddingStr}::vector) as score
+				FROM spec_content
+				WHERE embedding IS NOT NULL
+				${partNumber ? sql`AND part_number = ${partNumber}` : sql``}
+				${contentType ? sql`AND content_type = ${contentType}` : sql``}
+				ORDER BY embedding <=> ${embeddingStr}::vector
+				LIMIT ${limit}
+			`;
+
+			return results.map((r) => ({
+				id: r.id,
+				partNumber: r.part_number,
+				sectionId: r.section_id,
+				title: r.title,
+				content: r.content,
+				contentType: r.content_type,
+				score: r.score,
+			}));
+		},
+
+		// Get by section
+		async getBySection(partNumber: number, sectionId: string): Promise<SpecContent[]> {
+			const results = await sql<
+				Array<{
+					id: number;
+					part_number: number;
+					section_id: string | null;
+					title: string | null;
+					content: string;
+					content_type: string;
+				}>
+			>`
+				SELECT id, part_number, section_id, title, content, content_type
+				FROM spec_content
+				WHERE part_number = ${partNumber} AND section_id = ${sectionId}
+				ORDER BY id
+			`;
+
+			return results.map((r) => ({
+				id: r.id,
+				partNumber: r.part_number,
+				sectionId: r.section_id,
+				title: r.title,
+				content: r.content,
+				contentType: r.content_type,
+			}));
+		},
+
+		// Get stats
+		async getStats() {
+			const [stats] = await sql<[{ total: number; embedded: number }]>`
+				SELECT
+					COUNT(*) as total,
+					COUNT(*) FILTER (WHERE embedding IS NOT NULL) as embedded
+				FROM spec_content
+			`;
+			return {
+				total: Number(stats.total),
+				embedded: Number(stats.embedded),
+			};
+		},
+
+		// Clear all
+		async clearAll() {
+			await sql`TRUNCATE spec_content RESTART IDENTITY`;
+		},
+	};
 }