fix: Support large database dumps with streaming and pagination

.github/workflows/test.yaml

@@ -39,8 +39,7 @@ jobs:
         run: pnpm install --no-frozen-lockfile
 
       - name: Run tests
-        run: pnpm vitest --coverage.enabled true --coverage.reportOnFailure --coverage.reportsDirectory ./coverage || true
-        continue-on-error: true
+        run: pnpm vitest --coverage.enabled true --coverage.reportOnFailure --coverage.reportsDirectory ./coverage
 
       - name: Report Coverage
         uses: davelosert/vitest-coverage-report-action@v2

src/export/constants.ts

@@ -0,0 +1,2 @@
+// Shared constants for export operations
+export const CHUNK_SIZE = 500

src/export/csv.test.ts

@@ -63,7 +63,9 @@ describe('CSV Export Module', () => {
         expect(getTableData).toHaveBeenCalledWith(
             'users',
             mockDataSource,
-            mockConfig
+            mockConfig,
+            undefined,
+            undefined
         )
         expect(createExportResponse).toHaveBeenCalledWith(
             'id,name,age\n1,Alice,30\n2,Bob,25\n',
@@ -85,7 +87,9 @@ describe('CSV Export Module', () => {
         expect(getTableData).toHaveBeenCalledWith(
             'non_existent_table',
             mockDataSource,
-            mockConfig
+            mockConfig,
+            undefined,
+            undefined
         )
         expect(response.status).toBe(404)
 
@@ -113,7 +117,9 @@ describe('CSV Export Module', () => {
         expect(getTableData).toHaveBeenCalledWith(
             'empty_table',
             mockDataSource,
-            mockConfig
+            mockConfig,
+            undefined,
+            undefined
         )
         expect(createExportResponse).toHaveBeenCalledWith(
             '',

src/export/csv.ts

@@ -2,14 +2,26 @@ import { getTableData, createExportResponse } from './index'
 import { createResponse } from '../utils'
 import { DataSource } from '../types'
 import { StarbaseDBConfiguration } from '../handler'
+import { CHUNK_SIZE } from './constants'
+
+function escapeCsvValue(value: any): string {
+    if (value === null || value === undefined) return ''
+    const str = String(value)
+    if (str.includes(',') || str.includes('"') || str.includes('\n')) {
+        return `"${str.replace(/"/g, '""')}"`
+    }
+    return str
+}
 
 export async function exportTableToCsvRoute(
     tableName: string,
     dataSource: DataSource,
-    config: StarbaseDBConfiguration
+    config: StarbaseDBConfiguration,
+    limit?: number,
+    offset?: number
 ): Promise<Response> {
     try {
-        const data = await getTableData(tableName, dataSource, config)
+        const data = await getTableData(tableName, dataSource, config, limit, offset)
 
         if (data === null) {
             return createResponse(
@@ -19,29 +31,12 @@ export async function exportTableToCsvRoute(
             )
         }
 
-        // Convert the result to CSV
         let csvContent = ''
         if (data.length > 0) {
-            // Add headers
             csvContent += Object.keys(data[0]).join(',') + '\n'
-
-            // Add data rows
-            data.forEach((row: any) => {
-                csvContent +=
-                    Object.values(row)
-                        .map((value) => {
-                            if (
-                                typeof value === 'string' &&
-                                (value.includes(',') ||
-                                    value.includes('"') ||
-                                    value.includes('\n'))
-                            ) {
-                                return `"${value.replace(/"/g, '""')}"`
-                            }
-                            return value
-                        })
-                        .join(',') + '\n'
-            })
+            for (const row of data) {
+                csvContent += Object.values(row).map(escapeCsvValue).join(',') + '\n'
+            }
         }
 
         return createExportResponse(
@@ -54,3 +49,57 @@ export async function exportTableToCsvRoute(
         return createResponse(undefined, 'Failed to export table to CSV', 500)
     }
 }
+
+export async function exportTableToCsvStreamRoute(
+    tableName: string,
+    dataSource: DataSource,
+    config: StarbaseDBConfiguration
+): Promise<Response> {
+    try {
+        const firstChunk = await getTableData(tableName, dataSource, config, 1, 0)
+        if (firstChunk === null) {
+            return createResponse(undefined, `Table '${tableName}' does not exist.`, 404)
+        }
+
+        // Get column names from first row
+        const cols = firstChunk.length > 0 ? Object.keys(firstChunk[0]) : []
+
+        const stream = new ReadableStream({
+            async start(controller) {
+                try {
+                    // Write CSV header
+                    if (cols.length > 0) {
+                        controller.enqueue(new TextEncoder().encode(cols.join(',') + '\n'))
+                    }
+
+                    let offset = 0
+                    while (true) {
+                        const chunk = await getTableData(tableName, dataSource, config, CHUNK_SIZE, offset)
+                        if (!chunk || chunk.length === 0) break
+
+                        let csvPart = ''
+                        for (const row of chunk) {
+                            csvPart += Object.values(row).map(escapeCsvValue).join(',') + '\n'
+                        }
+                        controller.enqueue(new TextEncoder().encode(csvPart))
+                        offset += CHUNK_SIZE
+                    }
+                } catch (err: any) {
+                    controller.error(err)
+                    return
+                }
+                controller.close()
+            }
+        })
+
+        const headers = new Headers({
+            'Content-Type': 'text/csv',
+            'Content-Disposition': `attachment; filename="${tableName}_export.csv"`,
+            'Transfer-Encoding': 'chunked',
+        })
+        return new Response(stream, { headers })
+    } catch (error: any) {
+        console.error('CSV Export Error:', error)
+        return createResponse(undefined, 'Failed to export table to CSV', 500)
+    }
+}

src/export/dump.test.ts

@@ -42,20 +42,24 @@ describe('Database Dump Module', () => {
     it('should return a database dump when tables exist', async () => {
         vi.mocked(executeOperation)
             .mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }])
+            // Users table
             .mockResolvedValueOnce([
                 { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
             ])
             .mockResolvedValueOnce([
                 { id: 1, name: 'Alice' },
                 { id: 2, name: 'Bob' },
             ])
+            .mockResolvedValueOnce([]) // End of users data
+            // Orders table
             .mockResolvedValueOnce([
                 { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' },
             ])
             .mockResolvedValueOnce([
                 { id: 1, total: 99.99 },
                 { id: 2, total: 49.5 },
             ])
+            .mockResolvedValueOnce([]) // End of orders data
 
         const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
 

src/export/dump.ts

@@ -2,6 +2,7 @@ import { executeOperation } from '.'
 import { StarbaseDBConfiguration } from '../handler'
 import { DataSource } from '../types'
 import { createResponse } from '../utils'
+import { CHUNK_SIZE } from './constants'
 
 export async function dumpDatabaseRoute(
     dataSource: DataSource,
@@ -16,54 +17,75 @@ export async function dumpDatabaseRoute(
         )
 
         const tables = tablesResult.map((row: any) => row.name)
-        let dumpContent = 'SQLite format 3\0' // SQLite file header
 
-        // Iterate through all tables
-        for (const table of tables) {
-            // Get table schema
-            const schemaResult = await executeOperation(
-                [
-                    {
-                        sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`,
-                    },
-                ],
-                dataSource,
-                config
-            )
+        // Create a readable stream for progressive dump generation
+        const stream = new ReadableStream({
+            async start(controller) {
+                try {
+                    // SQLite file header
+                    controller.enqueue(new TextEncoder().encode('SQLite format 3\0'))
 
-            if (schemaResult.length) {
-                const schema = schemaResult[0].sql
-                dumpContent += `\n-- Table: ${table}\n${schema};\n\n`
-            }
+                    for (const table of tables) {
+                        // Get table schema
+                        const schemaResult = await executeOperation(
+                            [{ sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';` }],
+                            dataSource,
+                            config
+                        )
 
-            // Get table data
-            const dataResult = await executeOperation(
-                [{ sql: `SELECT * FROM ${table};` }],
-                dataSource,
-                config
-            )
+                        if (schemaResult && schemaResult.length > 0) {
+                            const schema = schemaResult[0].sql
+                            controller.enqueue(new TextEncoder().encode(`\n-- Table: ${table}\n${schema};\n\n`))
+                        }
 
-            for (const row of dataResult) {
-                const values = Object.values(row).map((value) =>
-                    typeof value === 'string'
-                        ? `'${value.replace(/'/g, "''")}'`
-                        : value
-                )
-                dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
-            }
+                        // Get table data in chunks
+                        let offset = 0
+                        let hasMore = true
+
+                        while (hasMore) {
+                            const dataResult = await executeOperation(
+                                [{ sql: `SELECT * FROM ${table} LIMIT ? OFFSET ?;`, params: [CHUNK_SIZE, offset] }],
+                                dataSource,
+                                config
+                            )
 
-            dumpContent += '\n'
-        }
+                            if (!dataResult || dataResult.length === 0) {
+                                hasMore = false
+                                break
+                            }
 
-        // Create a Blob from the dump content
-        const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' })
+                            let chunk = ''
+                            for (const row of dataResult) {
+                                const values = Object.values(row).map((value: any) =>
+                                    typeof value === 'string'
+                                        ? `'${value.replace(/'/g, "''")}'`
+                                        : value === null
+                                            ? 'NULL'
+                                            : value
+                                )
+                                chunk += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
+                            }
+                            controller.enqueue(new TextEncoder().encode(chunk))
+                            offset += CHUNK_SIZE
+                        }
+
+                        controller.enqueue(new TextEncoder().encode('\n'))
+                    }
+                } catch (err: any) {
+                    controller.error(err)
+                    return
+                }
+                controller.close()
+            }
+        })
 
         const headers = new Headers({
             'Content-Type': 'application/x-sqlite3',
             'Content-Disposition': 'attachment; filename="database_dump.sql"',
+            'Transfer-Encoding': 'chunked',
         })
 
-        return new Response(blob, { headers })
+        return new Response(stream, { headers })
     } catch (error: any) {
         console.error('Database Dump Error:', error)
         return createResponse(undefined, 'Failed to create database dump', 500)

src/export/index.ts

@@ -22,7 +22,9 @@ export async function executeOperation(
 export async function getTableData(
     tableName: string,
     dataSource: DataSource,
-    config: StarbaseDBConfiguration
+    config: StarbaseDBConfiguration,
+    limit?: number,
+    offset?: number
 ): Promise<any[] | null> {
     try {
         // Verify if the table exists
@@ -41,9 +43,22 @@ export async function getTableData(
             return null
         }
 
+        // Build query with optional pagination
+        let query = `SELECT * FROM ${tableName}`
+        const params: any[] = []
+        if (limit !== undefined) {
+            query += ` LIMIT ?`
+            params.push(limit)
+        }
+        if (offset !== undefined) {
+            query += ` OFFSET ?`
+            params.push(offset)
+        }
+        query += `;`
+
         // Get table data
         const dataResult = await executeOperation(
-            [{ sql: `SELECT * FROM ${tableName};` }],
+            [{ sql: query, params: params.length > 0 ? params : undefined }],
             dataSource,
             config
         )

src/export/json.test.ts

@@ -76,7 +76,9 @@ describe('JSON Export Module', () => {
         expect(getTableData).toHaveBeenCalledWith(
             'users',
             mockDataSource,
-            mockConfig
+            mockConfig,
+            undefined,
+            undefined
         )
         expect(createExportResponse).toHaveBeenCalledWith(
             JSON.stringify(mockData, null, 4),
@@ -101,6 +103,13 @@ describe('JSON Export Module', () => {
             mockConfig
         )
 
+        expect(getTableData).toHaveBeenCalledWith(
+            'empty_table',
+            mockDataSource,
+            mockConfig,
+            undefined,
+            undefined
+        )
         expect(createExportResponse).toHaveBeenCalledWith(
             '[]',
             'empty_table_export.json',
@@ -128,6 +137,13 @@ describe('JSON Export Module', () => {
             mockConfig
         )
 
+        expect(getTableData).toHaveBeenCalledWith(
+            'special_chars',
+            mockDataSource,
+            mockConfig,
+            undefined,
+            undefined
+        )
         expect(createExportResponse).toHaveBeenCalledWith(
             JSON.stringify(specialCharsData, null, 4),
             'special_chars_export.json',