diff --git a/.gitignore b/.gitignore index acc78571f0..a71c6785a6 100644 --- a/.gitignore +++ b/.gitignore @@ -155,4 +155,5 @@ dist .sentryclirc # IDE specifics -.idea \ No newline at end of file +.idea + diff --git a/check_results.txt b/check_results.txt new file mode 100644 index 0000000000..2f8cfc0d56 Binary files /dev/null and b/check_results.txt differ diff --git a/package.json b/package.json index 57e3e7a94a..20e74c58e0 100644 --- a/package.json +++ b/package.json @@ -90,5 +90,6 @@ "overrides": { "vite": "npm:rolldown-vite@latest", "minimatch": "10.2.1" - } + }, + "packageManager": "pnpm@10.30.3+sha512.c961d1e0a2d8e354ecaa5166b822516668b7f44cb5bd95122d590dd81922f606f5473b6d23ec4a5be05e7fcd18e8488d47d978bbe981872f1145d06e9a740017" } diff --git a/src/lib/actions/analytics.ts b/src/lib/actions/analytics.ts index 023e2fbeee..6cdc1760e4 100644 --- a/src/lib/actions/analytics.ts +++ b/src/lib/actions/analytics.ts @@ -156,6 +156,7 @@ export enum Click { DatabaseDatabaseDelete = 'click_database_delete', DatabaseImportCsv = 'click_database_import_csv', DatabaseExportCsv = 'click_database_export_csv', + DatabaseExportJson = 'click_database_export_json', DomainCreateClick = 'click_domain_create', DomainDeleteClick = 'click_domain_delete', DomainRetryDomainVerificationClick = 'click_domain_retry_domain_verification', @@ -283,6 +284,7 @@ export enum Submit { DatabaseUpdateName = 'submit_database_update_name', DatabaseImportCsv = 'submit_database_import_csv', DatabaseExportCsv = 'submit_database_export_csv', + DatabaseExportJson = 'submit_database_export_json', DatabaseBackupDelete = 'submit_database_backup_delete', DatabaseBackupPolicyCreate = 'submit_database_backup_policy_create', diff --git a/src/lib/components/index.ts b/src/lib/components/index.ts index 09b4d248c3..8a17ad97b7 100644 --- a/src/lib/components/index.ts +++ b/src/lib/components/index.ts @@ -14,6 +14,7 @@ export { default as CopyInput } from './copyInput.svelte'; export { default as UploadBox } from './uploadBox.svelte'; export { default as BackupRestoreBox } from './backupRestoreBox.svelte'; export { default as CsvExportBox } from './csvExportBox.svelte'; +export { default as JsonExportBox } from './jsonExportBox.svelte'; export { default as List } from './list.svelte'; export { default as ListItem } from './listItem.svelte'; export { default as Empty } from './empty.svelte'; diff --git a/src/lib/components/jsonExportBox.svelte b/src/lib/components/jsonExportBox.svelte new file mode 100644 index 0000000000..c80d6e2e26 --- /dev/null +++ b/src/lib/components/jsonExportBox.svelte @@ -0,0 +1,178 @@ + + +{#if showBox} + +
+
+

+ + Exporting rows ({jobCount}) + +

+ + +
+ +
+ {#each jobEntries as [key, job] (key)} +
+
    +
  • +
    +
    + + Exporting {job.tableName} {text(job)} + +
    +
    +
    +
    +
  • +
+
+ {/each} +
+
+
+{/if} + + diff --git a/src/lib/stores/jsonExport.ts b/src/lib/stores/jsonExport.ts new file mode 100644 index 0000000000..c15cd1151e --- /dev/null +++ b/src/lib/stores/jsonExport.ts @@ -0,0 +1,215 @@ +import { writable } from 'svelte/store'; +import { sdk } from '$lib/stores/sdk'; +import { Query } from '@appwrite.io/console'; +import type { Models } from '@appwrite.io/console'; + +export type JsonExportStatus = 'idle' | 'pending' | 'processing' | 'completed' | 'failed'; + +export type JsonExportJob = { + status: JsonExportStatus; + tableName: string; + filename: string; + fetchedRows: number; + totalRows: number; + error?: string; + + // Config + region: string; + project: string; + databaseId: string; + tableId: string; + columns: string[]; + queries: string[]; + wildcardQueries: string[]; + prettyPrint: boolean; +}; + +export type JsonExportConfig = { + region: string; + project: string; + databaseId: string; + tableId: string; + tableName: string; + filename: string; + columns: string[]; + queries: string[]; + wildcardQueries: string[]; + prettyPrint: boolean; +}; + +function createJsonExportStore() { + const { subscribe, set, update } = writable>(new Map()); + + async function startExport(config: JsonExportConfig) { + const jobId = `${config.databaseId}:${config.tableId}:${Date.now()}`; + + const job: JsonExportJob = { + status: 'pending', + tableName: config.tableName, + filename: config.filename, + fetchedRows: 0, + totalRows: 0, + region: config.region, + project: config.project, + databaseId: config.databaseId, + tableId: config.tableId, + columns: config.columns, + queries: config.queries, + wildcardQueries: config.wildcardQueries, + prettyPrint: config.prettyPrint + }; + + update((jobs) => { + jobs.set(jobId, job); + return new Map(jobs); + }); + + // Run the export in the background + runExport(jobId, config); + } + + async function runExport(jobId: string, config: JsonExportConfig) { + const PAGE_SIZE = 100; + const MAX_ROWS = 5000; + let offset = 0; + let allRows: Models.Row[] = []; + + try { + // Update to processing + update((jobs) => { + const job = jobs.get(jobId); + if (job) job.status = 'processing'; + return new Map(jobs); + }); + + // First fetch to get total + const firstBatch = await sdk + .forProject(config.region, config.project) + .tablesDB.listRows({ + databaseId: config.databaseId, + tableId: config.tableId, + queries: [ + Query.limit(PAGE_SIZE), + Query.offset(0), + ...config.queries, + ...config.wildcardQueries + ] + }); + + const total = firstBatch.total; + + if (total > MAX_ROWS) { + throw new Error( + `Table size (${total} rows) exceeds client-side export limit of ${MAX_ROWS}. Please apply filters to reduce the result set.` + ); + } + + allRows = [...firstBatch.rows]; + offset = PAGE_SIZE; + + update((jobs) => { + const job = jobs.get(jobId); + if (job) { + job.totalRows = total; + job.fetchedRows = allRows.length; + } + return new Map(jobs); + }); + + // Paginate remaining + while (allRows.length < total) { + const batch = await sdk + .forProject(config.region, config.project) + .tablesDB.listRows({ + databaseId: config.databaseId, + tableId: config.tableId, + queries: [ + Query.limit(PAGE_SIZE), + Query.offset(offset), + ...config.queries, + ...config.wildcardQueries + ] + }); + + // Guard against empty batches to prevent infinite loop + if (batch.rows.length === 0) { + break; + } + + allRows = [...allRows, ...batch.rows]; + offset += batch.rows.length; + + update((jobs) => { + const job = jobs.get(jobId); + if (job) { + job.fetchedRows = allRows.length; + } + return new Map(jobs); + }); + } + + // Filter to selected columns + const filteredRows = allRows.map((row) => { + const filtered: Record = {}; + for (const col of config.columns) { + if (col in row) { + filtered[col] = row[col]; + } + } + return filtered; + }); + + // Create and trigger download + const jsonString = config.prettyPrint + ? JSON.stringify(filteredRows, null, 2) + : JSON.stringify(filteredRows); + const blob = new Blob([jsonString], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = config.filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + + update((jobs) => { + const job = jobs.get(jobId); + if (job) { + job.status = 'completed'; + job.fetchedRows = filteredRows.length; + } + return new Map(jobs); + }); + } catch (error) { + update((jobs) => { + const job = jobs.get(jobId); + if (job) { + job.status = 'failed'; + job.error = error instanceof Error ? error.message : 'Export failed'; + } + return new Map(jobs); + }); + } + } + + function clear() { + set(new Map()); + } + + function removeJob(jobId: string) { + update((jobs) => { + jobs.delete(jobId); + return new Map(jobs); + }); + } + + return { + subscribe, + startExport, + clear, + removeJob + }; +} + +export const jsonExportStore = createJsonExportStore(); diff --git a/src/routes/(console)/project-[region]-[project]/+layout.svelte b/src/routes/(console)/project-[region]-[project]/+layout.svelte index ffe92bdca2..818811061b 100644 --- a/src/routes/(console)/project-[region]-[project]/+layout.svelte +++ b/src/routes/(console)/project-[region]-[project]/+layout.svelte @@ -1,5 +1,5 @@ @@ -221,7 +244,7 @@ Import CSV - + - - Export CSV - + + + { + hide(e); + trackEvent(Click.DatabaseExportCsv); + goto(getTableExportUrl()); + }}> + Export as CSV + + { + hide(e); + trackEvent(Click.DatabaseExportJson); + goto(getTableJsonExportUrl()); + }}> + Export as JSON + + + + + + + + + + + + {#each visibleColumns as column (column.key)} +
+ +
+ {/each} +
+ + {#if hasMoreColumns} +
+ +
+ {/if} + + + +
+ + + +
+ + + + Indicate if the JSON should be formatted for readability. + + +
+
+ + +
+ +
+ + {#if localTags.length > 0} + + { + removeLocalFilter(e.detail); + }} /> + + {/if} +
+
+
+ + + + + + + + + + +