From dfa3fdeb33a9dd38f15e6d0184891265fcd95dbc Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 9 May 2026 20:16:51 +0000 Subject: [PATCH 1/3] fix(csv-to-pg): preserve empty strings in text fields instead of converting to NULL The text type coercion was calling cleanseEmptyStrings() which treated empty strings as NULL tokens. This is incorrect for database export/re-import workflows where PostgreSQL distinguishes between '' and NULL. This caused the webauthn_settings migration to fail: the table has rp_id/rp_name as NOT NULL DEFAULT '', but the export converted '' to NULL, producing an INSERT that violates the NOT NULL constraint. The fix: for text fields, only treat actual null/undefined as NULL. Empty strings are preserved as valid text values. --- .../__snapshots__/export.test.ts.snap | 20 +++++++ packages/csv-to-pg/__tests__/export.test.ts | 54 +++++++++++++++++++ packages/csv-to-pg/src/parse.ts | 9 ++-- 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap b/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap index 715ddb30ab..004f9bf1d0 100644 --- a/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap +++ b/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap @@ -94,6 +94,26 @@ exports[`test case test case parser 1`] = ` ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', 'name here', 'description');" `; +exports[`test case text fields preserve empty strings instead of converting to NULL 1`] = ` +"INSERT INTO services_public.webauthn_settings ( + id, + database_id, + rp_id, + rp_name +) VALUES + ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '550e8400-e29b-41d4-a716-446655440000', '', '');" +`; + +exports[`test case text fields with null values produce NULL 1`] = ` +"INSERT INTO services_public.webauthn_settings ( + id, + database_id, + rp_id, + rp_name +) VALUES + ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '550e8400-e29b-41d4-a716-446655440000', NULL, NULL);" +`; + exports[`test case uuid[] arrays 1`] = ` "INSERT INTO metaschema_public.primary_key_constraint ( id, diff --git a/packages/csv-to-pg/__tests__/export.test.ts b/packages/csv-to-pg/__tests__/export.test.ts index 6366b59ac5..5507d5d262 100644 --- a/packages/csv-to-pg/__tests__/export.test.ts +++ b/packages/csv-to-pg/__tests__/export.test.ts @@ -276,6 +276,60 @@ it('empty array fields emit empty array literal', async () => { expect(sql).toMatchSnapshot(); }); +it('text fields preserve empty strings instead of converting to NULL', async () => { + const parser = new Parser({ + schema: 'services_public', + singleStmts: true, + table: 'webauthn_settings', + fields: { + id: 'uuid', + database_id: 'uuid', + rp_id: 'text', + rp_name: 'text' + } + }); + + const sql = await parser.parse([ + { + id: '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', + database_id: '550e8400-e29b-41d4-a716-446655440000', + rp_id: '', + rp_name: '' + } + ]); + + // Empty strings should be preserved as '' not converted to NULL + expect(sql).not.toContain('NULL'); + expect(sql).toMatchSnapshot(); +}); + +it('text fields with null values produce NULL', async () => { + const parser = new Parser({ + schema: 'services_public', + singleStmts: true, + table: 'webauthn_settings', + fields: { + id: 'uuid', + database_id: 'uuid', + rp_id: 'text', + rp_name: 'text' + } + }); + + const sql = await parser.parse([ + { + id: '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', + database_id: '550e8400-e29b-41d4-a716-446655440000', + rp_id: null, + rp_name: null + } + ]); + + // Actual null values should still produce NULL + expect(sql).toContain('NULL'); + expect(sql).toMatchSnapshot(); +}); + it('interval type with string value', async () => { const parser = new Parser({ schema: 'metaschema_modules_public', diff --git a/packages/csv-to-pg/src/parse.ts b/packages/csv-to-pg/src/parse.ts index 921ed04d0b..50510e6041 100644 --- a/packages/csv-to-pg/src/parse.ts +++ b/packages/csv-to-pg/src/parse.ts @@ -436,9 +436,12 @@ const getCoercionFunc = (type: string, from: string[], opts: FieldOptions, field case 'text': return (record: Record): Node => { const rawValue = record[from[0]]; - const value = parseFn(cleanseEmptyStrings(rawValue)); - if (isEmpty(value)) { - return makeNullOrThrow(fieldName, rawValue, type, required, 'value is empty or null'); + if (rawValue === null || rawValue === undefined) { + return makeNullOrThrow(fieldName, rawValue, type, required, 'value is null'); + } + const value = parseFn(rawValue); + if (value === null || value === undefined) { + return makeNullOrThrow(fieldName, rawValue, type, required, 'value is null'); } const val = nodes.aConst({ sval: ast.string({ sval: String(value) }) From 873d9ba8d20e086fe9a1db6a75af249e53293ace Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 9 May 2026 20:35:57 +0000 Subject: [PATCH 2/3] fix(csv-to-pg): add preserveEmptyStrings option for database export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of unconditionally changing the text coercion behavior, add a preserveEmptyStrings option that controls whether empty strings are treated as NULL tokens (the default CSV convention) or preserved as valid text values. - Default behavior unchanged: empty strings → NULL (CSV convention) - preserveEmptyStrings: true → empty strings preserved as '' - export-meta.ts sets preserveEmptyStrings: true since database data distinguishes between '' and NULL This fixes the webauthn_settings migration failure where rp_id/rp_name default to '' but the export converted them to NULL. --- .../__snapshots__/export.test.ts.snap | 14 ++++++-- packages/csv-to-pg/__tests__/export.test.ts | 35 +++++++++++++++++-- packages/csv-to-pg/src/parse.ts | 22 +++++++----- packages/csv-to-pg/src/parser.ts | 1 + pgpm/export/src/export-meta.ts | 3 +- 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap b/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap index 004f9bf1d0..fcc83fc97d 100644 --- a/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap +++ b/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap @@ -94,7 +94,17 @@ exports[`test case test case parser 1`] = ` ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', 'name here', 'description');" `; -exports[`test case text fields preserve empty strings instead of converting to NULL 1`] = ` +exports[`test case text fields convert empty strings to NULL by default 1`] = ` +"INSERT INTO services_public.webauthn_settings ( + id, + database_id, + rp_id, + rp_name +) VALUES + ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '550e8400-e29b-41d4-a716-446655440000', NULL, NULL);" +`; + +exports[`test case text fields preserve empty strings when preserveEmptyStrings is true 1`] = ` "INSERT INTO services_public.webauthn_settings ( id, database_id, @@ -104,7 +114,7 @@ exports[`test case text fields preserve empty strings instead of converting to N ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '550e8400-e29b-41d4-a716-446655440000', '', '');" `; -exports[`test case text fields with null values produce NULL 1`] = ` +exports[`test case text fields with null values produce NULL even with preserveEmptyStrings 1`] = ` "INSERT INTO services_public.webauthn_settings ( id, database_id, diff --git a/packages/csv-to-pg/__tests__/export.test.ts b/packages/csv-to-pg/__tests__/export.test.ts index 5507d5d262..3f6d919dc1 100644 --- a/packages/csv-to-pg/__tests__/export.test.ts +++ b/packages/csv-to-pg/__tests__/export.test.ts @@ -276,7 +276,7 @@ it('empty array fields emit empty array literal', async () => { expect(sql).toMatchSnapshot(); }); -it('text fields preserve empty strings instead of converting to NULL', async () => { +it('text fields convert empty strings to NULL by default', async () => { const parser = new Parser({ schema: 'services_public', singleStmts: true, @@ -298,16 +298,45 @@ it('text fields preserve empty strings instead of converting to NULL', async () } ]); - // Empty strings should be preserved as '' not converted to NULL + // Default behavior: empty strings are treated as NULL (CSV convention) + expect(sql).toContain('NULL'); + expect(sql).toMatchSnapshot(); +}); + +it('text fields preserve empty strings when preserveEmptyStrings is true', async () => { + const parser = new Parser({ + schema: 'services_public', + singleStmts: true, + table: 'webauthn_settings', + preserveEmptyStrings: true, + fields: { + id: 'uuid', + database_id: 'uuid', + rp_id: 'text', + rp_name: 'text' + } + }); + + const sql = await parser.parse([ + { + id: '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', + database_id: '550e8400-e29b-41d4-a716-446655440000', + rp_id: '', + rp_name: '' + } + ]); + + // With preserveEmptyStrings, empty strings are kept as '' not NULL expect(sql).not.toContain('NULL'); expect(sql).toMatchSnapshot(); }); -it('text fields with null values produce NULL', async () => { +it('text fields with null values produce NULL even with preserveEmptyStrings', async () => { const parser = new Parser({ schema: 'services_public', singleStmts: true, table: 'webauthn_settings', + preserveEmptyStrings: true, fields: { id: 'uuid', database_id: 'uuid', diff --git a/packages/csv-to-pg/src/parse.ts b/packages/csv-to-pg/src/parse.ts index 50510e6041..47ad8f62bb 100644 --- a/packages/csv-to-pg/src/parse.ts +++ b/packages/csv-to-pg/src/parse.ts @@ -239,7 +239,7 @@ const makeNullOrThrow = (fieldName: string, rawValue: unknown, type: string, req // type (int, text, etc) // from Array of keys that map to records found (e.g., ['lon', 'lat']) -const getCoercionFunc = (type: string, from: string[], opts: FieldOptions, fieldName: string): CoercionFunc => { +const getCoercionFunc = (type: string, from: string[], opts: FieldOptions, fieldName: string, globalOpts?: GlobalParseOptions): CoercionFunc => { const parseFn = opts.parse || identity; const required = opts.required || false; @@ -436,12 +436,10 @@ const getCoercionFunc = (type: string, from: string[], opts: FieldOptions, field case 'text': return (record: Record): Node => { const rawValue = record[from[0]]; - if (rawValue === null || rawValue === undefined) { - return makeNullOrThrow(fieldName, rawValue, type, required, 'value is null'); - } - const value = parseFn(rawValue); - if (value === null || value === undefined) { - return makeNullOrThrow(fieldName, rawValue, type, required, 'value is null'); + const cleansed = globalOpts?.preserveEmptyStrings ? rawValue : cleanseEmptyStrings(rawValue); + const value = parseFn(cleansed); + if (isEmpty(value)) { + return makeNullOrThrow(fieldName, rawValue, type, required, 'value is empty or null'); } const val = nodes.aConst({ sval: ast.string({ sval: String(value) }) @@ -528,8 +526,13 @@ interface ConfigFields { [key: string]: string | FieldOptions; } +export interface GlobalParseOptions { + preserveEmptyStrings?: boolean; +} + interface Config { fields: ConfigFields; + preserveEmptyStrings?: boolean; } interface TypesMap { @@ -537,6 +540,9 @@ interface TypesMap { } export const parseTypes = (config: Config): TypesMap => { + const globalOpts: GlobalParseOptions = { + preserveEmptyStrings: config.preserveEmptyStrings + }; return Object.entries(config.fields).reduce((m, v) => { let [key, value] = v; let type: string; @@ -555,7 +561,7 @@ export const parseTypes = (config: Config): TypesMap => { type = value.type!; from = getFromValue(value.from || key); } - m[key] = getCoercionFunc(type, from, value, key); + m[key] = getCoercionFunc(type, from, value, key, globalOpts); return m; }, {}); }; diff --git a/packages/csv-to-pg/src/parser.ts b/packages/csv-to-pg/src/parser.ts index d6e6a6a4f8..1ea08be799 100644 --- a/packages/csv-to-pg/src/parser.ts +++ b/packages/csv-to-pg/src/parser.ts @@ -15,6 +15,7 @@ interface ParserConfig { input?: string; debug?: boolean; fields: Record; + preserveEmptyStrings?: boolean; } interface CsvOptions { diff --git a/pgpm/export/src/export-meta.ts b/pgpm/export/src/export-meta.ts index 3949413f60..0b93d14537 100644 --- a/pgpm/export/src/export-meta.ts +++ b/pgpm/export/src/export-meta.ts @@ -96,7 +96,8 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams schema: tableConfig.schema, table: tableConfig.table, conflictDoNothing: tableConfig.conflictDoNothing, - fields: dynamicFields + fields: dynamicFields, + preserveEmptyStrings: true }); parsers[key] = parser; From aaa3f6c484ab828e6084b5228ac4f298ea310401 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 9 May 2026 21:22:38 +0000 Subject: [PATCH 3/3] fix: default preserveEmptyStrings to true Empty strings are valid text values in PostgreSQL and should be preserved by default. The setting can be set to false to opt into the CSV convention of treating empty strings as NULL. --- .../__tests__/__snapshots__/export.test.ts.snap | 6 +++--- packages/csv-to-pg/__tests__/export.test.ts | 17 ++++++++--------- packages/csv-to-pg/src/parse.ts | 3 ++- pgpm/export/src/export-meta.ts | 3 +-- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap b/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap index fcc83fc97d..11678d166a 100644 --- a/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap +++ b/packages/csv-to-pg/__tests__/__snapshots__/export.test.ts.snap @@ -94,7 +94,7 @@ exports[`test case test case parser 1`] = ` ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', 'name here', 'description');" `; -exports[`test case text fields convert empty strings to NULL by default 1`] = ` +exports[`test case text fields convert empty strings to NULL when preserveEmptyStrings is false 1`] = ` "INSERT INTO services_public.webauthn_settings ( id, database_id, @@ -104,7 +104,7 @@ exports[`test case text fields convert empty strings to NULL by default 1`] = ` ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '550e8400-e29b-41d4-a716-446655440000', NULL, NULL);" `; -exports[`test case text fields preserve empty strings when preserveEmptyStrings is true 1`] = ` +exports[`test case text fields preserve empty strings by default 1`] = ` "INSERT INTO services_public.webauthn_settings ( id, database_id, @@ -114,7 +114,7 @@ exports[`test case text fields preserve empty strings when preserveEmptyStrings ('450e3b3b-b68d-4abc-990c-65cb8a1dcdb4', '550e8400-e29b-41d4-a716-446655440000', '', '');" `; -exports[`test case text fields with null values produce NULL even with preserveEmptyStrings 1`] = ` +exports[`test case text fields with null values produce NULL 1`] = ` "INSERT INTO services_public.webauthn_settings ( id, database_id, diff --git a/packages/csv-to-pg/__tests__/export.test.ts b/packages/csv-to-pg/__tests__/export.test.ts index 3f6d919dc1..e9e037f58a 100644 --- a/packages/csv-to-pg/__tests__/export.test.ts +++ b/packages/csv-to-pg/__tests__/export.test.ts @@ -276,7 +276,7 @@ it('empty array fields emit empty array literal', async () => { expect(sql).toMatchSnapshot(); }); -it('text fields convert empty strings to NULL by default', async () => { +it('text fields preserve empty strings by default', async () => { const parser = new Parser({ schema: 'services_public', singleStmts: true, @@ -298,17 +298,17 @@ it('text fields convert empty strings to NULL by default', async () => { } ]); - // Default behavior: empty strings are treated as NULL (CSV convention) - expect(sql).toContain('NULL'); + // Default: empty strings are preserved as '' + expect(sql).not.toContain('NULL'); expect(sql).toMatchSnapshot(); }); -it('text fields preserve empty strings when preserveEmptyStrings is true', async () => { +it('text fields convert empty strings to NULL when preserveEmptyStrings is false', async () => { const parser = new Parser({ schema: 'services_public', singleStmts: true, table: 'webauthn_settings', - preserveEmptyStrings: true, + preserveEmptyStrings: false, fields: { id: 'uuid', database_id: 'uuid', @@ -326,17 +326,16 @@ it('text fields preserve empty strings when preserveEmptyStrings is true', async } ]); - // With preserveEmptyStrings, empty strings are kept as '' not NULL - expect(sql).not.toContain('NULL'); + // Opt-in: empty strings treated as NULL (CSV convention) + expect(sql).toContain('NULL'); expect(sql).toMatchSnapshot(); }); -it('text fields with null values produce NULL even with preserveEmptyStrings', async () => { +it('text fields with null values produce NULL', async () => { const parser = new Parser({ schema: 'services_public', singleStmts: true, table: 'webauthn_settings', - preserveEmptyStrings: true, fields: { id: 'uuid', database_id: 'uuid', diff --git a/packages/csv-to-pg/src/parse.ts b/packages/csv-to-pg/src/parse.ts index 47ad8f62bb..7e74e2bbf4 100644 --- a/packages/csv-to-pg/src/parse.ts +++ b/packages/csv-to-pg/src/parse.ts @@ -436,7 +436,8 @@ const getCoercionFunc = (type: string, from: string[], opts: FieldOptions, field case 'text': return (record: Record): Node => { const rawValue = record[from[0]]; - const cleansed = globalOpts?.preserveEmptyStrings ? rawValue : cleanseEmptyStrings(rawValue); + const preserve = globalOpts?.preserveEmptyStrings !== false; + const cleansed = preserve ? rawValue : cleanseEmptyStrings(rawValue); const value = parseFn(cleansed); if (isEmpty(value)) { return makeNullOrThrow(fieldName, rawValue, type, required, 'value is empty or null'); diff --git a/pgpm/export/src/export-meta.ts b/pgpm/export/src/export-meta.ts index 0b93d14537..3949413f60 100644 --- a/pgpm/export/src/export-meta.ts +++ b/pgpm/export/src/export-meta.ts @@ -96,8 +96,7 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams schema: tableConfig.schema, table: tableConfig.table, conflictDoNothing: tableConfig.conflictDoNothing, - fields: dynamicFields, - preserveEmptyStrings: true + fields: dynamicFields }); parsers[key] = parser;