diff --git a/package.json b/package.json index a706a40a..9e93991b 100644 --- a/package.json +++ b/package.json @@ -132,7 +132,8 @@ "csv-parse": "^5.6.0", "csv-stringify": "^6.6.0", "form-data": "^4.0.5", - "terminal-link": "^3.0.0" + "terminal-link": "^3.0.0", + "zod": "^4.2.1" }, "devDependencies": { "@oclif/core": "^4.8.0", diff --git a/schema/dataImportPlanSchema.json b/schema/dataImportPlanSchema.json deleted file mode 100644 index 066974d7..00000000 --- a/schema/dataImportPlanSchema.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "$comment": "Copyright (c) 2016, salesforce.com, inc. All rights reserved. Licensed under the BSD 3-Clause license. For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause", - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "array", - "title": "Data Import Plan", - "description": "Schema for data import plan JSON.", - "items": { - "type": "object", - "title": "SObject Type", - "description": "Definition of records to be insert per SObject Type", - "properties": { - "sobject": { - "type": "string", - "title": "Name of SObject", - "description": "Child file references must have SObject roots of this type" - }, - "saveRefs": { - "type": "boolean", - "title": "Save References", - "description": "Post-save, save references (Name/ID) to be used for reference replacement in subsequent saves. Applies to all data files for this SObject type.", - "default": false - }, - "resolveRefs": { - "type": "boolean", - "title": "Resolve References", - "description": "Pre-save, replace @ with ID from previous save. Applies to all data files for this SObject type.", - "default": false - }, - "files": { - "type": "array", - "title": "Files", - "description": "An array of files paths to load", - "items": { - "title": "Filepath.", - "description": "Filepath string or object to point to a JSON or XML file having data defined in SObject Tree format.", - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "file": { - "type": "string", - "title": "Filepath schema", - "description": "Filepath to JSON or XML file having data defined in SObject Tree format" - }, - "contentType": { - "title": "Filepath schema.", - "description": "If data file extension is not .json or .xml, provide content type.", - "enum": ["application/json", "application/xml"] - }, - "saveRefs": { - "type": "boolean", - "title": "Save References", - "description": "Post-save, save references (Name/ID) to be used for reference replacement in subsequent saves. Overrides SObject-level 'saveRefs' setting." - }, - "resolveRefs": { - "type": "boolean", - "title": "Resolve References", - "description": "Pre-save, replace @ with ID from previous save. Overrides SObject-level 'replaceRefs' setting." - } - }, - "required": ["file"] - } - ] - } - } - }, - "required": ["sobject", "files"] - } -} diff --git a/src/api/data/tree/importFiles.ts b/src/api/data/tree/importFiles.ts index e9638dd1..868fed42 100644 --- a/src/api/data/tree/importFiles.ts +++ b/src/api/data/tree/importFiles.ts @@ -25,7 +25,7 @@ import { getResultsIfNoError, transformRecordTypeEntries, } from './importCommon.js'; -import type { ImportResult, ResponseRefs, TreeResponse } from './importTypes.js'; +import type { ImportResult, ImportStatus, ResponseRefs, TreeResponse } from './importTypes.js'; import { hasUnresolvedRefs } from './functions.js'; export type FileInfo = { @@ -35,7 +35,7 @@ export type FileInfo = { sobject: string; }; -export const importFromFiles = async (conn: Connection, dataFilePaths: string[]): Promise => { +export const importFromFiles = async (conn: Connection, dataFilePaths: string[]): Promise => { const logger = Logger.childFromRoot('data:import:tree:importSObjectTreeFile'); const fileInfos = (await Promise.all(dataFilePaths.map(parseFile))).map(logFileInfo(logger)).map(validateNoRefs); await Promise.all(fileInfos.map(async (fi) => transformRecordTypeEntries(conn, fi.records))); @@ -43,7 +43,10 @@ export const importFromFiles = async (conn: Connection, dataFilePaths: string[]) const results = await Promise.allSettled( fileInfos.map((fi) => sendSObjectTreeRequest(conn)(fi.sobject)(JSON.stringify({ records: fi.records }))) ); - return results.map(getSuccessOrThrow).flatMap(getValueOrThrow(fileInfos)).map(addObjectTypes(refMap)); + return { + results: results.map(getSuccessOrThrow).flatMap(getValueOrThrow(fileInfos)).map(addObjectTypes(refMap)), + warnings: [], + }; }; const getSuccessOrThrow = (result: PromiseSettledResult): PromiseFulfilledResult => diff --git a/src/api/data/tree/importPlan.ts b/src/api/data/tree/importPlan.ts index 6b394414..253bfe09 100644 --- a/src/api/data/tree/importPlan.ts +++ b/src/api/data/tree/importPlan.ts @@ -15,14 +15,14 @@ */ import path from 'node:path'; import { EOL } from 'node:os'; -import { fileURLToPath } from 'node:url'; import fs from 'node:fs'; import { createHash } from 'node:crypto'; -import { AnyJson, isString } from '@salesforce/ts-types'; -import { Logger, SchemaValidator, SfError, Connection, Messages } from '@salesforce/core'; -import type { GenericObject, SObjectTreeInput } from '../../../types.js'; -import type { DataPlanPartFilesOnly, ImportResult } from './importTypes.js'; +import { isString } from '@salesforce/ts-types'; +import { Logger, Connection, Messages } from '@salesforce/core'; +import type { DataPlanPart, GenericObject, SObjectTreeInput } from '../../../types.js'; +import { DataImportPlanArraySchema, DataImportPlanArray } from '../../../schema/dataImportPlan.js'; +import type { ImportResult, ImportStatus } from './importTypes.js'; import { getResultsIfNoError, parseDataFileContents, @@ -37,7 +37,7 @@ Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-data', 'importApi'); // the "new" type for these. We're ignoring saveRefs/resolveRefs -export type EnrichedPlanPart = Omit & { +export type EnrichedPlanPart = Partial & { filePath: string; sobject: string; records: SObjectTreeInput[]; @@ -51,19 +51,17 @@ type ResultsSoFar = { const TREE_API_LIMIT = 200; const refRegex = (object: string): RegExp => new RegExp(`^@${object}Ref\\d+$`); -export const importFromPlan = async (conn: Connection, planFilePath: string): Promise => { +export const importFromPlan = async (conn: Connection, planFilePath: string): Promise => { const resolvedPlanPath = path.resolve(process.cwd(), planFilePath); const logger = Logger.childFromRoot('data:import:tree:importFromPlan'); - + const warnings: string[] = []; + const planResultObj = validatePlanContents(resolvedPlanPath, JSON.parse(fs.readFileSync(resolvedPlanPath, 'utf-8'))); + warnings.push(...planResultObj.warnings); const planContents = await Promise.all( - ( - await validatePlanContents(logger)( - resolvedPlanPath, - (await JSON.parse(fs.readFileSync(resolvedPlanPath, 'utf8'))) as DataPlanPartFilesOnly[] + planResultObj.parsedPlans + .flatMap((planPart) => + planPart.files.map((f) => ({ ...planPart, filePath: path.resolve(path.dirname(resolvedPlanPath), f) })) ) - ) - // there *shouldn't* be multiple files for the same sobject in a plan, but the legacy code allows that - .flatMap((dpp) => dpp.files.map((f) => ({ ...dpp, filePath: path.resolve(path.dirname(resolvedPlanPath), f) }))) .map(async (i) => ({ ...i, records: parseDataFileContents(i.filePath)(await fs.promises.readFile(i.filePath, 'utf-8')), @@ -72,7 +70,7 @@ export const importFromPlan = async (conn: Connection, planFilePath: string): Pr // using recursion to sequentially send the requests so we get refs back from each round const { results } = await getResults(conn)(logger)({ results: [], fingerprints: new Set() })(planContents); - return results; + return { results, warnings }; }; /** recursively splits files (for size or unresolved refs) and makes API calls, storing results for subsequent calls */ @@ -189,54 +187,37 @@ const replaceRefWithId = Object.entries(record).map(([k, v]) => [k, v === `@${ref.refId}` ? ref.id : v]) ) as SObjectTreeInput; -export const validatePlanContents = - (logger: Logger) => - async (planPath: string, planContents: unknown): Promise => { - const childLogger = logger.child('validatePlanContents'); - const planSchema = path.join( - path.dirname(fileURLToPath(import.meta.url)), - '..', - '..', - '..', - '..', - 'schema', - 'dataImportPlanSchema.json' - ); - - const val = new SchemaValidator(childLogger, planSchema); - try { - await val.validate(planContents as AnyJson); - const output = planContents as DataPlanPartFilesOnly[]; - if (hasRefs(output)) { - childLogger.warn( - "The plan contains the 'saveRefs' and/or 'resolveRefs' properties. These properties will be ignored and can be removed." - ); - } - if (!hasOnlySimpleFiles(output)) { - throw messages.createError('error.NonStringFiles'); - } - return planContents as DataPlanPartFilesOnly[]; - } catch (err) { - if (err instanceof Error && err.name === 'ValidationSchemaFieldError') { - throw messages.createError('error.InvalidDataImport', [planPath, err.message]); - } else if (err instanceof Error) { - throw SfError.wrap(err); - } - throw err; +export function validatePlanContents( + planPath: string, + planContents: unknown +): { parsedPlans: DataImportPlanArray; warnings: string[] } { + const warnings: string[] = []; + const parseResults = DataImportPlanArraySchema.safeParse(planContents); + + if (parseResults.error) { + throw messages.createError('error.InvalidDataImport', [ + planPath, + parseResults.error.issues.map((e) => e.message).join('\n'), + ]); + } + const parsedPlans: DataImportPlanArray = parseResults.data; + + for (const parsedPlan of parsedPlans) { + if (parsedPlan.saveRefs !== undefined || parsedPlan.resolveRefs !== undefined) { + warnings.push( + "The plan contains the 'saveRefs' and/or 'resolveRefs' properties. These properties will be ignored and can be removed." + ); + break; } - }; + } + return { parsedPlans, warnings }; +} const matchesRefFilter = (unresolvedRefRegex: RegExp) => (v: unknown): boolean => typeof v === 'string' && unresolvedRefRegex.test(v); -const hasOnlySimpleFiles = (planParts: DataPlanPartFilesOnly[]): boolean => - planParts.every((p) => p.files.every((f) => typeof f === 'string')); - -const hasRefs = (planParts: DataPlanPartFilesOnly[]): boolean => - planParts.some((p) => p.saveRefs !== undefined || p.resolveRefs !== undefined); - // TODO: change this implementation to use Object.groupBy when it's on all supported node versions const filterUnresolved = ( records: SObjectTreeInput[] diff --git a/src/api/data/tree/importTypes.ts b/src/api/data/tree/importTypes.ts index 70aec7e7..6a988fc9 100644 --- a/src/api/data/tree/importTypes.ts +++ b/src/api/data/tree/importTypes.ts @@ -13,15 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import type { Dictionary } from '@salesforce/ts-types'; -import type { DataPlanPart } from '../../../types.js'; - -/* - * Copyright (c) 2023, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ export type TreeResponse = TreeResponseSuccess | TreeResponseError; type TreeResponseSuccess = { @@ -48,10 +39,10 @@ export type ResponseRefs = { referenceId: string; id: string; }; -export type ImportResults = { - responseRefs?: ResponseRefs[]; - sobjectTypes?: Dictionary; - errors?: string[]; + +export type ImportStatus = { + results: ImportResult[]; + warnings: string[]; }; export type ImportResult = { @@ -59,10 +50,3 @@ export type ImportResult = { type: string; id: string; }; /** like the original DataPlanPart but without the non-string options inside files */ - -export type DataPlanPartFilesOnly = { - sobject: string; - files: string[]; - saveRefs: boolean; - resolveRefs: boolean; -} & Partial; diff --git a/src/commands/data/import/tree.ts b/src/commands/data/import/tree.ts index ac2551df..df82a142 100644 --- a/src/commands/data/import/tree.ts +++ b/src/commands/data/import/tree.ts @@ -61,9 +61,12 @@ export default class Import extends SfCommand { const conn = flags['target-org'].getConnection(flags['api-version']); try { - const results = flags.plan + const { results, warnings } = flags.plan ? await importFromPlan(conn, flags.plan) : await importFromFiles(conn, flags.files ?? []); + for (const warning of warnings) { + this.warn(warning); + } this.table({ data: results, diff --git a/src/schema/dataImportPlan.ts b/src/schema/dataImportPlan.ts new file mode 100644 index 00000000..7a7aa9f3 --- /dev/null +++ b/src/schema/dataImportPlan.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2025, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { z } from 'zod'; + +export const DataImportPlanArraySchema = z + .array( + z.object({ + sobject: z.string().describe('Child file references must have SObject roots of this type'), + saveRefs: z + .boolean() + .optional() + .describe( + 'Post-save, save references (Name/ID) to be used for reference replacement in subsequent saves. Applies to all data files for this SObject type.' + ), + resolveRefs: z + .boolean() + .optional() + .describe( + 'Pre-save, replace @ with ID from previous save. Applies to all data files for this SObject type.' + ), + files: z + .array( + z + .string('The `files` property of the plan objects must contain only strings') + .describe( + 'Filepath string or object to point to a JSON or XML file having data defined in SObject Tree format.' + ) + ) + .describe('An array of files paths to load'), + }) + ) + .describe('Schema for data import plan JSON'); + +export type DataImportPlanArray = z.infer; diff --git a/test/api/data/tree/importPlan.test.ts b/test/api/data/tree/importPlan.test.ts index 3ff25e4b..3a8daefd 100644 --- a/test/api/data/tree/importPlan.test.ts +++ b/test/api/data/tree/importPlan.test.ts @@ -16,8 +16,7 @@ /* eslint-disable camelcase */ // for salesforce __c style fields import { expect, assert } from 'chai'; -import { shouldThrow } from '@salesforce/core/testSetup'; -import { Logger } from '@salesforce/core'; +import { shouldThrowSync } from '@salesforce/core/testSetup'; import { replaceRefsInTheSameFile, EnrichedPlanPart, @@ -114,18 +113,7 @@ describe('importPlan', () => { }); }); describe('plan validation', () => { - // ensure no static rootLogger - // @ts-expect-error private stuff - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - Logger.rootLogger = undefined; - const logger = new Logger({ name: 'importPlanTest', useMemoryLogger: true }); - afterEach(() => { - // @ts-expect-error private stuff - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - logger.memoryLogger.loggedData = []; - }); - const validator = validatePlanContents(logger); - it('good plan in classic format', async () => { + it('good plan in classic format, one file', () => { const plan = [ { sobject: 'Account', @@ -134,10 +122,14 @@ describe('importPlan', () => { files: ['Account.json'], }, ]; - expect(await validator('some/path', plan)).to.equal(plan); - expect(getLogMessages(logger)).to.include('saveRefs'); + + const { parsedPlans, warnings } = validatePlanContents('some/path', plan); + expect(parsedPlans).to.deep.equal(plan); + expect(warnings).to.be.length(1); + expect(warnings[0]).to.include('saveRefs'); }); - it('good plan in classic format', async () => { + + it('good plan in classic format, multiple files', () => { const plan = [ { sobject: 'Account', @@ -146,9 +138,14 @@ describe('importPlan', () => { files: ['Account.json', 'Account2.json'], }, ]; - expect(await validator('some/path', plan)).to.equal(plan); + + const { parsedPlans, warnings } = validatePlanContents('some/path', plan); + expect(parsedPlans).to.deep.equal(plan); + expect(warnings).to.be.length(1); + expect(warnings[0]).to.include('saveRefs'); }); - it('throws on bad plan (missing the object)', async () => { + + it('throws on bad plan (missing sobject property)', () => { const plan = [ { saveRefs: true, @@ -156,15 +153,15 @@ describe('importPlan', () => { files: ['Account.json', 'Account2.json'], }, ]; + try { - await shouldThrow(validator('some/path', plan)); + shouldThrowSync(() => validatePlanContents('some/path', plan)); } catch (e) { assert(e instanceof Error); expect(e.name).to.equal('InvalidDataImportError'); } }); - // TODO: remove this test when schema moves to simple files only - it('throws when files are an object that meets current schema', async () => { + it('throws when files property contains non-strings', () => { const plan = [ { sobject: 'Account', @@ -174,27 +171,22 @@ describe('importPlan', () => { }, ]; try { - await shouldThrow(validator('some/path', plan)); + shouldThrowSync(() => validatePlanContents('some/plan', plan)); } catch (e) { assert(e instanceof Error); - expect(e.name, JSON.stringify(e)).to.equal('NonStringFilesError'); + expect(e.message).to.include('The `files` property of the plan objects must contain only strings'); } }); - it('good plan in new format is valid and produces no warnings', async () => { + it('good plan in new format', () => { const plan = [ { sobject: 'Account', files: ['Account.json'], }, ]; - expect(await validator('some/path', plan)).to.equal(plan); - expect(getLogMessages(logger)).to.not.include('saveRefs'); + const { parsedPlans, warnings } = validatePlanContents('some/path', plan); + expect(parsedPlans).to.deep.equal(plan); + expect(warnings).to.be.length(0); }); }); }); - -const getLogMessages = (logger: Logger): string => - logger - .getBufferedRecords() - .map((i) => i.msg) - .join('/n'); diff --git a/yarn.lock b/yarn.lock index 080e6888..04bd457f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7922,3 +7922,8 @@ zod@^4.1.12: version "4.1.13" resolved "https://registry.yarnpkg.com/zod/-/zod-4.1.13.tgz#93699a8afe937ba96badbb0ce8be6033c0a4b6b1" integrity sha512-AvvthqfqrAhNH9dnfmrfKzX5upOdjUVJYFqNSlkmGf64gRaTzlPwz99IHYnVs28qYAybvAlBV+H7pn0saFY4Ig== + +zod@^4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/zod/-/zod-4.2.1.tgz#07f0388c7edbfd5f5a2466181cb4adf5b5dbd57b" + integrity sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw==