chore: updated implementation for AM import

Author: naman-contentstack

.talismanrc

@@ -1,18 +1,12 @@
 fileignoreconfig:
-  - filename: pnpm-lock.yaml
-    checksum: 9c19eb613068c193fac35e72327198fbc86e759968391f07cc876c56b2b1a63d
-  - filename: packages/contentstack-export/test/unit/export/modules/base-class.test.ts
-    checksum: bd2b28305fff90ca26bce56b2c5c61751a62225d310a2553874e9ec009ed78e8
-  - filename: packages/contentstack-export/test/unit/export/modules/assets.test.ts
-    checksum: 73ff01e2d19c8d1384dca2ee7087f8c19e0b1fac6b29c75a02ca523a36b7cb92
-  - filename: packages/contentstack-export/src/types/default-config.ts
-    checksum: a204b00fc47046fd638f952c1326b6d88615dd96e37dc83e4c7e9404dc0435bb
-  - filename: packages/contentstack-export/src/types/index.ts
-    checksum: fa36c236abac338b03bf307102a99f25dddac9afe75b6b34fb82e318e7759799
-  - filename: packages/contentstack-export/src/config/index.ts
-    checksum: 3998e30abaf9838f86025c3422cab085441bc38e958ed9e63084f928dbb7995c
-  - filename: packages/contentstack-export/test/unit/export/modules/stack.test.ts
-    checksum: 79876b8f635037a2d8ba38dac055e7625bf85db6a3cf5729434e6a97e44857d6
-  - filename: packages/contentstack-export/src/export/modules/stack.ts
-    checksum: 375c0c5f58d43430b355050d122d3283083ca91891abe8105a4b4fd9433ece97
+  - filename: packages/contentstack-asset-management/src/export/base.ts
+    checksum: 01cb2158fea9cbe05449c04efb42d8d416d24868d411bd7300d8fa99c9a4ab01
+  - filename: packages/contentstack-asset-management/test/unit/export/base.test.ts
+    checksum: 164fc2e5a4337a2739903499b66eecc66a85bb9b50aa2e71079bdd046a195a94
+  - filename: packages/contentstack-asset-management/src/import/fields.ts
+    checksum: 1cd52254ddbfd186d8ade2c73fc799dd1caa0f10bdd3c6b151621c27207ee173
+  - filename: packages/contentstack-import/src/utils/asset-management-import-options.ts
+    checksum: 96b61b683109be2cc2dbab5231e1ded19282fbf176cd8492c75cc4861f2efea0
+  - filename: packages/contentstack-asset-management/src/import/asset-types.ts
+    checksum: fa2aeea704fd259628f9c86eacd3cf9f6f12543b06b387bd65db7df6b6f5fc49
 version: '1.0'

packages/contentstack-asset-management/package.json

@@ -29,7 +29,7 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "@contentstack/cli-utilities": "~2.0.0-beta"
+    "@contentstack/cli-utilities": "~2.0.0-beta.5"
   },
   "oclif": {
     "commands": "./lib/commands",

packages/contentstack-asset-management/src/constants/index.ts

@@ -1,5 +1,3 @@
-/** Fallback when export/import do not pass `chunkWriteBatchSize`. */
-export const FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE = 50;
 /** Fallback when export/import do not pass `chunkFileSizeMb`. */
 export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1;
 /** Fallback when import does not pass `apiConcurrency`. */
@@ -27,8 +25,6 @@ export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [
   'category_detail',
 ] as const;
 
-/** @deprecated Use FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE */
-export const BATCH_SIZE = FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE;
 /** @deprecated Use FALLBACK_AM_CHUNK_FILE_SIZE_MB */
 export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB;
 

packages/contentstack-asset-management/src/export/base.ts

@@ -5,11 +5,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack
 import type { AssetManagementAPIConfig } from '../types/asset-management-api';
 import type { ExportContext } from '../types/export-types';
 import { AssetManagementAdapter } from '../utils/asset-management-api-adapter';
-import {
-  AM_MAIN_PROCESS_NAME,
-  FALLBACK_AM_CHUNK_FILE_SIZE_MB,
-  FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE,
-} from '../constants/index';
+import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index';
 
 export type { ExportContext };
 
@@ -87,7 +83,6 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
       return;
     }
     const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB;
-    const batchSize = this.exportContext.chunkWriteBatchSize ?? FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE;
     const fs = new FsUtility({
       basePath: dir,
       indexFileName,
@@ -97,10 +92,7 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
       metaPickKeys,
       keepMetadata: true,
     });
-    for (let i = 0; i < items.length; i += batchSize) {
-      const batch = items.slice(i, i + batchSize);
-      fs.writeIntoFile(batch as Record<string, string>[], { mapKeyVal: true });
-    }
+    fs.writeIntoFile(items as Record<string, string>[], { mapKeyVal: true });
     fs.completeFile(true);
   }
 }

packages/contentstack-asset-management/src/export/spaces.ts

@@ -38,7 +38,6 @@ export class ExportSpaces {
       apiKey,
       context,
       securedAssets,
-      chunkWriteBatchSize,
       chunkFileSizeMb,
     } = this.options;
 
@@ -70,7 +69,6 @@ export class ExportSpaces {
       spacesRootPath,
       context,
       securedAssets,
-      chunkWriteBatchSize,
       chunkFileSizeMb,
     };
 

packages/contentstack-asset-management/src/import/asset-types.ts

@@ -1,3 +1,5 @@
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
 import omit from 'lodash/omit';
 import isEqual from 'lodash/isEqual';
 import { log } from '@contentstack/cli-utilities';
@@ -6,7 +8,9 @@ import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-man
 import { AssetManagementImportAdapter } from './base';
 import { FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
 import { runInBatches } from '../utils/concurrent-batch';
-import { readChunkedJsonItems } from '../utils/chunked-json-read';
+import { forEachChunkedJsonStore } from '../utils/chunked-json-reader';
+
+type AssetTypeToCreate = { uid: string; payload: Record<string, unknown> };
 
 /**
  * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs
@@ -30,15 +34,37 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter {
     const stripKeys = this.importContext.assetTypesImportInvalidKeys ?? [...FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS];
     const dir = this.getAssetTypesDir();
     const indexName = this.importContext.assetTypesFileName ?? 'asset-types.json';
-    const items = await readChunkedJsonItems<Record<string, unknown>>(dir, indexName, this.importContext.context);
+    const indexPath = join(dir, indexName);
 
-    if (items.length === 0) {
-      log.debug('No shared asset types to import', this.importContext.context);
+    if (!existsSync(indexPath)) {
+      log.debug('No shared asset types to import (index missing)', this.importContext.context);
       return;
     }
 
-    // Fetch existing asset types from the target org keyed by uid for diff comparison.
-    // Asset types are org-level; the spaceUid param in getWorkspaceAssetTypes is unused in the path.
+    const existingByUid = await this.loadExistingAssetTypesMap();
+
+    this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES);
+
+    await forEachChunkedJsonStore<Record<string, unknown>>(
+      dir,
+      indexName,
+      {
+        context: this.importContext.context,
+        chunkReadLogLabel: 'asset-types',
+        onOpenError: (e) =>
+          log.debug(`Could not open chunked asset-types index: ${e}`, this.importContext.context),
+        onEmptyIndexer: () =>
+          log.debug('No shared asset types to import (empty indexer)', this.importContext.context),
+      },
+      async (records) => {
+        const toCreate = this.buildAssetTypesToCreate(records, existingByUid, stripKeys);
+        await this.importAssetTypesCreates(toCreate);
+      },
+    );
+  }
+
+  /** Org-level asset types keyed by uid for diff; empty map if list API fails. */
+  private async loadExistingAssetTypesMap(): Promise<Map<string, Record<string, unknown>>> {
     const existingByUid = new Map<string, Record<string, unknown>>();
     try {
       const existing = await this.getWorkspaceAssetTypes('');
@@ -49,11 +75,15 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter {
     } catch (e) {
       log.debug(`Could not fetch existing asset types, will attempt to create all: ${e}`, this.importContext.context);
     }
+    return existingByUid;
+  }
 
-    this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES);
-
-    type ToCreate = { uid: string; payload: Record<string, unknown> };
-    const toCreate: ToCreate[] = [];
+  private buildAssetTypesToCreate(
+    items: Record<string, unknown>[],
+    existingByUid: Map<string, Record<string, unknown>>,
+    stripKeys: string[],
+  ): AssetTypeToCreate[] {
+    const toCreate: AssetTypeToCreate[] = [];
 
     for (const assetType of items) {
       const uid = assetType.uid as string;
@@ -82,6 +112,10 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter {
       toCreate.push({ uid, payload: omit(assetType, stripKeys) as Record<string, unknown> });
     }
 
+    return toCreate;
+  }
+
+  private async importAssetTypesCreates(toCreate: AssetTypeToCreate[]): Promise<void> {
     await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => {
       try {
         await this.createAssetType(payload as any);