Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .talismanrc
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ fileignoreconfig:
- filename: packages/contentstack-query-export/.env-example
checksum: 922c7aa9c788ab60b987de2b0a2aee6d90843c463a8bbc29201e4efe31081187
- filename: pnpm-lock.yaml
checksum: bb5303f2fe64f90ae95d2738363267fb0bfcfeb71f025c2110d4cec87ff84d95
checksum: 3d2eaabf1df366efee1759156465c6aefa68f30d372717de2cdc3e41946aa3d8
- filename: packages/contentstack-import/src/utils/build-import-spaces-options.ts
checksum: fe0cb6cb5903515982af1e3642f2a19233207d35f13dc205cebeda0aa399f8b5
- filename: packages/contentstack-export/src/export/modules/stack.ts
Expand Down
49 changes: 0 additions & 49 deletions packages/contentstack-asset-management/README.md

This file was deleted.

44 changes: 43 additions & 1 deletion packages/contentstack-asset-management/src/constants/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,17 @@ export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB;
export const AM_MAIN_PROCESS_NAME = 'Asset Management 2.0';

/**
* Process names for Asset Management 2.0 export progress (for tick labels).
* Process names for Asset Management 2.0 export/import progress.
*
* In the new per-space layout each entry below corresponds to a single row in
* the multibar:
* - {@link AM_FIELDS} / {@link AM_ASSET_TYPES} are the shared bootstrap rows
* (one execution per org, ahead of per-space work).
* - {@link AM_IMPORT_FIELDS} / {@link AM_IMPORT_ASSET_TYPES} are the import
* equivalents.
* - One additional row per space is added dynamically via
* {@link getSpaceProcessName} and ticks include folders + metadata + asset
* transfer for that space.
*/
export const PROCESS_NAMES = {
AM_SPACE_METADATA: 'Space metadata',
Expand All @@ -51,6 +61,38 @@ export const PROCESS_NAMES = {
AM_IMPORT_ASSETS: 'Import assets',
} as const;

/**
* Maximum visual length of a per-space process row label. The CLIProgressManager
* truncates anything over 20 characters; reserve 6 chars for the `Space ` prefix
* so the trailing space uid keeps 14 chars before truncation.
*/
const SPACE_PROCESS_NAME_PREFIX = 'Space ';
const SPACE_PROCESS_NAME_MAX_UID_LEN = 14;

/**
* Returns the multibar row label for a single AM 2.0 space.
* The label is bounded so CLIProgressManager.formatProcessName doesn't truncate
* it mid-string; the full uid is still used for tick item labels and structured
* logs, only the row label itself is shortened for display.
*/
export function getSpaceProcessName(spaceUid: string): string {
const safeUid = spaceUid ?? '';
const trimmed =
safeUid.length > SPACE_PROCESS_NAME_MAX_UID_LEN
? safeUid.substring(0, SPACE_PROCESS_NAME_MAX_UID_LEN)
: safeUid;
return `${SPACE_PROCESS_NAME_PREFIX}${trimmed}`;
}

/**
* Detects whether a process name belongs to a per-space progress row, used by
* the export/import strategy registries to aggregate counts for the final
* summary across all spaces.
*/
export function isSpaceProcessName(processName: string): boolean {
return typeof processName === 'string' && processName.startsWith(SPACE_PROCESS_NAME_PREFIX);
}

/**
* Status messages for each process (exporting, fetching, importing, failed).
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { getArrayFromResponse } from '../utils/export-helpers';
import { PROCESS_NAMES } from '../constants/index';

export default class ExportAssetTypes extends AssetManagementExportAdapter {
protected processName: string = PROCESS_NAMES.AM_ASSET_TYPES;

constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
super(apiConfig, exportContext);
}
Expand All @@ -24,7 +26,13 @@ export default class ExportAssetTypes extends AssetManagementExportAdapter {
} else {
log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context);
}
await this.writeItemsToChunkedJson(dir, 'asset-types.json', 'asset_types', ['uid', 'title', 'category', 'file_extension'], items);
this.tick(true, PROCESS_NAMES.AM_ASSET_TYPES, null);
await this.writeItemsToChunkedJson(
dir,
'asset-types.json',
'asset_types',
['uid', 'title', 'category', 'file_extension'],
items,
);
this.tick(true, `asset_types (${items.length})`, null);
}
}
23 changes: 14 additions & 9 deletions packages/contentstack-asset-management/src/export/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,17 @@ export default class ExportAssets extends AssetManagementExportAdapter {
this.getWorkspaceAssets(workspace.space_uid, workspace.uid),
]);

const assetItems = getAssetItems(assetsData);
const downloadableCount = assetItems.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))).length;
// Per-space total: 1 folder write + 1 metadata write + N per-asset downloads.
// The shared module-level total is just a placeholder before this point; update
// it now so the multibar row shows real progress as downloads tick in.
this.progressOrParent?.updateProcessTotal?.(this.processName, 2 + downloadableCount);

await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2));
this.tick(true, `folders: ${workspace.space_uid}`, null);
log.debug(`Wrote folders.json for space ${workspace.space_uid}`, this.exportContext.context);

const assetItems = getAssetItems(assetsData);
log.debug(
assetItems.length === 0
? `No assets for space ${workspace.space_uid}, wrote empty assets.json`
Expand All @@ -60,7 +66,7 @@ export default class ExportAssets extends AssetManagementExportAdapter {
: `Wrote ${assetItems.length} asset metadata record(s) for space ${workspace.space_uid}`,
this.exportContext.context,
);
this.tick(true, `assets: ${workspace.space_uid} (${assetItems.length})`, null);
this.tick(true, `metadata: ${workspace.space_uid} (${assetItems.length})`, null);

log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
await this.downloadWorkspaceAssets(assetsData, assetsDir, workspace.space_uid);
Expand All @@ -87,8 +93,6 @@ export default class ExportAssets extends AssetManagementExportAdapter {
`Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}`,
this.exportContext.context,
);
let lastError: string | null = null;
let allSuccess = true;
let downloadOk = 0;
let downloadFail = 0;

Expand Down Expand Up @@ -118,24 +122,25 @@ export default class ExportAssets extends AssetManagementExportAdapter {
const filePath = pResolve(assetFolderPath, filename);
await writeStreamToFile(nodeStream, filePath);
downloadOk += 1;
// Per-asset tick so the per-space progress bar moves in real time.
this.tick(true, `asset: ${filename}`, null);
log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context);
} catch (e) {
allSuccess = false;
downloadFail += 1;
lastError = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
this.tick(false, `asset: ${filename}`, err);
log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context);
}
});

this.tick(allSuccess, `downloads: ${spaceUid}`, lastError);
log.info(
allSuccess
downloadFail === 0
? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}`
: `Asset downloads for space ${spaceUid} completed with errors: ${downloadOk} succeeded, ${downloadFail} failed`,
this.exportContext.context,
);
log.debug(
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, allSuccess=${allSuccess}`,
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}`,
this.exportContext.context,
);
}
Expand Down
11 changes: 10 additions & 1 deletion packages/contentstack-asset-management/src/export/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
protected readonly exportContext: ExportContext;
protected progressManager: CLIProgressManager | null = null;
protected parentProgressManager: CLIProgressManager | null = null;
protected readonly processName: string = AM_MAIN_PROCESS_NAME;
protected processName: string = AM_MAIN_PROCESS_NAME;

constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
super(apiConfig);
Expand All @@ -30,6 +30,15 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
this.parentProgressManager = parent;
}

/**
* Override the default progress process name for {@link tick}/{@link updateStatus}
* calls. Used by the per-space orchestrator so each module's ticks land on the
* row for the space currently being exported.
*/
public setProcessName(name: string): void {
this.processName = name;
}

protected get progressOrParent(): CLIProgressManager | null {
return this.parentProgressManager ?? this.progressManager;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { getArrayFromResponse } from '../utils/export-helpers';
import { PROCESS_NAMES } from '../constants/index';

export default class ExportFields extends AssetManagementExportAdapter {
protected processName: string = PROCESS_NAMES.AM_FIELDS;

constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
super(apiConfig, exportContext);
}
Expand All @@ -25,6 +27,6 @@ export default class ExportFields extends AssetManagementExportAdapter {
log.debug(`Writing ${items.length} shared fields`, this.exportContext.context);
}
await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items);
this.tick(true, PROCESS_NAMES.AM_FIELDS, null);
this.tick(true, `fields (${items.length})`, null);
}
}
71 changes: 52 additions & 19 deletions packages/contentstack-asset-management/src/export/spaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@ import { log, CLIProgressManager, configHandler, handleAndLogError } from '@cont

import type { AssetManagementExportOptions, AssetManagementAPIConfig } from '../types/asset-management-api';
import type { ExportContext } from '../types/export-types';
import { AssetManagementAdapter } from '../utils/asset-management-api-adapter';
import { AM_MAIN_PROCESS_NAME, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
import { AM_MAIN_PROCESS_NAME, PROCESS_NAMES, getSpaceProcessName } from '../constants/index';
import ExportAssetTypes from './asset-types';
import ExportFields from './fields';
import ExportWorkspace from './workspaces';
Expand Down Expand Up @@ -55,12 +54,18 @@ export class ExportSpaces {
await mkdir(spacesRootPath, { recursive: true });
log.debug(`Spaces root path: ${spacesRootPath}`, context);

const totalSteps = 2 + linkedWorkspaces.length * 4;
const progress = this.createProgress();
progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps);
progress
.startProcess(AM_MAIN_PROCESS_NAME)
.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME);
// Multibar layout: two shared bootstrap rows + one row per space. Per-space
// totals start at 1 and are bumped to (2 + downloadableCount) inside
// ExportAssets.start once we know the asset count for that space.
progress.addProcess(PROCESS_NAMES.AM_FIELDS, 1);
progress.addProcess(PROCESS_NAMES.AM_ASSET_TYPES, 1);
const spaceProcessNames = new Map<string, string>();
for (const ws of linkedWorkspaces) {
const spaceProcess = getSpaceProcessName(ws.space_uid);
spaceProcessNames.set(ws.space_uid, spaceProcess);
progress.addProcess(spaceProcess, 1);
}

const apiConfig: AssetManagementAPIConfig = {
baseURL: assetManagementUrl,
Expand All @@ -82,39 +87,67 @@ export class ExportSpaces {
await mkdir(sharedAssetTypesDir, { recursive: true });

const firstSpaceUid = linkedWorkspaces[0].space_uid;
let bootstrapFailed = false;
let anySpaceFailed = false;
try {
progress.startProcess(PROCESS_NAMES.AM_FIELDS);
progress.startProcess(PROCESS_NAMES.AM_ASSET_TYPES);

const exportAssetTypes = new ExportAssetTypes(apiConfig, exportContext);
exportAssetTypes.setParentProgressManager(progress);
const exportFields = new ExportFields(apiConfig, exportContext);
exportFields.setParentProgressManager(progress);
await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]);
try {
await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]);
progress.completeProcess(PROCESS_NAMES.AM_FIELDS, true);
progress.completeProcess(PROCESS_NAMES.AM_ASSET_TYPES, true);
} catch (bootstrapErr) {
bootstrapFailed = true;
progress.completeProcess(PROCESS_NAMES.AM_FIELDS, false);
progress.completeProcess(PROCESS_NAMES.AM_ASSET_TYPES, false);
throw bootstrapErr;
}

for (const ws of linkedWorkspaces) {
progress.updateStatus(`Exporting space: ${ws.space_uid}...`, AM_MAIN_PROCESS_NAME);
const spaceProcess = spaceProcessNames.get(ws.space_uid)!;
progress.startProcess(spaceProcess);
log.debug(`Exporting space: ${ws.space_uid}`, context);
const spaceDir = pResolve(spacesRootPath, ws.space_uid);
try {
const exportWorkspace = new ExportWorkspace(apiConfig, exportContext);
exportWorkspace.setParentProgressManager(progress);
await exportWorkspace.start(ws, spaceDir, branchName || 'main');
await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
progress.completeProcess(spaceProcess, true);
log.debug(`Exported workspace structure for space ${ws.space_uid}`, context);
} catch (err) {
// Per-space failure: mark the row failed and continue with the next
// space so partial export results are preserved (matches import).
anySpaceFailed = true;
log.debug(`Failed to export workspace for space ${ws.space_uid}: ${err}`, context);
progress.tick(
false,
`space: ${ws.space_uid}`,
(err as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_SPACE_METADATA].FAILED,
AM_MAIN_PROCESS_NAME,
handleAndLogError(
err,
{ ...(context as Record<string, unknown>), spaceUid: ws.space_uid },
`Failed to export space ${ws.space_uid}`,
);
throw err;
progress.completeProcess(spaceProcess, false);
}
}

progress.completeProcess(AM_MAIN_PROCESS_NAME, true);
log.info('Asset Management export completed successfully', context);
log.info(
anySpaceFailed
? 'Asset Management export completed with errors in one or more spaces'
: 'Asset Management export completed successfully',
context,
);
log.debug('Asset Management 2.0 export completed', context);
} catch (err) {
progress.completeProcess(AM_MAIN_PROCESS_NAME, false);
if (!bootstrapFailed) {
// Mark any spaces that hadn't been processed as failed so the multibar
// doesn't leave dangling pending rows.
for (const [, spaceProcess] of spaceProcessNames) {
progress.completeProcess(spaceProcess, false);
}
}
handleAndLogError(err, { ...(context as Record<string, unknown>) }, 'Asset Management export failed');
throw err;
}
Expand Down
Loading
Loading