Skip to content

Commit d814fa7

Browse files
aman19Kaman19K
committed
fix: asset heap memoryb issue
1 parent b193cb2 commit d814fa7

File tree

18 files changed

+2854
-25
lines changed

18 files changed

+2854
-25
lines changed

‎packages/contentstack-import/src/commands/cm/stacks/import-recovery.ts‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
import { Command } from '@contentstack/cli-command';
2+
import { flags, FlagInput, cliux, log } from '@contentstack/cli-utilities';
3+
import { ImportRecoveryManager } from '../../../utils';
4+
5+
export default class ImportRecoveryCommand extends Command {
6+
static description = 'Analyze and recover from failed import operations';
7+
8+
static examples: string[] = [
9+
'csdx cm:stacks:import-recovery --data-dir <path>',
10+
'csdx cm:stacks:import-recovery --data-dir <path> --clean',
11+
'csdx cm:stacks:import-recovery --data-dir <path> --report'
12+
];
13+
14+
static flags: FlagInput = {
15+
'data-dir': flags.string({
16+
char: 'd',
17+
description: 'The path to the directory containing the import data and state files',
18+
required: true,
19+
}),
20+
clean: flags.boolean({
21+
description: 'Clean the import state to start fresh (creates backup)',
22+
default: false,
23+
}),
24+
report: flags.boolean({
25+
description: 'Generate a detailed recovery report',
26+
default: false,
27+
}),
28+
'output-file': flags.string({
29+
description: 'Save the recovery report to a file',
30+
}),
31+
};
32+
33+
static usage: string = 'cm:stacks:import-recovery --data-dir <value> [--clean] [--report] [--output-file <value>]';
34+
35+
async run(): Promise<void> {
36+
try {
37+
const { flags } = await this.parse(ImportRecoveryCommand);
38+
39+
const recoveryManager = ImportRecoveryManager.create(flags['data-dir']);
40+
41+
if (flags.clean) {
42+
await this.cleanImportState(recoveryManager);
43+
return;
44+
}
45+
46+
if (flags.report) {
47+
await this.generateReport(recoveryManager, flags['output-file']);
48+
return;
49+
}
50+
51+
// Default: analyze and provide recommendations
52+
await this.analyzeAndRecommend(recoveryManager);
53+
54+
} catch (error) {
55+
log.error(`Recovery command failed: ${error}`);
56+
cliux.print(`Error: ${error}`, { color: 'red' });
57+
}
58+
}
59+
60+
private async analyzeAndRecommend(recoveryManager: ImportRecoveryManager): Promise<void> {
61+
cliux.print('\n🔍 Analyzing import state...', { color: 'blue' });
62+
63+
const info = recoveryManager.analyzeImportState();
64+
const recommendation = recoveryManager.getRecoveryRecommendation(info);
65+
66+
// Display state information
67+
cliux.print('\n📊 Import State Summary:', { color: 'cyan' });
68+
cliux.print(` State File: ${info.stateFileExists ? '✅ Found' : '❌ Not Found'}`);
69+
cliux.print(` Assets Processed: ${info.mappingCounts.assets}`);
70+
cliux.print(` Folders Processed: ${info.mappingCounts.folders}`);
71+
cliux.print(` URL Mappings: ${info.mappingCounts.urls}`);
72+
73+
if (info.lastUpdated) {
74+
const lastUpdated = new Date(info.lastUpdated);
75+
cliux.print(` Last Updated: ${lastUpdated.toLocaleString()}`);
76+
}
77+
78+
if (info.estimatedProgress) {
79+
cliux.print(` Estimated Progress: ~${info.estimatedProgress}%`);
80+
}
81+
82+
// Display recommendation
83+
cliux.print(`\n💡 Recommendation: ${recommendation.action.toUpperCase()}`, {
84+
color: recommendation.action === 'resume' ? 'green' :
85+
recommendation.action === 'restart' ? 'yellow' : 'red'
86+
});
87+
cliux.print(` ${recommendation.reason}`);
88+
89+
if (recommendation.commands && recommendation.commands.length > 0) {
90+
cliux.print('\n📋 Commands:', { color: 'cyan' });
91+
recommendation.commands.forEach(cmd => {
92+
if (cmd.startsWith('#')) {
93+
cliux.print(` ${cmd}`, { color: 'gray' });
94+
} else if (cmd.trim() === '') {
95+
cliux.print('');
96+
} else {
97+
cliux.print(` ${cmd}`, { color: 'white' });
98+
}
99+
});
100+
}
101+
102+
if (recommendation.warnings && recommendation.warnings.length > 0) {
103+
cliux.print('\n⚠️ Warnings:', { color: 'yellow' });
104+
recommendation.warnings.forEach(warning => {
105+
cliux.print(` ${warning}`);
106+
});
107+
}
108+
109+
cliux.print('\n💡 Tip: Use --report flag for a detailed analysis or --clean to start fresh\n');
110+
}
111+
112+
private async cleanImportState(recoveryManager: ImportRecoveryManager): Promise<void> {
113+
cliux.print('\n🧹 Cleaning import state...', { color: 'yellow' });
114+
115+
const info = recoveryManager.analyzeImportState();
116+
117+
if (!info.stateFileExists) {
118+
cliux.print('✅ No import state found. Nothing to clean.', { color: 'green' });
119+
return;
120+
}
121+
122+
if (info.mappingCounts.assets > 0 || info.mappingCounts.folders > 0) {
123+
cliux.print(`⚠️ Warning: This will remove progress for ${info.mappingCounts.assets} assets and ${info.mappingCounts.folders} folders.`, { color: 'yellow' });
124+
125+
const confirm = await cliux.confirm('Are you sure you want to clean the import state? (y/N)');
126+
if (!confirm) {
127+
cliux.print('❌ Operation cancelled.', { color: 'red' });
128+
return;
129+
}
130+
}
131+
132+
const success = recoveryManager.cleanImportState();
133+
134+
if (success) {
135+
cliux.print('✅ Import state cleaned successfully. You can now start a fresh import.', { color: 'green' });
136+
cliux.print('💡 A backup of the previous state has been created.', { color: 'blue' });
137+
} else {
138+
cliux.print('❌ Failed to clean import state. Check the logs for details.', { color: 'red' });
139+
}
140+
}
141+
142+
private async generateReport(recoveryManager: ImportRecoveryManager, outputFile?: string): Promise<void> {
143+
cliux.print('\n📄 Generating recovery report...', { color: 'blue' });
144+
145+
const report = recoveryManager.generateRecoveryReport();
146+
147+
if (outputFile) {
148+
try {
149+
const fs = require('fs');
150+
fs.writeFileSync(outputFile, report);
151+
cliux.print(`✅ Recovery report saved to: ${outputFile}`, { color: 'green' });
152+
} catch (error) {
153+
cliux.print(`❌ Failed to save report to file: ${error}`, { color: 'red' });
154+
cliux.print('\n📄 Recovery Report:', { color: 'cyan' });
155+
cliux.print(report);
156+
}
157+
} else {
158+
cliux.print('\n📄 Recovery Report:', { color: 'cyan' });
159+
cliux.print(report);
160+
}
161+
}
162+
}

‎packages/contentstack-import/src/commands/cm/stacks/import.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,20 @@ export default class ImportCommand extends Command {
108108
description: 'Skips entry publishing during the import process',
109109
default: false,
110110
}),
111+
'force-backup': flags.boolean({
112+
description: 'Forces backup creation even for large datasets that would normally skip backup for memory optimization.',
113+
default: false,
114+
}),
115+
'disable-memory-optimization': flags.boolean({
116+
description: 'Disables memory optimization features and uses legacy processing (not recommended for large datasets).',
117+
default: false,
118+
}),
119+
'memory-threshold': flags.integer({
120+
description: 'Memory threshold in MB for triggering garbage collection (default: 768MB for large datasets).',
121+
}),
122+
'asset-concurrency': flags.integer({
123+
description: 'Number of concurrent asset uploads (default: 10).',
124+
}),
111125
};
112126

113127
static usage: string =

‎packages/contentstack-import/src/config/index.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,13 +93,23 @@ const config: DefaultConfig = {
9393
assetBatchLimit: 1,
9494
fileName: 'assets.json',
9595
importSameStructure: true,
96-
uploadAssetsConcurrency: 2,
96+
uploadAssetsConcurrency: 10, // Increased from 2 to 10 based on customer success
9797
displayExecutionTime: false,
98-
importFoldersConcurrency: 1,
98+
importFoldersConcurrency: 5, // Increased from 1 to 5 for better performance
9999
includeVersionedAssets: false,
100100
host: 'https://api.contentstack.io',
101101
folderValidKeys: ['name', 'parent_uid'],
102102
validKeys: ['title', 'parent_uid', 'description', 'tags'],
103+
// New memory management configuration
104+
enableMemoryMonitoring: true, // Enable memory monitoring by default
105+
memoryThresholdMB: 768, // Memory pressure threshold for large datasets
106+
enableIncrementalPersistence: true, // Enable incremental state saving
107+
maxRetries: 5, // Retry logic for failed uploads
108+
retryDelay: 2000, // Delay between retries (ms)
109+
enableRateLimiting: true, // Enable rate limiting
110+
rateLimitDelay: 200, // Delay between API calls (ms)
111+
backupSkipThresholdGB: 1, // Skip backup for datasets larger than 1GB
112+
queueClearInterval: 100, // Clear completed queue items every N items
103113
},
104114
'assets-old': {
105115
dirName: 'assets',

0 commit comments

Comments
 (0)