diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..f533c8b --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,42 @@ +name: Performance Benchmarks + +on: + # Run on pushes to main branches and feature branches with 'perf' in name + push: + branches: [ master, 'feature/*perf*' ] + + # Allow manual triggering + workflow_dispatch: + inputs: + reason: + description: 'Reason for running benchmark' + required: false + default: 'Manual benchmark run' + + # Run on release + release: + types: [published] + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Use Node.js LTS + uses: actions/setup-node@v4 + with: + node-version: 'lts/*' + + - name: Install dependencies + run: npm ci + + - name: Run benchmark + run: | + echo "Starting benchmark run..." + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "Reason: ${{ github.event.inputs.reason }}" + fi + xvfb-run -a npm run benchmark diff --git a/.gitignore b/.gitignore index 6f11d3d..fbfe2d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ out node_modules .vscode-test -*.vsix \ No newline at end of file +*.vsix +benchmark-results-*.json \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 8533606..aae6b81 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -23,6 +23,19 @@ ], "outFiles": [ "${workspaceRoot}/out/test/**/*.js" ], "preLaunchTask": "compileAndCopyTestResources" + }, + { + "name": "Launch Benchmarks", + "type": "extensionHost", + "request": "launch", + "runtimeExecutable": "${execPath}", + "args": [ + "--disable-extensions", + "--extensionDevelopmentPath=${workspaceRoot}", + "--extensionTestsPath=${workspaceRoot}/out/test/systemTests/benchmarkRunner.js" + ], + "outFiles": [ "${workspaceRoot}/out/test/**/*.js" ], + "preLaunchTask": "compileAndCopyTestResources" } ] } diff --git a/README.md b/README.md index 0db3a0a..e2fc959 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![OVSX](https://img.shields.io/open-vsx/v/darkriszty/markdown-table-prettify?color=success&label=Open%20VSX)](https://open-vsx.org/extension/darkriszty/markdown-table-prettify) [![Docker image](https://img.shields.io/docker/v/darkriszty/prettify-md?color=success&label=Docker)](https://hub.docker.com/r/darkriszty/prettify-md/tags?page=1&ordering=last_updated) [![NPM package](https://img.shields.io/npm/v/markdown-table-prettify?color=success)](https://www.npmjs.com/package/markdown-table-prettify) +[![Benchmarks](https://github.com/darkriszty/MarkdownTablePrettify-VSCodeExt/actions/workflows/benchmark.yml/badge.svg)](https://github.com/darkriszty/MarkdownTablePrettify-VSCodeExt/actions/workflows/benchmark.yml) Makes tables more readable for humans. Compatible with the Markdown writer plugin's table formatter feature in Atom. diff --git a/package.json b/package.json index 40b8971..3e1b369 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "compile": "tsc -p ./", "pretest": "npm run compile", "test": "npx gulp copy-systemTest-resources && node ./out/test/index.js", + "benchmark": "npm run compile && npx gulp copy-systemTest-resources && node ./out/test/systemTests/benchmarkTestRunner.js", "prettify-md": "node ./out/cli/index.js", "check-md": "node ./out/cli/index.js --check" }, diff --git a/src/extension/extension.ts b/src/extension/extension.ts index 469e23d..a472895 100644 --- a/src/extension/extension.ts +++ b/src/extension/extension.ts @@ -1,11 +1,20 @@ 'use strict'; import * as vscode from 'vscode'; -import { getSupportLanguageIds, getDocumentRangePrettyfier, getDocumentPrettyfier, getDocumentPrettyfierCommand } from './prettyfierFactory'; +import { getSupportLanguageIds, getDocumentRangePrettyfier, getDocumentPrettyfier, getDocumentPrettyfierCommand, invalidateCache } from './prettyfierFactory'; // This method is called when the extension is activated. // The extension is activated the very first time the command is executed. export function activate(context: vscode.ExtensionContext): void { + // Invalidate cache when configuration changes + context.subscriptions.push( + vscode.workspace.onDidChangeConfiguration(event => { + if (event.affectsConfiguration("markdownTablePrettify")) { + invalidateCache(); + } + }) + ); + const supportedLanguageIds = getSupportLanguageIds(); for (let language of supportedLanguageIds) { context.subscriptions.push( diff --git a/src/extension/prettyfierFactory.ts b/src/extension/prettyfierFactory.ts index ef0a12a..0a3657e 100644 --- a/src/extension/prettyfierFactory.ts +++ b/src/extension/prettyfierFactory.ts @@ -24,6 +24,12 @@ import { SingleTablePrettyfier } from '../prettyfiers/singleTablePrettyfier'; import { TableStringWriter } from "../writers/tableStringWriter"; import { ValuePaddingProvider } from '../writers/valuePaddingProvider'; +let cachedMultiTablePrettyfier: MultiTablePrettyfier | null = null; + +export function invalidateCache() { + cachedMultiTablePrettyfier = null; +} + export function getSupportLanguageIds() { return [ "markdown", ...getConfigurationValue>("extendedLanguages", []) ]; } @@ -41,15 +47,21 @@ export function getDocumentPrettyfierCommand(): TableDocumentPrettyfierCommand { } function getMultiTablePrettyfier(): MultiTablePrettyfier { + if (cachedMultiTablePrettyfier) { + return cachedMultiTablePrettyfier; + } + const loggers = getLoggers(); const sizeLimitCheker = getSizeLimitChecker(loggers); const columnPadding: number = getConfigurationValue("columnPadding", 0); - return new MultiTablePrettyfier( + cachedMultiTablePrettyfier = new MultiTablePrettyfier( new TableFinder(new TableValidator(new SelectionInterpreter(true))), getSingleTablePrettyfier(loggers, sizeLimitCheker, columnPadding), sizeLimitCheker ); + + return cachedMultiTablePrettyfier; } function getSingleTablePrettyfier(loggers: ILogger[], sizeLimitCheker: ConfigSizeLimitChecker, columnPadding: number): SingleTablePrettyfier { diff --git a/src/padCalculation/padCalculatorSelector.ts b/src/padCalculation/padCalculatorSelector.ts index 76b7880..e41a476 100644 --- a/src/padCalculation/padCalculatorSelector.ts +++ b/src/padCalculation/padCalculatorSelector.ts @@ -6,7 +6,19 @@ import * as CenterAlignment from "./center"; import { Alignment } from "../models/alignment"; export class PadCalculatorSelector { - public select(table: Table, column: number) : BasePadCalculator { + private static readonly leftFirstColumn = new LeftAlignment.FirstColumnPadCalculator(); + private static readonly leftMiddleColumn = new LeftAlignment.MiddleColumnPadCalculator(); + private static readonly leftLastColumn = new LeftAlignment.LastColumnPadCalculator(); + + private static readonly centerFirstColumn = new CenterAlignment.FirstColumnPadCalculator(); + private static readonly centerMiddleColumn = new CenterAlignment.MiddleColumnPadCalculator(); + private static readonly centerLastColumn = new CenterAlignment.LastColumnPadCalculator(); + + private static readonly rightFirstColumn = new RightAlignment.FirstColumnPadCalculator(); + private static readonly rightMiddleColumn = new RightAlignment.MiddleColumnPadCalculator(); + private static readonly rightLastColumn = new RightAlignment.LastColumnPadCalculator(); + + public select(table: Table, column: number) : BasePadCalculator { switch (table.alignments[column]) { case Alignment.Center: return this.centerAlignmentPadCalculator(table, column); case Alignment.Right: return this.rightAlignmentPadCalculator(table, column); @@ -14,21 +26,21 @@ export class PadCalculatorSelector { } } - private leftAlignmentPadCalculator(table: Table, column: number) : BasePadCalculator { - if (column == 0) return new LeftAlignment.FirstColumnPadCalculator(); - if (column == table.columnCount - 1) return new LeftAlignment.LastColumnPadCalculator(); - return new LeftAlignment.MiddleColumnPadCalculator(); + private leftAlignmentPadCalculator(table: Table, column: number) : BasePadCalculator { + if (column == 0) return PadCalculatorSelector.leftFirstColumn; + if (column == table.columnCount - 1) return PadCalculatorSelector.leftLastColumn; + return PadCalculatorSelector.leftMiddleColumn; } - private centerAlignmentPadCalculator(table: Table, column: number) : BasePadCalculator { - if (column == 0) return new CenterAlignment.FirstColumnPadCalculator(); - if (column == table.columnCount - 1) return new CenterAlignment.LastColumnPadCalculator(); - return new CenterAlignment.MiddleColumnPadCalculator(); + private centerAlignmentPadCalculator(table: Table, column: number) : BasePadCalculator { + if (column == 0) return PadCalculatorSelector.centerFirstColumn; + if (column == table.columnCount - 1) return PadCalculatorSelector.centerLastColumn; + return PadCalculatorSelector.centerMiddleColumn; } - private rightAlignmentPadCalculator(table: Table, column: number) : BasePadCalculator { - if (column == 0) return new RightAlignment.FirstColumnPadCalculator(); - if (column == table.columnCount - 1) return new RightAlignment.LastColumnPadCalculator(); - return new RightAlignment.MiddleColumnPadCalculator(); + private rightAlignmentPadCalculator(table: Table, column: number) : BasePadCalculator { + if (column == 0) return PadCalculatorSelector.rightFirstColumn; + if (column == table.columnCount - 1) return PadCalculatorSelector.rightLastColumn; + return PadCalculatorSelector.rightMiddleColumn; } } \ No newline at end of file diff --git a/src/viewModelFactories/alignmentMarking/alignmentMarkerStrategy.ts b/src/viewModelFactories/alignmentMarking/alignmentMarkerStrategy.ts index ce75c29..802c7f2 100644 --- a/src/viewModelFactories/alignmentMarking/alignmentMarkerStrategy.ts +++ b/src/viewModelFactories/alignmentMarking/alignmentMarkerStrategy.ts @@ -1,16 +1,25 @@ import { Alignment } from "../../models/alignment"; import { IAlignmentMarker, LeftAlignmentMarker, RightAlignmentMarker, CenterAlignmentMarker, NotSetAlignmentMarker } from "."; - export class AlignmentMarkerStrategy { - constructor(private _markerChar: string) { } + private readonly _leftMarker: IAlignmentMarker; + private readonly _rightMarker: IAlignmentMarker; + private readonly _centerMarker: IAlignmentMarker; + private readonly _notSetMarker: IAlignmentMarker; + + constructor(private _markerChar: string) { + this._leftMarker = new LeftAlignmentMarker(this._markerChar); + this._rightMarker = new RightAlignmentMarker(this._markerChar); + this._centerMarker = new CenterAlignmentMarker(this._markerChar); + this._notSetMarker = new NotSetAlignmentMarker(); + } public markerFor(alignment: Alignment): IAlignmentMarker { switch (alignment) { - case Alignment.Left: return new LeftAlignmentMarker(this._markerChar); - case Alignment.Right: return new RightAlignmentMarker(this._markerChar); - case Alignment.Center: return new CenterAlignmentMarker(this._markerChar); - default: return new NotSetAlignmentMarker(); + case Alignment.Left: return this._leftMarker; + case Alignment.Right: return this._rightMarker; + case Alignment.Center: return this._centerMarker; + default: return this._notSetMarker; } } } \ No newline at end of file diff --git a/src/writers/tableStringWriter.ts b/src/writers/tableStringWriter.ts index aa9e4e8..040d838 100644 --- a/src/writers/tableStringWriter.ts +++ b/src/writers/tableStringWriter.ts @@ -14,36 +14,36 @@ export class TableStringWriter { if (table.rows == null) throw new Error("Table rows can't be null."); if (table.columnCount == 0) throw new Error("Table must have at least one column."); - let buffer = ""; - buffer += this.writeRowViewModel(table.header, table, true); - buffer += this.writeRowViewModel(table.separator, table, table.rowCount > 0); - buffer += this.writeRows(table); + const buffer: string[] = []; + buffer.push(this.writeRowViewModel(table.header, table, true)); + buffer.push(this.writeRowViewModel(table.separator, table, table.rowCount > 0)); + buffer.push(this.writeRows(table)); - return buffer; + return buffer.join(''); } private writeRows(table: TableViewModel): string { - let buffer = ""; + const buffer: string[] = []; for (let row = 0; row < table.rowCount; row++) { - buffer += this.writeRowViewModel(table.rows[row], table, row != table.rowCount - 1); + buffer.push(this.writeRowViewModel(table.rows[row], table, row != table.rowCount - 1)); } - return buffer; + return buffer.join(''); } private writeRowViewModel(row: RowViewModel, table: TableViewModel, addEndOfLine: boolean): string { - let buffer = ""; - buffer += table.leftPad; - buffer += this.getLeftBorderIfNeeded(table); + const buffer: string[] = []; + buffer.push(table.leftPad); + buffer.push(this.getLeftBorderIfNeeded(table)); for (let col = 0; col < table.columnCount; col++) { - buffer += this._valuePaddingProvider.getLeftPadding(); - buffer += row.getValueAt(col); - buffer += this._valuePaddingProvider.getRightPadding(table, col); - buffer += this.getSeparatorIfNeeded(table, col); + buffer.push(this._valuePaddingProvider.getLeftPadding()); + buffer.push(row.getValueAt(col)); + buffer.push(this._valuePaddingProvider.getRightPadding(table, col)); + buffer.push(this.getSeparatorIfNeeded(table, col)); } - buffer += this.getRightBorderIfNeeded(table); + buffer.push(this.getRightBorderIfNeeded(table)); if (addEndOfLine) - buffer += row.EOL; - return buffer; + buffer.push(row.EOL); + return buffer.join(''); } private getSeparatorIfNeeded(table: TableViewModel, currentColumn: number): string { diff --git a/test/systemTests/benchmarkRunner.ts b/test/systemTests/benchmarkRunner.ts new file mode 100644 index 0000000..d91e25c --- /dev/null +++ b/test/systemTests/benchmarkRunner.ts @@ -0,0 +1,456 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as vscode from 'vscode'; +import { getDistinctTestFileNames } from './systemTestFileReader'; +import { getDocumentPrettyfier } from '../../src/extension/prettyfierFactory'; + +interface BenchmarkResults { + factoryCreation: { + iterations: number; + times: number[]; + average: number; + median: number; + min: number; + max: number; + standardDeviation: number; + confidenceInterval95: { lower: number; upper: number }; + totalDuration: number; + }; + documentFormatting: { + [size: string]: { + files: string[]; + iterations: number; + times: number[]; + average: number; + median: number; + min: number; + max: number; + standardDeviation: number; + confidenceInterval95: { lower: number; upper: number }; + totalDuration: number; + }; + }; + overallDuration: number; + timestamp: string; +} + +class PerformanceBenchmark { + private results: BenchmarkResults; + private testFiles: { name: string; size: 'small' | 'medium' | 'large' }[] = []; + private overallStartTime: bigint = BigInt(0); + + private readonly baseline = { + factoryCreation: { average: 0.0001675, median: 0.0001522, standardDeviation: 0.0001 }, + documentFormatting: { + small: { average: 0.1429, median: 0.1162, standardDeviation: 0.05 }, + medium: { average: 0.2358, median: 0.2186, standardDeviation: 0.08 }, + large: { average: 4.2929, median: 2.2453, standardDeviation: 2.0 } + } + }; + + private readonly thresholds = { improvement: 0.5, warning: 1.5, failure: 2 }; + + constructor() { + this.results = { + factoryCreation: { + iterations: 0, + times: [], + average: 0, + median: 0, + min: 0, + max: 0, + standardDeviation: 0, + confidenceInterval95: { lower: 0, upper: 0 }, + totalDuration: 0 + }, + documentFormatting: {}, + overallDuration: 0, + timestamp: new Date().toISOString() + }; + } + + private checkPerformanceRegression(): void { + console.log(`\nšŸ“Š Comparing against hard-coded baseline`); + + let hasRegression = false; + + // Factory creation comparison + const factoryRegression = this.comparePerformance( + 'Factory creation', + this.results.factoryCreation.average, + this.baseline.factoryCreation.average, + 6 + ); + hasRegression = hasRegression || factoryRegression; + + // Document formatting comparisons + for (const [size, results] of Object.entries(this.results.documentFormatting)) { + if (this.baseline.documentFormatting[size]) { + const documentRegression = this.comparePerformance( + `Document formatting (${size})`, + results.average, + this.baseline.documentFormatting[size].average + ); + hasRegression = hasRegression || documentRegression; + } + } + + if (hasRegression) { + throw new Error('Performance regression detected! Benchmark failed due to >25% performance degradation.'); + } + } + + private comparePerformance(component: string, current: number, baseline: number, precision: number = 3): boolean { + const ratio = current / baseline; + const change = (ratio - 1) * 100; + + let isFailure: boolean = false; + let message: string; + + if (ratio <= this.thresholds.improvement) { + message = `šŸŽ‰ ${component}: ${current.toFixed(precision)}ms vs baseline ${baseline.toFixed(precision)}ms (${Math.abs(change).toFixed(1)}% better)`; + } else if (ratio >= this.thresholds.failure) { + isFailure = true; + message = `āŒ ${component} FAILURE: ${current.toFixed(precision)}ms vs baseline ${baseline.toFixed(precision)}ms (${change.toFixed(1)}% slower)`; + } else if (ratio >= this.thresholds.warning) { + message = `āš ļø ${component}: ${current.toFixed(precision)}ms vs baseline ${baseline.toFixed(precision)}ms (${change.toFixed(1)}% slower)`; + } else { + message = `āœ… ${component}: ${current.toFixed(precision)}ms vs baseline ${baseline.toFixed(precision)}ms (${change.toFixed(1)}% change)`; + } + + console.log(message); + return isFailure; + } + + async loadTestFiles(): Promise { + const resourcesDir = path.resolve(__dirname, "resources/"); + const files = fs.readdirSync(resourcesDir); + const distinctTests = getDistinctTestFileNames(files); + + for (const fileNameRoot of distinctTests) { + // Determine file size based on content + const inputPath = path.join(resourcesDir, `${fileNameRoot}-input.md`); + if (fs.existsSync(inputPath)) { + const content = fs.readFileSync(inputPath, 'utf-8'); + const size = this.categorizeFileSize(content); + this.testFiles.push({ name: fileNameRoot, size }); + } + } + + console.log(`Loaded ${this.testFiles.length} test files:`); + const sizeGroups = this.testFiles.reduce((acc, file) => { + acc[file.size] = (acc[file.size] || 0) + 1; + return acc; + }, {} as Record); + console.log(` Small: ${sizeGroups.small || 0}`); + console.log(` Medium: ${sizeGroups.medium || 0}`); + console.log(` Large: ${sizeGroups.large || 0}`); + } + + private categorizeFileSize(content: string): 'small' | 'medium' | 'large' { + const lines = content.split('\n').length; + if (lines <= 20) return 'small'; + if (lines <= 50) return 'medium'; + return 'large'; + } + + async benchmarkFactoryCreation(iterations: number = 100000): Promise { + console.log('šŸ“¦ Testing factory creation overhead...'); + + // Warmup phase to mitigate JIT compilation effects + console.log(' šŸ”„ Warming up JIT compiler...'); + for (let i = 0; i < Math.min(10000, iterations / 10); i++) { + getDocumentPrettyfier(); + } + + const times: number[] = []; + const sectionStart = process.hrtime.bigint(); + + for (let i = 0; i < iterations; i++) { + const start = process.hrtime.bigint(); + getDocumentPrettyfier(); + const end = process.hrtime.bigint(); + times.push(Number(end - start) / 1_000_000); // Convert to milliseconds + } + + const sectionEnd = process.hrtime.bigint(); + const totalDuration = Number(sectionEnd - sectionStart) / 1_000_000; // Convert to milliseconds + + // Remove outliers for more stable results + const cleanTimes = this.removeOutliers(times); + console.log(` šŸ“Š Removed ${times.length - cleanTimes.length} outliers (${((times.length - cleanTimes.length) / times.length * 100).toFixed(1)}%)`); + + const average = cleanTimes.reduce((a, b) => a + b, 0) / cleanTimes.length; + const standardDeviation = this.calculateStandardDeviation(cleanTimes); + const confidenceInterval95 = this.calculateConfidenceInterval(cleanTimes); + + this.results.factoryCreation = { + iterations, + times: cleanTimes, + average, + median: this.calculateMedian(cleanTimes), + min: Math.min(...cleanTimes), + max: Math.max(...cleanTimes), + standardDeviation, + confidenceInterval95, + totalDuration + }; + } + + async benchmarkDocumentFormatting(): Promise { + console.log('šŸ“„ Testing document formatting by size...'); + + const sizeGroups = this.testFiles.reduce((acc, file) => { + if (!acc[file.size]) acc[file.size] = []; + acc[file.size].push(file); + return acc; + }, {} as Record); + + for (const [size, files] of Object.entries(sizeGroups)) { + console.log(` šŸ“‹ Testing ${size} files...`); + const iterations = this.getIterationsForSize(size as any); + const times: number[] = []; + + // Create prettyfier once and reuse + const prettyfier = getDocumentPrettyfier(); + + // Warmup phase - process each file a few times + console.log(` šŸ”„ Warming up with ${size} files...`); + const warmupIterations = Math.min(50, Math.floor(iterations / 20)); + for (let i = 0; i < warmupIterations; i++) { + const file = files[i % files.length]; + const document = await this.openDocument(`resources/${file.name}-input.md`); + prettyfier.provideDocumentFormattingEdits(document, {} as any, {} as any); + } + + const sectionStart = process.hrtime.bigint(); + + for (let i = 0; i < iterations; i++) { + const file = files[i % files.length]; + const document = await this.openDocument(`resources/${file.name}-input.md`); + + const start = process.hrtime.bigint(); + prettyfier.provideDocumentFormattingEdits(document, {} as any, {} as any); + const end = process.hrtime.bigint(); + + times.push(Number(end - start) / 1_000_000); + } + + const sectionEnd = process.hrtime.bigint(); + const totalDuration = Number(sectionEnd - sectionStart) / 1_000_000; // Convert to milliseconds + + // Remove outliers for more stable results + const cleanTimes = this.removeOutliers(times); + console.log(` šŸ“Š Removed ${times.length - cleanTimes.length} outliers (${((times.length - cleanTimes.length) / times.length * 100).toFixed(1)}%)`); + + const average = cleanTimes.reduce((a, b) => a + b, 0) / cleanTimes.length; + const standardDeviation = this.calculateStandardDeviation(cleanTimes); + const confidenceInterval95 = this.calculateConfidenceInterval(cleanTimes); + + this.results.documentFormatting[size] = { + files: files.map(f => f.name), + iterations, + times: cleanTimes, + average, + median: this.calculateMedian(cleanTimes), + min: Math.min(...cleanTimes), + max: Math.max(...cleanTimes), + standardDeviation, + confidenceInterval95, + totalDuration + }; + } + } + + private async openDocument(relativePath: string): Promise { + const fullPath = path.resolve(__dirname, relativePath); + const uri = vscode.Uri.file(fullPath); + return await vscode.workspace.openTextDocument(uri); + } + + private getIterationsForSize(size: 'small' | 'medium' | 'large'): number { + switch (size) { + case 'small': return 25000; + case 'medium': return 8000; + case 'large': return 600; + default: return 15000; + } + } + + private calculateMedian(times: number[]): number { + const sorted = [...times].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[mid - 1] + sorted[mid]) / 2 + : sorted[mid]; + } + + private removeOutliers(times: number[]): number[] { + if (times.length < 10) return times; // Don't remove outliers from small datasets + + const sorted = [...times].sort((a, b) => a - b); + const q1Index = Math.floor(sorted.length * 0.25); + const q3Index = Math.floor(sorted.length * 0.75); + const q1 = sorted[q1Index]; + const q3 = sorted[q3Index]; + const iqr = q3 - q1; + + // Use more conservative outlier detection (3x IQR instead of 1.5x) + const lowerBound = q1 - 3 * iqr; + const upperBound = q3 + 3 * iqr; + + return times.filter(time => time >= lowerBound && time <= upperBound); + } + + private calculateStandardDeviation(times: number[]): number { + const mean = times.reduce((a, b) => a + b, 0) / times.length; + const squaredDeviations = times.map(time => Math.pow(time - mean, 2)); + const variance = squaredDeviations.reduce((a, b) => a + b, 0) / times.length; + return Math.sqrt(variance); + } + + private calculateConfidenceInterval(times: number[]): { lower: number; upper: number } { + const mean = times.reduce((a, b) => a + b, 0) / times.length; + const stdDev = this.calculateStandardDeviation(times); + const standardError = stdDev / Math.sqrt(times.length); + + // 95% confidence interval using t-distribution approximation (1.96 for large samples) + const marginOfError = 1.96 * standardError; + + return { + lower: mean - marginOfError, + upper: mean + marginOfError + }; + } + + private calculateCoefficientOfVariation(times: number[]): number { + const mean = times.reduce((a, b) => a + b, 0) / times.length; + const stdDev = this.calculateStandardDeviation(times); + return (stdDev / mean) * 100; // Return as percentage + } + + private getStabilityRating(coefficientOfVariation: number): string { + if (coefficientOfVariation <= 5) return '🟢 Excellent'; + if (coefficientOfVariation <= 10) return '🟔 Good'; + if (coefficientOfVariation <= 20) return '🟠 Fair'; + return 'šŸ”“ Poor'; + } + + async runFullBenchmark(): Promise { + console.log('šŸš€ Starting Performance Benchmark Suite'); + console.log(`\nUsing ${this.testFiles.length} real test files from system tests\n`); + + this.overallStartTime = process.hrtime.bigint(); + + await this.benchmarkFactoryCreation(); + await this.benchmarkDocumentFormatting(); + + const overallEnd = process.hrtime.bigint(); + this.results.overallDuration = Number(overallEnd - this.overallStartTime) / 1_000_000; // Convert to milliseconds + + this.printResults(); + this.saveResults(); + + this.checkPerformanceRegression(); + } + + private printResults(): void { + console.log('\n' + '='.repeat(100)); + console.log('šŸ“Š PERFORMANCE BENCHMARK RESULTS'); + console.log('='.repeat(100)); + + // Factory creation results + const factory = this.results.factoryCreation; + const factoryCv = this.calculateCoefficientOfVariation(factory.times); + console.log(`\nšŸŽÆ Factory Creation:`); + console.log(` Iterations: ${factory.iterations}`); + console.log(` Average: ${factory.average.toFixed(6)}ms ± ${factory.standardDeviation.toFixed(6)}ms`); + console.log(` Median: ${factory.median.toFixed(6)}ms`); + console.log(` 95% CI: [${factory.confidenceInterval95.lower.toFixed(6)}, ${factory.confidenceInterval95.upper.toFixed(6)}]ms`); + console.log(` Range: [${factory.min.toFixed(6)}, ${factory.max.toFixed(6)}]ms`); + console.log(` Stability: ${factoryCv.toFixed(1)}% CV ${this.getStabilityRating(factoryCv)}`); + console.log(` Total Duration: ${factory.totalDuration.toFixed(3)}ms`); + + // Document formatting results + for (const [size, results] of Object.entries(this.results.documentFormatting)) { + const cv = this.calculateCoefficientOfVariation(results.times); + console.log(`\nšŸŽÆ Document Formatting (${size}):`); + const fileList = results.files.length <= 3 + ? results.files.join(', ') + : `${results.files.slice(0, 3).join(', ')}...`; + console.log(` Test files: ${results.files.length} files (${fileList})`); + console.log(` Iterations: ${results.iterations}`); + console.log(` Average: ${results.average.toFixed(3)}ms ± ${results.standardDeviation.toFixed(3)}ms`); + console.log(` Median: ${results.median.toFixed(3)}ms`); + console.log(` 95% CI: [${results.confidenceInterval95.lower.toFixed(3)}, ${results.confidenceInterval95.upper.toFixed(3)}]ms`); + console.log(` Range: [${results.min.toFixed(3)}, ${results.max.toFixed(3)}]ms`); + console.log(` Stability: ${cv.toFixed(1)}% CV ${this.getStabilityRating(cv)}`); + console.log(` Total Duration: ${results.totalDuration.toFixed(3)}ms`); + } + + console.log('\n' + '='.repeat(100)); + console.log(`ā±ļø OVERALL BENCHMARK DURATION: ${this.results.overallDuration.toFixed(3)}ms`); + console.log('='.repeat(100)); + console.log('šŸ’” TIPS FOR CONSISTENT BENCHMARKING:'); + console.log(' • Close other applications to reduce system interference'); + console.log(' • Run multiple times and compare confidence intervals'); + console.log(' • Look for CV (Coefficient of Variation) < 10% for reliable measurements'); + console.log(' • Focus on trends across multiple runs rather than absolute values'); + console.log('='.repeat(100)); + } + + private saveResults(): void { + // Skip file creation in CI environments + const isCI = process.env.CI || process.env.GITHUB_ACTIONS; + if (isCI) { + console.log('\nšŸ“Š Running in CI - skipping file creation'); + return; + } + + const fileName = `benchmark-results-${this.results.timestamp.replace(/[:.]/g, '-')}.json`; + const filePath = path.resolve(__dirname, '../../..', fileName); + + // Create a copy of results without the "times" arrays + const resultsToSave = { + ...this.results, + factoryCreation: { + ...this.results.factoryCreation, + times: undefined // Exclude times array + }, + documentFormatting: Object.fromEntries( + Object.entries(this.results.documentFormatting).map(([size, data]) => [ + size, + { + ...data, + times: undefined // Exclude times arrays + } + ]) + ) + }; + + fs.writeFileSync(filePath, JSON.stringify(resultsToSave, null, 2)); + console.log(`\nšŸ’¾ Results saved to: ${fileName}`); + } +} + +// Standalone benchmark runner +async function runBenchmark() { + const benchmark = new PerformanceBenchmark(); + await benchmark.loadTestFiles(); + await benchmark.runFullBenchmark(); +} + +// Export for potential use in other contexts +export { PerformanceBenchmark, runBenchmark }; + +// VS Code test runner entry point (for benchmark-only execution) +export async function run(): Promise { + console.log('Starting VS Code Performance Benchmark...\n'); + try { + await runBenchmark(); + console.log('\nāœ… Benchmark completed successfully!'); + } catch (error) { + console.error('\nāŒ Benchmark failed:', error); + throw error; + } +} diff --git a/test/systemTests/benchmarkTestRunner.ts b/test/systemTests/benchmarkTestRunner.ts new file mode 100644 index 0000000..e5b86d7 --- /dev/null +++ b/test/systemTests/benchmarkTestRunner.ts @@ -0,0 +1,21 @@ +import * as path from 'path'; +import { runTests } from '@vscode/test-electron'; + +async function main() { + try { + const extensionDevelopmentPath = path.resolve(__dirname, '../../'); + + const extensionTestsPath = path.resolve(__dirname, './benchmarkRunner.js'); + + await runTests({ + extensionDevelopmentPath, + extensionTestsPath, + launchArgs: ['--disable-extensions'] // Run without other extensions for cleaner benchmark + }); + } catch (err) { + console.error('Failed to run benchmark'); + process.exit(1); + } +} + +main();