From 96a674a321efa8f2d0df1b409f190f91dd079885 Mon Sep 17 00:00:00 2001 From: Andre Wiggins Date: Thu, 15 Oct 2020 22:54:40 -0700 Subject: [PATCH 1/2] Add support for custom measurement units --- config.schema.json | 9 +++++++ src/configfile.ts | 8 ++++++ src/csv.ts | 4 +-- src/defaults.ts | 4 ++- src/format.ts | 25 ++++++++++-------- src/json-output.ts | 2 +- src/runner.ts | 6 ++--- src/specs.ts | 3 +++ src/test/config_test.ts | 16 +++--------- src/test/configfile_test.ts | 27 ++++++++++---------- src/test/format_test.ts | 49 ++++++++++++++++++++++++++++++++++++ src/test/json-output_test.ts | 6 +++++ src/test/specs_test.ts | 34 ++++++++----------------- src/test/test_helpers.ts | 12 ++++----- src/types.ts | 5 ++-- 15 files changed, 134 insertions(+), 76 deletions(-) diff --git a/config.schema.json b/config.schema.json index dca9e375..5af4f0f1 100644 --- a/config.schema.json +++ b/config.schema.json @@ -13,6 +13,9 @@ }, "name": { "type": "string" + }, + "unit": { + "type": "string" } }, "required": [ @@ -218,6 +221,9 @@ }, "name": { "type": "string" + }, + "unit": { + "type": "string" } }, "required": [ @@ -361,6 +367,9 @@ }, "name": { "type": "string" + }, + "unit": { + "type": "string" } }, "required": [ diff --git a/src/configfile.ts b/src/configfile.ts index aaea2a5b..219c9e3c 100644 --- a/src/configfile.ts +++ b/src/configfile.ts @@ -362,6 +362,14 @@ async function parseBenchmark(benchmark: ConfigFileBenchmark, root: string): spec.measurement = [benchmark.measurement]; } + if (spec.measurement) { + for (const measurement of spec.measurement) { + if (measurement.unit == null) { + measurement.unit = defaults.measurementUnit; + } + } + } + const url = benchmark.url; if (url !== undefined) { if (isHttpUrl(url)) { diff --git a/src/csv.ts b/src/csv.ts index b05ca70c..910a2d5a 100644 --- a/src/csv.ts +++ b/src/csv.ts @@ -69,11 +69,11 @@ export function formatCsvRaw(results: ResultStatsWithDifferences[]): string { for (let r = 0; r < results.length; r++) { const {result} = results[r]; headers.push(result.name); - for (let m = 0; m < result.millis.length; m++) { + for (let m = 0; m < result.rawData.length; m++) { if (rows[m] === undefined) { rows[m] = []; } - rows[m][r] = result.millis[m]; + rows[m][r] = result.rawData[m]; } } return csvStringify([headers, ...rows]); diff --git a/src/defaults.ts b/src/defaults.ts index 3b7c2580..dbc62697 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -24,13 +24,15 @@ export const mode = 'automatic'; export const resolveBareModules = true; export const forceCleanNpmInstall = false; export const measurementExpression = 'window.tachometerResult'; +export const measurementUnit = 'ms'; export function measurement(url: LocalUrl|RemoteUrl): Measurement { if (url.kind === 'remote') { return { mode: 'performance', entryName: 'first-contentful-paint', + unit: measurementUnit }; } - return {mode: 'callback'}; + return {mode: 'callback', unit: measurementUnit}; } diff --git a/src/format.ts b/src/format.ts index e0d266f4..2c962c5c 100644 --- a/src/format.ts +++ b/src/format.ts @@ -16,7 +16,7 @@ import {UAParser} from 'ua-parser-js'; import ansi = require('ansi-escape-sequences'); import {Difference, ConfidenceInterval, ResultStats, ResultStatsWithDifferences} from './stats'; -import {BenchmarkSpec, BenchmarkResult} from './types'; +import {BenchmarkSpec, BenchmarkResult, Measurement} from './types'; export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'].map( (frame) => ansi.format(`[blue]{${frame}}`)); @@ -93,7 +93,7 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults { if (diff === null) { return ansi.format('\n[gray]{-} '); } - return formatDifference(diff); + return formatDifference(diff, r.result.measurement); }, }); } @@ -262,7 +262,7 @@ const browserDimension: Dimension = { const sampleSizeDimension: Dimension = { label: 'Sample size', - format: (r: ResultStats) => r.result.millis.length.toString(), + format: (r: ResultStats) => r.result.rawData.length.toString(), }; const bytesSentDimension: Dimension = { @@ -275,25 +275,28 @@ const runtimeConfidenceIntervalDimension: Dimension = { tableConfig: { alignment: 'right', }, - format: (r: ResultStats) => formatConfidenceInterval(r.stats.meanCI, milli), + format: (r: ResultStats) => formatConfidenceInterval( + r.stats.meanCI, formatMeasure(r.result.measurement)), }; -function formatDifference({absolute, relative}: Difference): string { +function formatDifference( + {absolute, relative}: Difference, measurement: Measurement): string { + const format = formatMeasure(measurement); let word, rel, abs; if (absolute.low > 0 && relative.low > 0) { word = `[bold red]{slower}`; rel = formatConfidenceInterval(relative, percent); - abs = formatConfidenceInterval(absolute, milli); + abs = formatConfidenceInterval(absolute, format); } else if (absolute.high < 0 && relative.high < 0) { word = `[bold green]{faster}`; rel = formatConfidenceInterval(negate(relative), percent); - abs = formatConfidenceInterval(negate(absolute), milli); + abs = formatConfidenceInterval(negate(absolute), format); } else { word = `[bold blue]{unsure}`; rel = formatConfidenceInterval(relative, (n) => colorizeSign(n, percent)); - abs = formatConfidenceInterval(absolute, (n) => colorizeSign(n, milli)); + abs = formatConfidenceInterval(absolute, (n) => colorizeSign(n, format)); } return ansi.format(`${word}\n${rel}\n${abs}`); @@ -303,9 +306,9 @@ function percent(n: number): string { return (n * 100).toFixed(0) + '%'; } -function milli(n: number): string { - return n.toFixed(2) + 'ms'; -} +const formatMeasure = (measurement: Measurement) => (n: number) => { + return n.toFixed(2) + measurement.unit; +}; function negate(ci: ConfidenceInterval): ConfidenceInterval { return { diff --git a/src/json-output.ts b/src/json-output.ts index 671c8b28..48459313 100644 --- a/src/json-output.ts +++ b/src/json-output.ts @@ -83,7 +83,7 @@ export function jsonOutput(results: ResultStatsWithDifferences[]): high: result.stats.meanCI.high, }, differences, - samples: result.result.millis, + samples: result.result.rawData, }); } return {benchmarks}; diff --git a/src/runner.ts b/src/runner.ts index a1ea6445..baf907d3 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -154,7 +154,7 @@ export class Runner { if (primary === undefined) { specResults[newResult.measurementIndex] = newResult; } else { - primary.millis.push(...newResult.millis); + primary.rawData.push(...newResult.rawData); } } } @@ -320,7 +320,7 @@ export class Runner { version: spec.url.kind === 'local' && spec.url.version !== undefined ? spec.url.version.label : '', - millis: [measurementResults[measurementIndex]], + rawData: [measurementResults[measurementIndex]], bytesSent: session ? session.bytesSent : 0, browser: spec.browser, userAgent: session ? session.userAgent : '', @@ -332,7 +332,7 @@ export class Runner { for (const results of this.results.values()) { for (let r = 0; r < results.length; r++) { const result = results[r]; - resultStats.push({result, stats: summaryStats(result.millis)}); + resultStats.push({result, stats: summaryStats(result.rawData)}); } } return computeDifferences(resultStats); diff --git a/src/specs.ts b/src/specs.ts index 06f98e3b..8eb26690 100644 --- a/src/specs.ts +++ b/src/specs.ts @@ -73,17 +73,20 @@ export async function specsFromOpts(opts: Opts): Promise { if (opts.measure === 'callback') { measurement = { mode: 'callback', + unit: defaults.measurementUnit, }; } else if (opts.measure === 'fcp') { measurement = { mode: 'performance', entryName: 'first-contentful-paint', + unit: defaults.measurementUnit, }; } else if (opts.measure === 'global') { measurement = { mode: 'expression', expression: opts['measurement-expression'] || defaults.measurementExpression, + unit: defaults.measurementUnit, }; } else if (opts.measure !== undefined) { throwUnreachable( diff --git a/src/test/config_test.ts b/src/test/config_test.ts index e91e419a..d5f1e45b 100644 --- a/src/test/config_test.ts +++ b/src/test/config_test.ts @@ -60,9 +60,7 @@ suite('makeConfig', function() { width: 1024, }, }, - measurement: [{ - mode: 'callback', - }], + measurement: [{mode: 'callback', unit: 'ms'}], name: 'random-global.html', url: { kind: 'local', @@ -103,9 +101,7 @@ suite('makeConfig', function() { width: 1024, }, }, - measurement: [{ - mode: 'callback', - }], + measurement: [{mode: 'callback', unit: 'ms'}], // TODO(aomarks) Why does this have a forward-slash? name: '/random-global.html', url: { @@ -146,9 +142,7 @@ suite('makeConfig', function() { width: 1024, }, }, - measurement: [{ - mode: 'callback', - }], + measurement: [{mode: 'callback', unit: 'ms'}], // TODO(aomarks) Why does this have a forward-slash? name: '/random-global.html', url: { @@ -196,9 +190,7 @@ suite('makeConfig', function() { width: 1024, }, }, - measurement: [{ - mode: 'callback', - }], + measurement: [{mode: 'callback', unit: 'ms'}], // TODO(aomarks) Why does this have a forward-slash? name: '/random-global.html', url: { diff --git a/src/test/configfile_test.ts b/src/test/configfile_test.ts index 47d8380a..60ecb100 100644 --- a/src/test/configfile_test.ts +++ b/src/test/configfile_test.ts @@ -111,6 +111,7 @@ suite('config', () => { { mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms' }, ], url: { @@ -130,6 +131,7 @@ suite('config', () => { measurement: [ { mode: 'callback', + unit: 'ms', }, ], url: { @@ -156,6 +158,7 @@ suite('config', () => { { mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms' }, ], url: { @@ -197,6 +200,7 @@ suite('config', () => { { mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms' }, ], browser: defaultBrowser, @@ -209,9 +213,7 @@ suite('config', () => { queryString: '?foo=bar', }, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], browser: defaultBrowser, }, @@ -261,9 +263,7 @@ suite('config', () => { queryString: '?foo=a', }, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], browser: defaultBrowser, }, @@ -275,10 +275,7 @@ suite('config', () => { queryString: '?foo=b', }, measurement: [ - { - mode: 'expression', - expression: 'window.foo', - }, + {mode: 'expression', expression: 'window.foo', unit: 'ms'}, ], browser: defaultBrowser, }, @@ -290,10 +287,7 @@ suite('config', () => { queryString: '?foo=c', }, measurement: [ - { - mode: 'performance', - entryName: 'foo-measure', - }, + {mode: 'performance', entryName: 'foo-measure', unit: 'ms'}, ], browser: defaultBrowser, }, @@ -339,6 +333,7 @@ suite('config', () => { { mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms', }, ], browser: defaultBrowser, @@ -350,6 +345,7 @@ suite('config', () => { { mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms', }, ], browser: { @@ -363,6 +359,7 @@ suite('config', () => { measurement: [ { mode: 'callback', + unit: 'ms', }, ], browser: defaultBrowser, @@ -373,6 +370,7 @@ suite('config', () => { measurement: [ { mode: 'callback', + unit: 'ms', }, ], browser: { @@ -417,6 +415,7 @@ suite('config', () => { { mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms', }, ], browser: { diff --git a/src/test/format_test.ts b/src/test/format_test.ts index de7d255f..94c8b57b 100644 --- a/src/test/format_test.ts +++ b/src/test/format_test.ts @@ -165,6 +165,55 @@ suite('format', () => { │ bar │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ │ │ │ │ │ 68% - 132% │ - │ │ │ │ │ 7.97ms - 12.03ms │ │ +└───────────┴──────────┴───────────────────┴──────────────────┴──────────────────┘ + `; + assert.equal(actual, expected.trim() + '\n'); + }); + + test('2 labels, custom units', async () => { + const config: ConfigFile = { + benchmarks: [ + { + name: 'bench1', + url: 'http://example.com?p=bar', + browser: { + name: 'chrome', + }, + measurement: { + name: 'measure1', + mode: 'expression', + expression: 'test', + unit: 'μs' + } + }, + { + name: 'bench2', + url: 'http://example.com?p=bar', + browser: { + name: 'chrome', + }, + measurement: { + name: 'measure1', + mode: 'expression', + expression: 'test', + unit: 'μs' + } + }, + ], + }; + + const actual = await fakeResultTable(config); + const expected = ` +┌───────────┬──────────┬───────────────────┬──────────────────┬──────────────────┐ +│ Benchmark │ Bytes │ Avg time │ vs bench1 │ vs bench2 │ +├───────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ bench1 │ 1.00 KiB │ 8.56μs - 11.44μs │ │ faster │ +│ │ │ │ - │ 42% - 58% │ +│ │ │ │ │ 7.97μs - 12.03μs │ +├───────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ bench2 │ 2.00 KiB │ 18.56μs - 21.44μs │ slower │ │ +│ │ │ │ 68% - 132% │ - │ +│ │ │ │ 7.97μs - 12.03μs │ │ └───────────┴──────────┴───────────────────┴──────────────────┴──────────────────┘ `; assert.equal(actual, expected.trim() + '\n'); diff --git a/src/test/json-output_test.ts b/src/test/json-output_test.ts index cf8b2e44..2b503c3c 100644 --- a/src/test/json-output_test.ts +++ b/src/test/json-output_test.ts @@ -68,6 +68,7 @@ suite('jsonOutput', () => { name: 'fcp', mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms' }, browser: { name: 'chrome', @@ -106,6 +107,7 @@ suite('jsonOutput', () => { name: 'fcp', mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms' }, browser: { name: 'chrome', @@ -174,6 +176,7 @@ suite('jsonOutput', () => { name: 'Metric 1', mode: 'performance', entryName: 'metric1', + unit: 'ms' }, browser: { name: 'chrome', @@ -211,6 +214,7 @@ suite('jsonOutput', () => { name: 'Metric 2', mode: 'performance', entryName: 'metric2', + unit: 'ms' }, browser: { name: 'chrome', @@ -245,6 +249,7 @@ suite('jsonOutput', () => { name: 'Metric 1', mode: 'performance', entryName: 'metric1', + unit: 'ms' }, browser: { name: 'chrome', @@ -282,6 +287,7 @@ suite('jsonOutput', () => { name: 'Metric 2', mode: 'performance', entryName: 'metric2', + unit: 'ms' }, browser: { name: 'chrome', diff --git a/src/test/specs_test.ts b/src/test/specs_test.ts index 4c08b21d..6f4252ea 100644 --- a/src/test/specs_test.ts +++ b/src/test/specs_test.ts @@ -69,6 +69,7 @@ suite('specsFromOpts', () => { { mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms' }, ], }, @@ -91,6 +92,7 @@ suite('specsFromOpts', () => { { mode: 'performance', entryName: 'first-contentful-paint', + unit: 'ms' }, ], }, @@ -112,9 +114,7 @@ suite('specsFromOpts', () => { }, browser: defaultBrowser, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], }, ]; @@ -135,9 +135,7 @@ suite('specsFromOpts', () => { }, browser: defaultBrowser, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], }, ]; @@ -158,9 +156,7 @@ suite('specsFromOpts', () => { }, browser: defaultBrowser, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], }, ]; @@ -181,9 +177,7 @@ suite('specsFromOpts', () => { }, browser: defaultBrowser, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], }, ]; @@ -204,9 +198,7 @@ suite('specsFromOpts', () => { }, browser: defaultBrowser, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], }, ]; @@ -227,9 +219,7 @@ suite('specsFromOpts', () => { }, browser: defaultBrowser, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], }, ]; @@ -259,9 +249,7 @@ suite('specsFromOpts', () => { }, browser: defaultBrowser, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], }, { @@ -279,9 +267,7 @@ suite('specsFromOpts', () => { }, browser: defaultBrowser, measurement: [ - { - mode: 'callback', - }, + {mode: 'callback', unit: 'ms'}, ], }, ]; diff --git a/src/test/test_helpers.ts b/src/test/test_helpers.ts index e1272929..a3ac3090 100644 --- a/src/test/test_helpers.ts +++ b/src/test/test_helpers.ts @@ -43,13 +43,13 @@ export async function fakeResults(configFile: ConfigFile): const results = []; for (let i = 0; i < config.benchmarks.length; i++) { const {name, url, browser, measurement} = config.benchmarks[i]; - const averageMillis = (i + 1) * 10; + const averageData = (i + 1) * 10; const bytesSent = (i + 1) * 1024; - const millis = [ + const rawData = [ // Split the sample size in half to add +/- 5ms variance, just to make // things a little more interesting. - ...new Array(Math.floor(config.sampleSize / 2)).fill(averageMillis - 5), - ...new Array(Math.ceil(config.sampleSize / 2)).fill(averageMillis + 5), + ...new Array(Math.floor(config.sampleSize / 2)).fill(averageData - 5), + ...new Array(Math.ceil(config.sampleSize / 2)).fill(averageData + 5), ]; for (let measurementIndex = 0; measurementIndex < measurement.length; measurementIndex++) { @@ -57,7 +57,7 @@ export async function fakeResults(configFile: ConfigFile): name : `${name} [${measurement[measurementIndex].name}]`; results.push({ - stats: summaryStats(millis), + stats: summaryStats(rawData), result: { name: resultName, measurement: measurement[measurementIndex], @@ -66,7 +66,7 @@ export async function fakeResults(configFile: ConfigFile): version: url.kind === 'local' && url.version !== undefined ? url.version.label : '', - millis, + rawData, bytesSent, browser, userAgent: userAgents.get(browser.name) || '', diff --git a/src/types.ts b/src/types.ts index 793ea0d9..2f32b3b6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -81,6 +81,7 @@ export type Measurement = export interface MeasurementBase { name?: string; + unit?: string; } export interface CallbackMeasurement extends MeasurementBase { @@ -146,9 +147,9 @@ export interface BenchmarkResult { */ measurementIndex: number; /** - * Millisecond measurements for each sample. + * Raw measurement results for each sample. */ - millis: number[]; + rawData: number[]; queryString: string; version: string; browser: BrowserConfig; From 47995657a326371b0d834c5c6a2a8d3f7e3a6e23 Mon Sep 17 00:00:00 2001 From: Andre Wiggins Date: Thu, 15 Oct 2020 23:04:22 -0700 Subject: [PATCH 2/2] Add changelog and readme doc for the `unit` field --- CHANGELOG.md | 4 +++- README.md | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 486073ac..3e104872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). - +## Unreleased + +- Add support for defining the unit of a measurement. ## [0.5.5] 2020-09-21 diff --git a/README.md b/README.md index c815828d..52f71066 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,8 @@ If `measurement` is an array, then all of the given measurements will be retrieved from each page load. Each measurement from a page is treated as its own benchmark. -A measurement can specify a `name` property that will be used to display its -results. +A measurement can specify a `name` and/or a `unit` property that will be used to +display its results. #### Performance API