foster/deno
0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-03-03 09:31:22 -05:00
Code Issues Packages 1 Wiki Activity

feat(std/testing): benching progress callback (#5941)

Browse source
This commit is contained in:
Szalay Kristóf 2020-06-02 23:06:13 +02:00 • committed by GitHub
parent 23dc9c13db
commit 1db98f10b8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 345 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

180
std/testing/bench.ts
View file

@ -1,4 +1,5 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { deepAssign } from "../_util/deep_assign.ts";
const { noColor } = Deno;
@ -23,30 +24,69 @@ export interface BenchmarkFunction {
export interface BenchmarkDefinition {
func: BenchmarkFunction;
name: string;
/** Defines how many times the provided `func` should be benchmarked in succession */
runs?: number;
}
/** Defines runBenchmark's run constraints by matching benchmark names. */
export interface BenchmarkRunOptions {
/** Only benchmarks which name match this regexp will be run*/
only?: RegExp;
/** Benchmarks which name match this regexp will be skipped */
skip?: RegExp;
/** Setting it to true prevents default benchmarking progress logs to the commandline*/
silent?: boolean;
}
export interface BenchmarkResult {
name: string;
totalMs: number;
runsCount?: number;
runsAvgMs?: number;
runsMs?: number[];
/** Defines clearBenchmark's constraints by matching benchmark names. */
export interface BenchmarkClearOptions {
/** Only benchmarks which name match this regexp will be removed */
only?: RegExp;
/** Benchmarks which name match this regexp will be kept */
skip?: RegExp;
}
/** Defines the result of a single benchmark */
export interface BenchmarkResult {
/** The name of the benchmark */
name: string;
/** The total time it took to run a given bechmark */
totalMs: number;
/** Times the benchmark was run in succession. Only defined if `runs` for this bench is greater than 1. */
runsCount?: number;
/** The average time of running the benchmark in milliseconds. Only defined if `runs` for this bench is greater than 1. */
measuredRunsAvgMs?: number;
/** The individual measurements in millisecond it took to run the benchmark. Only defined if `runs` for this bench is greater than 1. */
measuredRunsMs?: number[];
}
/** Defines the result of a `runBenchmarks` call */
export interface BenchmarkRunResult {
measured: number;
/** How many benchmark were ignored by the provided `only` and `skip` */
filtered: number;
/** The individual results for each benchmark that was run */
results: BenchmarkResult[];
}
/** Defines the current progress during the run of `runBenchmarks` */
export interface BenchmarkRunProgress extends BenchmarkRunResult {
/** List of the queued benchmarks to run with their name and their run count */
queued: Array<{ name: string; runsCount: number }>;
/** The currently running benchmark with its name, run count and the already finished measurements in milliseconds */
running?: { name: string; runsCount: number; measuredRunsMs: number[] };
/** Indicates in which state benchmarking currently is */
state: ProgressState;
}
/** Defines the states `BenchmarkRunProgress` can be in */
export enum ProgressState {
BenchmarkingStart = "benchmarking_start",
BenchStart = "bench_start",
BenchPartialResult = "bench_partial_result",
BenchResult = "bench_result",
BenchmarkingEnd = "benchmarking_end",
}
export class BenchmarkRunError extends Error {
benchmarkName?: string;
constructor(msg: string, benchmarkName?: string) {
@ -119,26 +159,56 @@ export function bench(
}
}
/** Runs all registered and non-skipped benchmarks serially. */
export async function runBenchmarks({
/** Clears benchmark candidates which name matches `only` and doesn't match `skip`.
* Removes all candidates if options were not provided */
export function clearBenchmarks({
only = /[^\s]/,
skip = /^\s*$/,
silent,
}: BenchmarkRunOptions = {}): Promise<BenchmarkRunResult> {
skip = /$^/,
}: BenchmarkClearOptions = {}): void {
const keep = candidates.filter(
({ name }): boolean => !only.test(name) || skip.test(name)
);
candidates.splice(0, candidates.length);
candidates.push(...keep);
}
/**
* Runs all registered and non-skipped benchmarks serially.
*
* @param [progressCb] provides the possibility to get updates of the current progress during the run of the benchmarking
* @returns results of the benchmarking
*/
export async function runBenchmarks(
{ only = /[^\s]/, skip = /^\s*$/, silent }: BenchmarkRunOptions = {},
progressCb?: (progress: BenchmarkRunProgress) => void
): Promise<BenchmarkRunResult> {
// Filtering candidates by the "only" and "skip" constraint
const benchmarks: BenchmarkDefinition[] = candidates.filter(
({ name }): boolean => only.test(name) && !skip.test(name)
);
// Init main counters and error flag
const filtered = candidates.length - benchmarks.length;
let measured = 0;
let failError: Error | undefined = undefined;
// Setting up a shared benchmark clock and timer
const clock: BenchmarkClock = { start: NaN, stop: NaN };
const b = createBenchmarkTimer(clock);
// Init progress data
const progress: BenchmarkRunProgress = {
// bench.run is already ensured with verifyOr1Run on register
queued: benchmarks.map((bench) => ({
name: bench.name,
runsCount: bench.runs!,
})),
results: [],
filtered,
state: ProgressState.BenchmarkingStart,
};
// Publish initial progress data
publishProgress(progress, ProgressState.BenchmarkingStart, progressCb);
if (!silent) {
// Iterating given benchmark definitions (await-in-loop)
console.log(
"running",
benchmarks.length,
@ -146,14 +216,25 @@ export async function runBenchmarks({
);
}
// Initializing results array
const benchmarkResults: BenchmarkResult[] = [];
// Iterating given benchmark definitions (await-in-loop)
for (const { name, runs = 0, func } of benchmarks) {
if (!silent) {
// See https://github.com/denoland/deno/pull/1452 about groupCollapsed
console.groupCollapsed(`benchmark ${name} ... `);
}
// Remove benchmark from queued
const queueIndex = progress.queued.findIndex(
(queued) => queued.name === name && queued.runsCount === runs
);
if (queueIndex != -1) {
progress.queued.splice(queueIndex, 1);
}
// Init the progress of the running benchmark
progress.running = { name, runsCount: runs, measuredRunsMs: [] };
// Publish starting of a benchmark
publishProgress(progress, ProgressState.BenchStart, progressCb);
// Trying benchmark.func
let result = "";
try {
@ -162,38 +243,59 @@ export async function runBenchmarks({
await func(b);
// Making sure the benchmark was started/stopped properly
assertTiming(clock, name);
result = `${clock.stop - clock.start}ms`;
// Calculate length of run
const measuredMs = clock.stop - clock.start;
result = `${measuredMs}ms`;
// Adding one-time run to results
benchmarkResults.push({ name, totalMs: clock.stop - clock.start });
progress.results.push({ name, totalMs: measuredMs });
// Clear currently running
delete progress.running;
// Publish one-time run benchmark finish
publishProgress(progress, ProgressState.BenchResult, progressCb);
} else if (runs > 1) {
// Averaging runs
let pendingRuns = runs;
let totalMs = 0;
// Initializing array holding individual runs ms
const runsMs = [];
// Would be better 2 not run these serially
while (true) {
// b is a benchmark timer interfacing an unset (NaN) benchmark clock
await func(b);
// Making sure the benchmark was started/stopped properly
assertTiming(clock, name);
// Calculate length of run
const measuredMs = clock.stop - clock.start;
// Summing up
totalMs += clock.stop - clock.start;
totalMs += measuredMs;
// Adding partial result
runsMs.push(clock.stop - clock.start);
progress.running.measuredRunsMs.push(measuredMs);
// Publish partial benchmark results
publishProgress(
progress,
ProgressState.BenchPartialResult,
progressCb
);
// Resetting the benchmark clock
clock.start = clock.stop = NaN;
// Once all ran
if (!--pendingRuns) {
result = `${runs} runs avg: ${totalMs / runs}ms`;
// Adding result of multiple runs
benchmarkResults.push({
progress.results.push({
name,
totalMs,
runsCount: runs,
runsAvgMs: totalMs / runs,
runsMs,
measuredRunsAvgMs: totalMs / runs,
measuredRunsMs: progress.running.measuredRunsMs,
});
// Clear currently running
delete progress.running;
// Publish results of a multiple run benchmark
publishProgress(progress, ProgressState.BenchResult, progressCb);
break;
}
}
@ -215,29 +317,47 @@ export async function runBenchmarks({
console.groupEnd();
}
measured++;
// Resetting the benchmark clock
clock.start = clock.stop = NaN;
}
// Indicate finished running
delete progress.queued;
// Publish final result in Cb too
publishProgress(progress, ProgressState.BenchmarkingEnd, progressCb);
if (!silent) {
// Closing results
console.log(
`benchmark result: ${!!failError ? red("FAIL") : blue("DONE")}. ` +
`${measured} measured; ${filtered} filtered`
`${progress.results.length} measured; ${filtered} filtered`
);
}
// Making sure the program exit code is not zero in case of failure
// Throw error if there was a failing benchmark
if (!!failError) {
throw failError;
}
const benchmarkRunResult = {
measured,
filtered,
results: benchmarkResults,
results: progress.results,
};
return benchmarkRunResult;
}
function publishProgress(
progress: BenchmarkRunProgress,
state: ProgressState,
progressCb?: (progress: BenchmarkRunProgress) => void
): void {
progressCb && progressCb(cloneProgressWithState(progress, state));
}
function cloneProgressWithState(
progress: BenchmarkRunProgress,
state: ProgressState
): BenchmarkRunProgress {
return deepAssign({}, progress, { state }) as BenchmarkRunProgress;
}

205
std/testing/bench_test.ts
View file

@ -1,5 +1,12 @@
const { test } = Deno;
import { bench, runBenchmarks, BenchmarkRunError } from "./bench.ts";
import {
bench,
runBenchmarks,
BenchmarkRunError,
clearBenchmarks,
BenchmarkRunProgress,
ProgressState,
} from "./bench.ts";
import {
assertEquals,
assert,
@ -63,26 +70,27 @@ test({
const benchResult = await runBenchmarks({ skip: /throw/ });
assertEquals(benchResult.measured, 5);
assertEquals(benchResult.filtered, 1);
assertEquals(benchResult.results.length, 5);
const resultWithMultipleRunsFiltered = benchResult.results.filter(
(r) => r.name === "runs100ForIncrementX1e6"
({ name }) => name === "runs100ForIncrementX1e6"
);
assertEquals(resultWithMultipleRunsFiltered.length, 1);
const resultWithMultipleRuns = resultWithMultipleRunsFiltered[0];
assert(!!resultWithMultipleRuns.runsCount);
assert(!!resultWithMultipleRuns.runsAvgMs);
assert(!!resultWithMultipleRuns.runsMs);
assert(!!resultWithMultipleRuns.measuredRunsAvgMs);
assert(!!resultWithMultipleRuns.measuredRunsMs);
assertEquals(resultWithMultipleRuns.runsCount, 100);
assertEquals(resultWithMultipleRuns.runsMs!.length, 100);
assertEquals(resultWithMultipleRuns.measuredRunsMs!.length, 100);
clearBenchmarks();
},
});
test({
name: "benchWithoutName",
name: "Bench without name should throw",
fn() {
assertThrows(
(): void => {
@ -95,7 +103,7 @@ test({
});
test({
name: "benchWithoutStop",
name: "Bench without stop should throw",
fn: async function (): Promise<void> {
await assertThrowsAsync(
async (): Promise<void> => {
@ -112,7 +120,7 @@ test({
});
test({
name: "benchWithoutStart",
name: "Bench without start should throw",
fn: async function (): Promise<void> {
await assertThrowsAsync(
async (): Promise<void> => {
@ -129,7 +137,7 @@ test({
});
test({
name: "benchStopBeforeStart",
name: "Bench with stop before start should throw",
fn: async function (): Promise<void> {
await assertThrowsAsync(
async (): Promise<void> => {
@ -145,3 +153,180 @@ test({
);
},
});
test({
name: "clearBenchmarks should clear all candidates",
fn: async function (): Promise<void> {
dummyBench("test");
clearBenchmarks();
const benchingResults = await runBenchmarks({ silent: true });
assertEquals(benchingResults.filtered, 0);
assertEquals(benchingResults.results.length, 0);
},
});
test({
name: "clearBenchmarks with only as option",
fn: async function (): Promise<void> {
// to reset candidates
clearBenchmarks();
dummyBench("test");
dummyBench("onlyclear");
clearBenchmarks({ only: /only/ });
const benchingResults = await runBenchmarks({ silent: true });
assertEquals(benchingResults.filtered, 0);
assertEquals(benchingResults.results.length, 1);
assertEquals(benchingResults.results[0].name, "test");
},
});
test({
name: "clearBenchmarks with skip as option",
fn: async function (): Promise<void> {
// to reset candidates
clearBenchmarks();
dummyBench("test");
dummyBench("skipclear");
clearBenchmarks({ skip: /skip/ });
const benchingResults = await runBenchmarks({ silent: true });
assertEquals(benchingResults.filtered, 0);
assertEquals(benchingResults.results.length, 1);
assertEquals(benchingResults.results[0].name, "skipclear");
},
});
test({
name: "clearBenchmarks with only and skip as option",
fn: async function (): Promise<void> {
// to reset candidates
clearBenchmarks();
dummyBench("test");
dummyBench("clearonly");
dummyBench("clearskip");
dummyBench("clearonly");
clearBenchmarks({ only: /clear/, skip: /skip/ });
const benchingResults = await runBenchmarks({ silent: true });
assertEquals(benchingResults.filtered, 0);
assertEquals(benchingResults.results.length, 2);
assert(!!benchingResults.results.find(({ name }) => name === "test"));
assert(!!benchingResults.results.find(({ name }) => name === "clearskip"));
},
});
test({
name: "progressCallback of runBenchmarks",
fn: async function (): Promise<void> {
clearBenchmarks();
dummyBench("skip");
dummyBench("single");
dummyBench("multiple", 2);
const progressCallbacks: BenchmarkRunProgress[] = [];
const benchingResults = await runBenchmarks(
{ skip: /skip/, silent: true },
(progress) => {
// needs to be deep copied
progressCallbacks.push(progress);
}
);
let pc = 0;
// Assert initial progress before running
let progress = progressCallbacks[pc++];
assertEquals(progress.state, ProgressState.BenchmarkingStart);
assertEquals(progress.filtered, 1);
assertEquals(progress.queued.length, 2);
assertEquals(progress.running, undefined);
assertEquals(progress.results, []);
// Assert start of bench "single"
progress = progressCallbacks[pc++];
assertEquals(progress.state, ProgressState.BenchStart);
assertEquals(progress.filtered, 1);
assertEquals(progress.queued.length, 1);
assert(!!progress.queued.find(({ name }) => name == "multiple"));
assertEquals(progress.running, {
name: "single",
runsCount: 1,
measuredRunsMs: [],
});
assertEquals(progress.results, []);
// Assert result of bench "single"
progress = progressCallbacks[pc++];
assertEquals(progress.state, ProgressState.BenchResult);
assertEquals(progress.queued.length, 1);
assertEquals(progress.running, undefined);
assertEquals(progress.results.length, 1);
assert(!!progress.results.find(({ name }) => name == "single"));
// Assert start of bench "multiple"
progress = progressCallbacks[pc++];
assertEquals(progress.state, ProgressState.BenchStart);
assertEquals(progress.queued.length, 0);
assertEquals(progress.running, {
name: "multiple",
runsCount: 2,
measuredRunsMs: [],
});
assertEquals(progress.results.length, 1);
// Assert first result of bench "multiple"
progress = progressCallbacks[pc++];
assertEquals(progress.state, ProgressState.BenchPartialResult);
assertEquals(progress.queued.length, 0);
assertEquals(progress.running!.measuredRunsMs.length, 1);
assertEquals(progress.results.length, 1);
// Assert second result of bench "multiple"
progress = progressCallbacks[pc++];
assertEquals(progress.state, ProgressState.BenchPartialResult);
assertEquals(progress.queued.length, 0);
assertEquals(progress.running!.measuredRunsMs.length, 2);
assertEquals(progress.results.length, 1);
// Assert finish of bench "multiple"
progress = progressCallbacks[pc++];
assertEquals(progress.state, ProgressState.BenchResult);
assertEquals(progress.queued.length, 0);
assertEquals(progress.running, undefined);
assertEquals(progress.results.length, 2);
assert(!!progress.results.find(({ name }) => name == "single"));
const resultOfMultiple = progress.results.filter(
({ name }) => name == "multiple"
);
assertEquals(resultOfMultiple.length, 1);
assert(!!resultOfMultiple[0].measuredRunsMs);
assert(!!resultOfMultiple[0].measuredRunsAvgMs);
assertEquals(resultOfMultiple[0].measuredRunsMs!.length, 2);
// The last progress should equal the final result from promise except the state property
progress = progressCallbacks[pc++];
assertEquals(progress.state, ProgressState.BenchmarkingEnd);
delete progress.state;
assertEquals(progress, benchingResults);
},
});
function dummyBench(name: string, runs = 1): void {
bench({
name,
runs,
func(b) {
b.start();
b.stop();
},
});
}