diff --git a/std/testing/bench.ts b/std/testing/bench.ts index 1ea07472e8..1d27040f54 100644 --- a/std/testing/bench.ts +++ b/std/testing/bench.ts @@ -1,4 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { deepAssign } from "../_util/deep_assign.ts"; const { noColor } = Deno; @@ -23,30 +24,69 @@ export interface BenchmarkFunction { export interface BenchmarkDefinition { func: BenchmarkFunction; name: string; + /** Defines how many times the provided `func` should be benchmarked in succession */ runs?: number; } /** Defines runBenchmark's run constraints by matching benchmark names. */ export interface BenchmarkRunOptions { + /** Only benchmarks which name match this regexp will be run*/ only?: RegExp; + /** Benchmarks which name match this regexp will be skipped */ skip?: RegExp; + /** Setting it to true prevents default benchmarking progress logs to the commandline*/ silent?: boolean; } -export interface BenchmarkResult { - name: string; - totalMs: number; - runsCount?: number; - runsAvgMs?: number; - runsMs?: number[]; +/** Defines clearBenchmark's constraints by matching benchmark names. */ +export interface BenchmarkClearOptions { + /** Only benchmarks which name match this regexp will be removed */ + only?: RegExp; + /** Benchmarks which name match this regexp will be kept */ + skip?: RegExp; } +/** Defines the result of a single benchmark */ +export interface BenchmarkResult { + /** The name of the benchmark */ + name: string; + /** The total time it took to run a given bechmark */ + totalMs: number; + /** Times the benchmark was run in succession. Only defined if `runs` for this bench is greater than 1. */ + runsCount?: number; + /** The average time of running the benchmark in milliseconds. Only defined if `runs` for this bench is greater than 1. */ + measuredRunsAvgMs?: number; + /** The individual measurements in millisecond it took to run the benchmark. Only defined if `runs` for this bench is greater than 1. */ + measuredRunsMs?: number[]; +} + +/** Defines the result of a `runBenchmarks` call */ export interface BenchmarkRunResult { - measured: number; + /** How many benchmark were ignored by the provided `only` and `skip` */ filtered: number; + /** The individual results for each benchmark that was run */ results: BenchmarkResult[]; } +/** Defines the current progress during the run of `runBenchmarks` */ +export interface BenchmarkRunProgress extends BenchmarkRunResult { + /** List of the queued benchmarks to run with their name and their run count */ + queued: Array<{ name: string; runsCount: number }>; + /** The currently running benchmark with its name, run count and the already finished measurements in milliseconds */ + running?: { name: string; runsCount: number; measuredRunsMs: number[] }; + /** Indicates in which state benchmarking currently is */ + state: ProgressState; +} + +/** Defines the states `BenchmarkRunProgress` can be in */ +export enum ProgressState { + BenchmarkingStart = "benchmarking_start", + BenchStart = "bench_start", + BenchPartialResult = "bench_partial_result", + BenchResult = "bench_result", + BenchmarkingEnd = "benchmarking_end", +} + export class BenchmarkRunError extends Error { benchmarkName?: string; constructor(msg: string, benchmarkName?: string) { @@ -119,26 +159,56 @@ export function bench( } } -/** Runs all registered and non-skipped benchmarks serially. */ -export async function runBenchmarks({ +/** Clears benchmark candidates which name matches `only` and doesn't match `skip`. + * Removes all candidates if options were not provided */ +export function clearBenchmarks({ only = /[^\s]/, - skip = /^\s*$/, - silent, -}: BenchmarkRunOptions = {}): Promise { + skip = /$^/, +}: BenchmarkClearOptions = {}): void { + const keep = candidates.filter( + ({ name }): boolean => !only.test(name) || skip.test(name) + ); + candidates.splice(0, candidates.length); + candidates.push(...keep); +} + +/** + * Runs all registered and non-skipped benchmarks serially. + * + * @param [progressCb] provides the possibility to get updates of the current progress during the run of the benchmarking + * @returns results of the benchmarking + */ +export async function runBenchmarks( + { only = /[^\s]/, skip = /^\s*$/, silent }: BenchmarkRunOptions = {}, + progressCb?: (progress: BenchmarkRunProgress) => void +): Promise { // Filtering candidates by the "only" and "skip" constraint const benchmarks: BenchmarkDefinition[] = candidates.filter( ({ name }): boolean => only.test(name) && !skip.test(name) ); // Init main counters and error flag const filtered = candidates.length - benchmarks.length; - let measured = 0; let failError: Error | undefined = undefined; // Setting up a shared benchmark clock and timer const clock: BenchmarkClock = { start: NaN, stop: NaN }; const b = createBenchmarkTimer(clock); + // Init progress data + const progress: BenchmarkRunProgress = { + // bench.run is already ensured with verifyOr1Run on register + queued: benchmarks.map((bench) => ({ + name: bench.name, + runsCount: bench.runs!, + })), + results: [], + filtered, + state: ProgressState.BenchmarkingStart, + }; + + // Publish initial progress data + publishProgress(progress, ProgressState.BenchmarkingStart, progressCb); + if (!silent) { - // Iterating given benchmark definitions (await-in-loop) console.log( "running", benchmarks.length, @@ -146,14 +216,25 @@ export async function runBenchmarks({ ); } - // Initializing results array - const benchmarkResults: BenchmarkResult[] = []; + // Iterating given benchmark definitions (await-in-loop) for (const { name, runs = 0, func } of benchmarks) { if (!silent) { // See https://github.com/denoland/deno/pull/1452 about groupCollapsed console.groupCollapsed(`benchmark ${name} ... `); } + // Remove benchmark from queued + const queueIndex = progress.queued.findIndex( + (queued) => queued.name === name && queued.runsCount === runs + ); + if (queueIndex != -1) { + progress.queued.splice(queueIndex, 1); + } + // Init the progress of the running benchmark + progress.running = { name, runsCount: runs, measuredRunsMs: [] }; + // Publish starting of a benchmark + publishProgress(progress, ProgressState.BenchStart, progressCb); + // Trying benchmark.func let result = ""; try { @@ -162,38 +243,59 @@ export async function runBenchmarks({ await func(b); // Making sure the benchmark was started/stopped properly assertTiming(clock, name); - result = `${clock.stop - clock.start}ms`; + // Calculate length of run + const measuredMs = clock.stop - clock.start; + + result = `${measuredMs}ms`; // Adding one-time run to results - benchmarkResults.push({ name, totalMs: clock.stop - clock.start }); + progress.results.push({ name, totalMs: measuredMs }); + // Clear currently running + delete progress.running; + // Publish one-time run benchmark finish + publishProgress(progress, ProgressState.BenchResult, progressCb); } else if (runs > 1) { // Averaging runs let pendingRuns = runs; let totalMs = 0; - // Initializing array holding individual runs ms - const runsMs = []; + // Would be better 2 not run these serially while (true) { // b is a benchmark timer interfacing an unset (NaN) benchmark clock await func(b); // Making sure the benchmark was started/stopped properly assertTiming(clock, name); + + // Calculate length of run + const measuredMs = clock.stop - clock.start; + // Summing up - totalMs += clock.stop - clock.start; + totalMs += measuredMs; // Adding partial result - runsMs.push(clock.stop - clock.start); + progress.running.measuredRunsMs.push(measuredMs); + // Publish partial benchmark results + publishProgress( + progress, + ProgressState.BenchPartialResult, + progressCb + ); + // Resetting the benchmark clock clock.start = clock.stop = NaN; // Once all ran if (!--pendingRuns) { result = `${runs} runs avg: ${totalMs / runs}ms`; // Adding result of multiple runs - benchmarkResults.push({ + progress.results.push({ name, totalMs, runsCount: runs, - runsAvgMs: totalMs / runs, - runsMs, + measuredRunsAvgMs: totalMs / runs, + measuredRunsMs: progress.running.measuredRunsMs, }); + // Clear currently running + delete progress.running; + // Publish results of a multiple run benchmark + publishProgress(progress, ProgressState.BenchResult, progressCb); break; } } @@ -215,29 +317,47 @@ export async function runBenchmarks({ console.groupEnd(); } - measured++; // Resetting the benchmark clock clock.start = clock.stop = NaN; } + // Indicate finished running + delete progress.queued; + // Publish final result in Cb too + publishProgress(progress, ProgressState.BenchmarkingEnd, progressCb); + if (!silent) { // Closing results console.log( `benchmark result: ${!!failError ? red("FAIL") : blue("DONE")}. ` + - `${measured} measured; ${filtered} filtered` + `${progress.results.length} measured; ${filtered} filtered` ); } - // Making sure the program exit code is not zero in case of failure + // Throw error if there was a failing benchmark if (!!failError) { throw failError; } const benchmarkRunResult = { - measured, filtered, - results: benchmarkResults, + results: progress.results, }; return benchmarkRunResult; } + +function publishProgress( + progress: BenchmarkRunProgress, + state: ProgressState, + progressCb?: (progress: BenchmarkRunProgress) => void +): void { + progressCb && progressCb(cloneProgressWithState(progress, state)); +} + +function cloneProgressWithState( + progress: BenchmarkRunProgress, + state: ProgressState +): BenchmarkRunProgress { + return deepAssign({}, progress, { state }) as BenchmarkRunProgress; +} diff --git a/std/testing/bench_test.ts b/std/testing/bench_test.ts index ef004807f2..8e34b27a99 100644 --- a/std/testing/bench_test.ts +++ b/std/testing/bench_test.ts @@ -1,5 +1,12 @@ const { test } = Deno; -import { bench, runBenchmarks, BenchmarkRunError } from "./bench.ts"; +import { + bench, + runBenchmarks, + BenchmarkRunError, + clearBenchmarks, + BenchmarkRunProgress, + ProgressState, +} from "./bench.ts"; import { assertEquals, assert, @@ -63,26 +70,27 @@ test({ const benchResult = await runBenchmarks({ skip: /throw/ }); - assertEquals(benchResult.measured, 5); assertEquals(benchResult.filtered, 1); assertEquals(benchResult.results.length, 5); const resultWithMultipleRunsFiltered = benchResult.results.filter( - (r) => r.name === "runs100ForIncrementX1e6" + ({ name }) => name === "runs100ForIncrementX1e6" ); assertEquals(resultWithMultipleRunsFiltered.length, 1); const resultWithMultipleRuns = resultWithMultipleRunsFiltered[0]; assert(!!resultWithMultipleRuns.runsCount); - assert(!!resultWithMultipleRuns.runsAvgMs); - assert(!!resultWithMultipleRuns.runsMs); + assert(!!resultWithMultipleRuns.measuredRunsAvgMs); + assert(!!resultWithMultipleRuns.measuredRunsMs); assertEquals(resultWithMultipleRuns.runsCount, 100); - assertEquals(resultWithMultipleRuns.runsMs!.length, 100); + assertEquals(resultWithMultipleRuns.measuredRunsMs!.length, 100); + + clearBenchmarks(); }, }); test({ - name: "benchWithoutName", + name: "Bench without name should throw", fn() { assertThrows( (): void => { @@ -95,7 +103,7 @@ test({ }); test({ - name: "benchWithoutStop", + name: "Bench without stop should throw", fn: async function (): Promise { await assertThrowsAsync( async (): Promise => { @@ -112,7 +120,7 @@ test({ }); test({ - name: "benchWithoutStart", + name: "Bench without start should throw", fn: async function (): Promise { await assertThrowsAsync( async (): Promise => { @@ -129,7 +137,7 @@ test({ }); test({ - name: "benchStopBeforeStart", + name: "Bench with stop before start should throw", fn: async function (): Promise { await assertThrowsAsync( async (): Promise => { @@ -145,3 +153,180 @@ test({ ); }, }); + +test({ + name: "clearBenchmarks should clear all candidates", + fn: async function (): Promise { + dummyBench("test"); + + clearBenchmarks(); + const benchingResults = await runBenchmarks({ silent: true }); + + assertEquals(benchingResults.filtered, 0); + assertEquals(benchingResults.results.length, 0); + }, +}); + +test({ + name: "clearBenchmarks with only as option", + fn: async function (): Promise { + // to reset candidates + clearBenchmarks(); + + dummyBench("test"); + dummyBench("onlyclear"); + + clearBenchmarks({ only: /only/ }); + const benchingResults = await runBenchmarks({ silent: true }); + + assertEquals(benchingResults.filtered, 0); + assertEquals(benchingResults.results.length, 1); + assertEquals(benchingResults.results[0].name, "test"); + }, +}); + +test({ + name: "clearBenchmarks with skip as option", + fn: async function (): Promise { + // to reset candidates + clearBenchmarks(); + + dummyBench("test"); + dummyBench("skipclear"); + + clearBenchmarks({ skip: /skip/ }); + const benchingResults = await runBenchmarks({ silent: true }); + + assertEquals(benchingResults.filtered, 0); + assertEquals(benchingResults.results.length, 1); + assertEquals(benchingResults.results[0].name, "skipclear"); + }, +}); + +test({ + name: "clearBenchmarks with only and skip as option", + fn: async function (): Promise { + // to reset candidates + clearBenchmarks(); + + dummyBench("test"); + dummyBench("clearonly"); + dummyBench("clearskip"); + dummyBench("clearonly"); + + clearBenchmarks({ only: /clear/, skip: /skip/ }); + const benchingResults = await runBenchmarks({ silent: true }); + + assertEquals(benchingResults.filtered, 0); + assertEquals(benchingResults.results.length, 2); + assert(!!benchingResults.results.find(({ name }) => name === "test")); + assert(!!benchingResults.results.find(({ name }) => name === "clearskip")); + }, +}); + +test({ + name: "progressCallback of runBenchmarks", + fn: async function (): Promise { + clearBenchmarks(); + dummyBench("skip"); + dummyBench("single"); + dummyBench("multiple", 2); + + const progressCallbacks: BenchmarkRunProgress[] = []; + + const benchingResults = await runBenchmarks( + { skip: /skip/, silent: true }, + (progress) => { + // needs to be deep copied + progressCallbacks.push(progress); + } + ); + + let pc = 0; + // Assert initial progress before running + let progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchmarkingStart); + assertEquals(progress.filtered, 1); + assertEquals(progress.queued.length, 2); + assertEquals(progress.running, undefined); + assertEquals(progress.results, []); + + // Assert start of bench "single" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchStart); + assertEquals(progress.filtered, 1); + assertEquals(progress.queued.length, 1); + assert(!!progress.queued.find(({ name }) => name == "multiple")); + assertEquals(progress.running, { + name: "single", + runsCount: 1, + measuredRunsMs: [], + }); + assertEquals(progress.results, []); + + // Assert result of bench "single" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchResult); + assertEquals(progress.queued.length, 1); + assertEquals(progress.running, undefined); + assertEquals(progress.results.length, 1); + assert(!!progress.results.find(({ name }) => name == "single")); + + // Assert start of bench "multiple" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchStart); + assertEquals(progress.queued.length, 0); + assertEquals(progress.running, { + name: "multiple", + runsCount: 2, + measuredRunsMs: [], + }); + assertEquals(progress.results.length, 1); + + // Assert first result of bench "multiple" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchPartialResult); + assertEquals(progress.queued.length, 0); + assertEquals(progress.running!.measuredRunsMs.length, 1); + assertEquals(progress.results.length, 1); + + // Assert second result of bench "multiple" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchPartialResult); + assertEquals(progress.queued.length, 0); + assertEquals(progress.running!.measuredRunsMs.length, 2); + assertEquals(progress.results.length, 1); + + // Assert finish of bench "multiple" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchResult); + assertEquals(progress.queued.length, 0); + assertEquals(progress.running, undefined); + assertEquals(progress.results.length, 2); + assert(!!progress.results.find(({ name }) => name == "single")); + const resultOfMultiple = progress.results.filter( + ({ name }) => name == "multiple" + ); + assertEquals(resultOfMultiple.length, 1); + assert(!!resultOfMultiple[0].measuredRunsMs); + assert(!!resultOfMultiple[0].measuredRunsAvgMs); + assertEquals(resultOfMultiple[0].measuredRunsMs!.length, 2); + + // The last progress should equal the final result from promise except the state property + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchmarkingEnd); + delete progress.state; + assertEquals(progress, benchingResults); + }, +}); + +function dummyBench(name: string, runs = 1): void { + bench({ + name, + runs, + func(b) { + b.start(); + b.stop(); + }, + }); +}