diff --git a/cspell.json b/cspell.json index 904d24be1..e3a282d13 100644 --- a/cspell.json +++ b/cspell.json @@ -284,6 +284,7 @@ "url's", "valign", "valtype", + "walltime", "wasi", "wasm", "watchings", @@ -304,7 +305,6 @@ "commithash", "formaters", "akait", - "Akait", "evenstensberg", "Stensberg", "ovflowd", diff --git a/package.json b/package.json index fdf3140fe..fca08e2dc 100644 --- a/package.json +++ b/package.json @@ -110,7 +110,7 @@ "devDependencies": { "@babel/core": "^7.27.1", "@babel/preset-react": "^7.27.1", - "@codspeed/core": "^4.0.1", + "@codspeed/core": "^5.0.1", "@eslint/js": "^9.36.0", "@eslint/markdown": "^7.3.0", "@stylistic/eslint-plugin": "^5.4.0", diff --git a/test/BenchmarkTestCases.benchmark.mjs b/test/BenchmarkTestCases.benchmark.mjs index 0fa9b6932..2f6eeb3e6 100644 --- a/test/BenchmarkTestCases.benchmark.mjs +++ b/test/BenchmarkTestCases.benchmark.mjs @@ -3,6 +3,15 @@ import fs from "fs/promises"; import { Session } from "inspector"; import path from "path"; import { fileURLToPath, pathToFileURL } from "url"; +import { + InstrumentHooks, + getCodspeedRunnerMode, + getGitDir, + getV8Flags, + mongoMeasurement, + setupCore, + teardownCore +} from "@codspeed/core"; import { simpleGit } from "simple-git"; import { Bench, hrtimeNow } from "tinybench"; @@ -12,32 +21,6 @@ const git = simpleGit(rootPath); const REV_LIST_REGEXP = /^([a-f0-9]+)\s*([a-f0-9]+)\s*([a-f0-9]+)?\s*$/; -const getV8Flags = () => { - const nodeVersionMajor = Number.parseInt( - process.version.slice(1).split(".")[0], - 10 - ); - const flags = [ - "--hash-seed=1", - "--random-seed=1", - "--no-opt", - "--predictable", - "--predictable-gc-schedule", - "--interpreted-frames-native-stack", - "--allow-natives-syntax", - "--expose-gc", - "--no-concurrent-sweeping", - "--max-old-space-size=4096" - ]; - if (nodeVersionMajor < 18) { - flags.push("--no-randomize-hashes"); - } - if (nodeVersionMajor < 20) { - flags.push("--no-scavenge-task"); - } - return flags; -}; - const checkV8Flags = () => { const requiredFlags = getV8Flags(); const actualFlags = process.execArgv; @@ -248,6 +231,8 @@ for (const baselineInfo of baselineRevisions) { } } +const baseOutputPath = path.join(__dirname, "js", "benchmark"); + function buildConfiguration( test, baseline, @@ -385,105 +370,239 @@ const scenarios = [ } ]; -const baseOutputPath = path.join(__dirname, "js", "benchmark"); +function getStackTrace(belowFn) { + const oldLimit = Error.stackTraceLimit; + Error.stackTraceLimit = Infinity; + const dummyObject = {}; + const v8Handler = Error.prepareStackTrace; + Error.prepareStackTrace = (dummyObject, v8StackTrace) => v8StackTrace; + Error.captureStackTrace(dummyObject, belowFn || getStackTrace); + const v8StackTrace = dummyObject.stack; + Error.prepareStackTrace = v8Handler; + Error.stackTraceLimit = oldLimit; + return v8StackTrace; +} + +function getCallingFile() { + const stack = getStackTrace(); + let callingFile = stack[2].getFileName(); // [here, withCodSpeed, actual caller] + const gitDir = getGitDir(callingFile); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + if (callingFile.startsWith("file://")) { + callingFile = fileURLToPath(callingFile); + } + return path.relative(gitDir, callingFile); +} + +const taskUriMap = new WeakMap(); + +function getOrCreateUriMap(bench) { + let uriMap = taskUriMap.get(bench); + if (!uriMap) { + uriMap = new Map(); + taskUriMap.set(bench, uriMap); + } + return uriMap; +} + +function getTaskUri(bench, taskName, rootCallingFile) { + const uriMap = taskUriMap.get(bench); + return uriMap?.get(taskName) || `${rootCallingFile}::${taskName}`; +} const withCodSpeed = async (/** @type {import("tinybench").Bench} */ bench) => { - const { Measurement, getGitDir, mongoMeasurement, setupCore, teardownCore } = - await import("@codspeed/core"); + const codspeedRunnerMode = getCodspeedRunnerMode(); - if (!Measurement.isInstrumented()) { - const rawRun = bench.run; - bench.run = async () => { - console.warn( - `[CodSpeed] ${bench.tasks.length} benches detected but no instrumentation found, falling back to tinybench` - ); - return await rawRun.bind(bench)(); - }; + if (codspeedRunnerMode === "disabled") { return bench; } - const getStackTrace = (belowFn) => { - const oldLimit = Error.stackTraceLimit; - Error.stackTraceLimit = Infinity; - const dummyObject = {}; - const v8Handler = Error.prepareStackTrace; - Error.prepareStackTrace = (dummyObject, v8StackTrace) => v8StackTrace; - Error.captureStackTrace(dummyObject, belowFn || getStackTrace); - const v8StackTrace = dummyObject.stack; - Error.prepareStackTrace = v8Handler; - Error.stackTraceLimit = oldLimit; - return v8StackTrace; - }; - - const getCallingFile = () => { - const stack = getStackTrace(); - let callingFile = stack[2].getFileName(); // [here, withCodSpeed, actual caller] - const gitDir = getGitDir(callingFile); - if (gitDir === undefined) { - throw new Error("Could not find a git repository"); - } - if (callingFile.startsWith("file://")) { - callingFile = fileURLToPath(callingFile); - } - return path.relative(gitDir, callingFile); - }; - const rawAdd = bench.add; + const uriMap = getOrCreateUriMap(bench); bench.add = (name, fn, opts) => { const callingFile = getCallingFile(); - const uri = `${callingFile}::${name}`; - const options = { ...opts, uri }; - return rawAdd.bind(bench)(name, fn, options); + let uri = callingFile; + if (bench.name !== undefined) { + uri += `::${bench.name}`; + } + uri += `::${name}`; + uriMap.set(name, uri); + return rawAdd.bind(bench)(name, fn, opts); }; const rootCallingFile = getCallingFile(); - bench.run = async function run() { - const iterations = bench.opts.iterations - 1; - console.log("[CodSpeed] running"); - setupCore(); - for (const task of bench.tasks) { - await bench.opts.setup?.(task, "run"); - await task.fnOpts.beforeAll?.call(task); - const samples = []; - async function iteration() { - try { - await task.fnOpts.beforeEach?.call(task, "run"); - const start = bench.opts.now(); - await task.fn(); - samples.push(bench.opts.now() - start || 0); - await task.fnOpts.afterEach?.call(this, "run"); - } catch (err) { - if (bench.opts.throws) { - throw err; - } + + if (codspeedRunnerMode === "instrumented") { + const setupBenchRun = () => { + setupCore(); + console.log( + "[CodSpeed] running with @codspeed/tinybench (instrumented mode)" + ); + }; + const finalizeBenchRun = () => { + teardownCore(); + console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); + return bench.tasks; + }; + + const wrapFunctionWithFrame = (fn, isAsync) => { + if (isAsync) { + return async function __codspeed_root_frame__() { + await fn(); + }; + } + + return function __codspeed_root_frame__() { + fn(); + }; + }; + + const logTaskCompletion = (uri, status) => { + console.log(`[CodSpeed] ${status} ${uri}`); + }; + + const taskCompletionMessage = () => + InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + + const iterationAsync = async (task) => { + try { + await task.fnOpts.beforeEach?.call(task, "run"); + const start = bench.opts.now(); + await task.fn(); + const end = bench.opts.now() - start || 0; + await task.fnOpts.afterEach?.call(this, "run"); + return [start, end]; + } catch (err) { + if (bench.opts.throws) { + throw err; } } - while (samples.length < iterations) { - await iteration(); - } - // Codspeed Measure - const uri = - task.opts && "uri" in task.options - ? task.opts.uri - : `${rootCallingFile}::${task.name}`; - await task.fnOpts.beforeEach?.call(task); - await mongoMeasurement.start(uri); - await (async function __codspeed_root_frame__() { - Measurement.startInstrumentation(); - await task.fn(); - Measurement.stopInstrumentation(uri); - })(); - await mongoMeasurement.stop(uri); - await task.fnOpts.afterEach?.call(task); - console.log(`[Codspeed] ✔ Measured ${uri}`); - await task.fnOpts.afterAll?.call(task); + }; + const wrapWithInstrumentHooksAsync = async (fn, uri) => { + InstrumentHooks.startBenchmark(); + const result = await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + return result; + }; + + const runTaskAsync = async (task, uri) => { + const { fnOpts, fn } = task; + + // Custom setup + await bench.opts.setup?.(task, "run"); + + await fnOpts?.beforeAll?.call(task, "run"); + + // Custom warmup + // We don't run `optimizeFunction` because our function is never optimized, instead we just warmup webpack + const samples = []; + + while (samples.length < bench.opts.iterations - 1) { + samples.push(await iterationAsync(task)); + } + + await fnOpts?.beforeEach?.call(task, "run"); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithInstrumentHooksAsync(wrapFunctionWithFrame(fn, true), uri); + await mongoMeasurement.stop(uri); + await fnOpts?.afterEach?.call(task, "run"); + console.log(`[Codspeed] ✔ Measured ${uri}`); + await fnOpts?.afterAll?.call(task, "run"); + + // Custom teardown await bench.opts.teardown?.(task, "run"); - task.processRunResult({ latencySamples: samples }); - } - teardownCore(); - console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); - return bench.tasks; - }; + + logTaskCompletion(uri, taskCompletionMessage()); + }; + + const iteration = (task) => { + try { + task.fnOpts.beforeEach?.call(task, "run"); + const start = bench.opts.now(); + task.fn(); + const end = bench.opts.now() - start || 0; + task.fnOpts.afterEach?.call(this, "run"); + return [start, end]; + } catch (err) { + if (bench.opts.throws) { + throw err; + } + } + }; + + const wrapWithInstrumentHooks = (fn, uri) => { + InstrumentHooks.startBenchmark(); + const result = fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + return result; + }; + + const runTaskSync = (task, uri) => { + const { fnOpts, fn } = task; + + // Custom setup + bench.opts.setup?.(task, "run"); + + fnOpts?.beforeAll?.call(task, "run"); + + // Custom warmup + const samples = []; + + while (samples.length < bench.opts.iterations - 1) { + samples.push(iteration(task)); + } + + fnOpts?.beforeEach?.call(task, "run"); + + wrapWithInstrumentHooks(wrapFunctionWithFrame(fn, false), uri); + + fnOpts?.afterEach?.call(task, "run"); + console.log(`[Codspeed] ✔ Measured ${uri}`); + fnOpts?.afterAll?.call(task, "run"); + + // Custom teardown + bench.opts.teardown?.(task, "run"); + + logTaskCompletion(uri, taskCompletionMessage()); + }; + + const finalizeAsyncRun = () => { + finalizeBenchRun(); + }; + const finalizeSyncRun = () => { + finalizeBenchRun(); + }; + + bench.run = async () => { + setupBenchRun(); + + for (const task of bench.tasks) { + const uri = getTaskUri(task.bench, task.name, rootCallingFile); + await runTaskAsync(task, uri); + } + + return finalizeAsyncRun(); + }; + + bench.runSync = () => { + setupBenchRun(); + + for (const task of bench.tasks) { + const uri = getTaskUri(task.bench, task.name, rootCallingFile); + runTaskSync(task, uri); + } + + return finalizeSyncRun(); + }; + } else if (codspeedRunnerMode === "walltime") { + // We don't need it + } + return bench; }; @@ -495,7 +614,6 @@ const bench = await withCodSpeed( warmupIterations: 2, iterations: 8, setup(task, mode) { - global.gc(); console.log(`Setup (${mode} mode): ${task.name}`); }, teardown(task, mode) { diff --git a/yarn.lock b/yarn.lock index 6f644b9b8..2289312b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -332,14 +332,14 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@codspeed/core@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@codspeed/core/-/core-4.0.1.tgz#91049cce17b8c1d1b4b6cbc481f5ddc1145d6e1e" - integrity sha512-fJ53arfgtzCDZa8DuGJhpTZ3Ll9A1uW5nQ2jSJnfO4Hl5MRD2cP8P4vPvIUAGbdbjwCxR1jat6cW8OloMJkJXw== +"@codspeed/core@^5.0.1": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@codspeed/core/-/core-5.0.1.tgz#6145c898a86a6d56a169611c3e9657a8b97c7642" + integrity sha512-4g5ZyFAin8QywK4+0FK1uXG3GLRPu0oc3xbP+7OUhhFxbwpzFuaJtKmnTofMqLy9/pHH6Bl/7H0/DTVH3cpFkA== dependencies: axios "^1.4.0" find-up "^6.3.0" - form-data "^4.0.0" + form-data "^4.0.4" node-gyp-build "^4.6.0" "@cspell/cspell-bundled-dicts@9.1.3": @@ -3878,7 +3878,7 @@ fork-ts-checker-webpack-plugin@^9.0.2: semver "^7.3.5" tapable "^2.2.1" -form-data@^4.0.0, form-data@^4.0.4: +form-data@^4.0.4: version "4.0.4" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.4.tgz#784cdcce0669a9d68e94d11ac4eea98088edd2c4" integrity sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==