From 154195fa4d947111d26f0563791fb01a0cd2463a Mon Sep 17 00:00:00 2001 From: alexander-akait Date: Mon, 29 Sep 2025 22:54:19 +0300 Subject: [PATCH 1/3] chore: update codespeed --- cspell.json | 2 +- test/BenchmarkTestCases.benchmark.mjs | 344 +++++++++++++++++--------- yarn.lock | 12 +- 3 files changed, 240 insertions(+), 118 deletions(-) diff --git a/cspell.json b/cspell.json index 81ff64131..97e64c2ce 100644 --- a/cspell.json +++ b/cspell.json @@ -284,6 +284,7 @@ "url's", "valign", "valtype", + "walltime", "wasi", "wasm", "watchings", @@ -304,7 +305,6 @@ "commithash", "formaters", "akait", - "Akait", "evenstensberg", "Stensberg", "ovflowd", diff --git a/test/BenchmarkTestCases.benchmark.mjs b/test/BenchmarkTestCases.benchmark.mjs index 0fa9b6932..ddad81447 100644 --- a/test/BenchmarkTestCases.benchmark.mjs +++ b/test/BenchmarkTestCases.benchmark.mjs @@ -3,6 +3,16 @@ import fs from "fs/promises"; import { Session } from "inspector"; import path from "path"; import { fileURLToPath, pathToFileURL } from "url"; +import { + InstrumentHooks, + getCodspeedRunnerMode, + getGitDir, + getV8Flags, + mongoMeasurement, + optimizeFunction, + setupCore, + teardownCore +} from "@codspeed/core"; import { simpleGit } from "simple-git"; import { Bench, hrtimeNow } from "tinybench"; @@ -12,32 +22,6 @@ const git = simpleGit(rootPath); const REV_LIST_REGEXP = /^([a-f0-9]+)\s*([a-f0-9]+)\s*([a-f0-9]+)?\s*$/; -const getV8Flags = () => { - const nodeVersionMajor = Number.parseInt( - process.version.slice(1).split(".")[0], - 10 - ); - const flags = [ - "--hash-seed=1", - "--random-seed=1", - "--no-opt", - "--predictable", - "--predictable-gc-schedule", - "--interpreted-frames-native-stack", - "--allow-natives-syntax", - "--expose-gc", - "--no-concurrent-sweeping", - "--max-old-space-size=4096" - ]; - if (nodeVersionMajor < 18) { - flags.push("--no-randomize-hashes"); - } - if (nodeVersionMajor < 20) { - flags.push("--no-scavenge-task"); - } - return flags; -}; - const checkV8Flags = () => { const requiredFlags = getV8Flags(); const actualFlags = process.execArgv; @@ -248,6 +232,8 @@ for (const baselineInfo of baselineRevisions) { } } +const baseOutputPath = path.join(__dirname, "js", "benchmark"); + function buildConfiguration( test, baseline, @@ -385,105 +371,242 @@ const scenarios = [ } ]; -const baseOutputPath = path.join(__dirname, "js", "benchmark"); +function getStackTrace(belowFn) { + const oldLimit = Error.stackTraceLimit; + Error.stackTraceLimit = Infinity; + const dummyObject = {}; + const v8Handler = Error.prepareStackTrace; + Error.prepareStackTrace = (dummyObject, v8StackTrace) => v8StackTrace; + Error.captureStackTrace(dummyObject, belowFn || getStackTrace); + const v8StackTrace = dummyObject.stack; + Error.prepareStackTrace = v8Handler; + Error.stackTraceLimit = oldLimit; + return v8StackTrace; +} + +function getCallingFile() { + const stack = getStackTrace(); + let callingFile = stack[2].getFileName(); // [here, withCodSpeed, actual caller] + const gitDir = getGitDir(callingFile); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + if (callingFile.startsWith("file://")) { + callingFile = fileURLToPath(callingFile); + } + return path.relative(gitDir, callingFile); +} + +const taskUriMap = new WeakMap(); + +function getOrCreateUriMap(bench) { + let uriMap = taskUriMap.get(bench); + if (!uriMap) { + uriMap = new Map(); + taskUriMap.set(bench, uriMap); + } + return uriMap; +} + +function getTaskUri(bench, taskName, rootCallingFile) { + const uriMap = taskUriMap.get(bench); + return uriMap?.get(taskName) || `${rootCallingFile}::${taskName}`; +} const withCodSpeed = async (/** @type {import("tinybench").Bench} */ bench) => { - const { Measurement, getGitDir, mongoMeasurement, setupCore, teardownCore } = - await import("@codspeed/core"); + const codspeedRunnerMode = getCodspeedRunnerMode(); - if (!Measurement.isInstrumented()) { - const rawRun = bench.run; - bench.run = async () => { - console.warn( - `[CodSpeed] ${bench.tasks.length} benches detected but no instrumentation found, falling back to tinybench` - ); - return await rawRun.bind(bench)(); - }; + if (codspeedRunnerMode === "disabled") { return bench; } - const getStackTrace = (belowFn) => { - const oldLimit = Error.stackTraceLimit; - Error.stackTraceLimit = Infinity; - const dummyObject = {}; - const v8Handler = Error.prepareStackTrace; - Error.prepareStackTrace = (dummyObject, v8StackTrace) => v8StackTrace; - Error.captureStackTrace(dummyObject, belowFn || getStackTrace); - const v8StackTrace = dummyObject.stack; - Error.prepareStackTrace = v8Handler; - Error.stackTraceLimit = oldLimit; - return v8StackTrace; - }; - - const getCallingFile = () => { - const stack = getStackTrace(); - let callingFile = stack[2].getFileName(); // [here, withCodSpeed, actual caller] - const gitDir = getGitDir(callingFile); - if (gitDir === undefined) { - throw new Error("Could not find a git repository"); - } - if (callingFile.startsWith("file://")) { - callingFile = fileURLToPath(callingFile); - } - return path.relative(gitDir, callingFile); - }; - const rawAdd = bench.add; + const uriMap = getOrCreateUriMap(bench); bench.add = (name, fn, opts) => { const callingFile = getCallingFile(); - const uri = `${callingFile}::${name}`; - const options = { ...opts, uri }; - return rawAdd.bind(bench)(name, fn, options); + let uri = callingFile; + if (bench.name !== undefined) { + uri += `::${bench.name}`; + } + uri += `::${name}`; + uriMap.set(name, uri); + return rawAdd.bind(bench)(name, fn, opts); }; const rootCallingFile = getCallingFile(); - bench.run = async function run() { - const iterations = bench.opts.iterations - 1; - console.log("[CodSpeed] running"); - setupCore(); - for (const task of bench.tasks) { - await bench.opts.setup?.(task, "run"); - await task.fnOpts.beforeAll?.call(task); - const samples = []; - async function iteration() { - try { - await task.fnOpts.beforeEach?.call(task, "run"); - const start = bench.opts.now(); - await task.fn(); - samples.push(bench.opts.now() - start || 0); - await task.fnOpts.afterEach?.call(this, "run"); - } catch (err) { - if (bench.opts.throws) { - throw err; - } + + if (codspeedRunnerMode === "instrumented") { + const setupBenchRun = () => { + setupCore(); + console.log( + "[CodSpeed] running with @codspeed/tinybench (instrumented mode)" + ); + }; + const finalizeBenchRun = () => { + teardownCore(); + console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); + return bench.tasks; + }; + + const wrapFunctionWithFrame = (fn, isAsync) => { + if (isAsync) { + return async function __codspeed_root_frame__() { + await fn(); + }; + } + + return function __codspeed_root_frame__() { + fn(); + }; + }; + + const logTaskCompletion = (uri, status) => { + console.log(`[CodSpeed] ${status} ${uri}`); + }; + + const taskCompletionMessage = () => + InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + + const iterationAsync = async (task) => { + try { + await task.fnOpts.beforeEach?.call(task, "run"); + const start = bench.opts.now(); + await task.fn(); + const end = bench.opts.now() - start || 0; + await task.fnOpts.afterEach?.call(this, "run"); + return [start, end]; + } catch (err) { + if (bench.opts.throws) { + throw err; } } - while (samples.length < iterations) { - await iteration(); - } - // Codspeed Measure - const uri = - task.opts && "uri" in task.options - ? task.opts.uri - : `${rootCallingFile}::${task.name}`; - await task.fnOpts.beforeEach?.call(task); - await mongoMeasurement.start(uri); - await (async function __codspeed_root_frame__() { - Measurement.startInstrumentation(); - await task.fn(); - Measurement.stopInstrumentation(uri); - })(); - await mongoMeasurement.stop(uri); - await task.fnOpts.afterEach?.call(task); - console.log(`[Codspeed] ✔ Measured ${uri}`); - await task.fnOpts.afterAll?.call(task); + }; + const wrapWithInstrumentHooksAsync = async (fn, uri) => { + InstrumentHooks.startBenchmark(); + const result = await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + return result; + }; + + const runTaskAsync = async (task, uri) => { + const { fnOpts, fn } = task; + + // Custom setup + await bench.opts.setup?.(task, "run"); + + await fnOpts?.beforeAll?.call(task, "run"); + + // Custom warmup + const samples = []; + while (samples.length < bench.opts.iterations - 1) { + samples.push(await iterationAsync()); + } + + await optimizeFunction(async () => { + await fnOpts?.beforeEach?.call(task, "run"); + await fn(); + await fnOpts?.afterEach?.call(task, "run"); + }); + + await fnOpts?.beforeEach?.call(task, "run"); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithInstrumentHooksAsync(wrapFunctionWithFrame(fn, true), uri); + await mongoMeasurement.stop(uri); + await fnOpts?.afterEach?.call(task, "run"); + console.log(`[Codspeed] ✔ Measured ${uri}`); + await fnOpts?.afterAll?.call(task, "run"); + + // Custom teardown await bench.opts.teardown?.(task, "run"); - task.processRunResult({ latencySamples: samples }); - } - teardownCore(); - console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); - return bench.tasks; - }; + + logTaskCompletion(uri, taskCompletionMessage()); + }; + + const iteration = (task) => { + try { + task.fnOpts.beforeEach?.call(task, "run"); + const start = bench.opts.now(); + task.fn(); + const end = bench.opts.now() - start || 0; + task.fnOpts.afterEach?.call(this, "run"); + return [start, end]; + } catch (err) { + if (bench.opts.throws) { + throw err; + } + } + }; + + const wrapWithInstrumentHooks = (fn, uri) => { + InstrumentHooks.startBenchmark(); + const result = fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + return result; + }; + + const runTaskSync = (task, uri) => { + const { fnOpts, fn } = task; + + // Custom setup + bench.opts.setup?.(task, "run"); + + fnOpts?.beforeAll?.call(task, "run"); + + // Custom warmup + const samples = []; + while (samples.length < bench.opts.iterations - 1) { + samples.push(iteration()); + } + + fnOpts?.beforeEach?.call(task, "run"); + + wrapWithInstrumentHooks(wrapFunctionWithFrame(fn, false), uri); + + fnOpts?.afterEach?.call(task, "run"); + console.log(`[Codspeed] ✔ Measured ${uri}`); + fnOpts?.afterAll?.call(task, "run"); + + // Custom teardown + bench.opts.teardown?.(task, "run"); + + logTaskCompletion(uri, taskCompletionMessage()); + }; + + const finalizeAsyncRun = () => { + finalizeBenchRun(); + }; + const finalizeSyncRun = () => { + finalizeBenchRun(); + }; + + bench.run = async () => { + setupBenchRun(); + + for (const task of bench.tasks) { + const uri = getTaskUri(task.bench, task.name, rootCallingFile); + await runTaskAsync(task, uri); + } + + return finalizeAsyncRun(); + }; + + bench.runSync = () => { + setupBenchRun(); + + for (const task of bench.tasks) { + const uri = getTaskUri(task.bench, task.name, rootCallingFile); + runTaskSync(task, uri); + } + + return finalizeSyncRun(); + }; + } else if (codspeedRunnerMode === "walltime") { + // TODO + } + return bench; }; @@ -495,7 +618,6 @@ const bench = await withCodSpeed( warmupIterations: 2, iterations: 8, setup(task, mode) { - global.gc(); console.log(`Setup (${mode} mode): ${task.name}`); }, teardown(task, mode) { diff --git a/yarn.lock b/yarn.lock index 781968722..5e6da1e3a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -332,14 +332,14 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@codspeed/core@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@codspeed/core/-/core-4.0.1.tgz#91049cce17b8c1d1b4b6cbc481f5ddc1145d6e1e" - integrity sha512-fJ53arfgtzCDZa8DuGJhpTZ3Ll9A1uW5nQ2jSJnfO4Hl5MRD2cP8P4vPvIUAGbdbjwCxR1jat6cW8OloMJkJXw== +"@codspeed/core@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@codspeed/core/-/core-5.0.0.tgz#52eaf954f86ef298608b7afd5f4dcc2b28f27057" + integrity sha512-XlxFwMyYqY/7ndmzWouWLTF/zOlciUcjkjVk4GhJ+m47e1A0wTYsLYU9reY1iHc1gPyJBBTfa41oqCjgcdJs8Q== dependencies: axios "^1.4.0" find-up "^6.3.0" - form-data "^4.0.0" + form-data "^4.0.4" node-gyp-build "^4.6.0" "@cspell/cspell-bundled-dicts@9.1.3": @@ -3892,7 +3892,7 @@ fork-ts-checker-webpack-plugin@^9.0.2: semver "^7.3.5" tapable "^2.2.1" -form-data@^4.0.0, form-data@^4.0.4: +form-data@^4.0.4: version "4.0.4" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.4.tgz#784cdcce0669a9d68e94d11ac4eea98088edd2c4" integrity sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow== From 59be3dc128d4efa4cf8818c57564e5bae56b37fa Mon Sep 17 00:00:00 2001 From: alexander-akait Date: Mon, 29 Sep 2025 22:58:58 +0300 Subject: [PATCH 2/3] fix: logic --- test/BenchmarkTestCases.benchmark.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/BenchmarkTestCases.benchmark.mjs b/test/BenchmarkTestCases.benchmark.mjs index ddad81447..f7f94a630 100644 --- a/test/BenchmarkTestCases.benchmark.mjs +++ b/test/BenchmarkTestCases.benchmark.mjs @@ -500,7 +500,7 @@ const withCodSpeed = async (/** @type {import("tinybench").Bench} */ bench) => { // Custom warmup const samples = []; while (samples.length < bench.opts.iterations - 1) { - samples.push(await iterationAsync()); + samples.push(await iterationAsync(task)); } await optimizeFunction(async () => { @@ -558,7 +558,7 @@ const withCodSpeed = async (/** @type {import("tinybench").Bench} */ bench) => { // Custom warmup const samples = []; while (samples.length < bench.opts.iterations - 1) { - samples.push(iteration()); + samples.push(iteration(task)); } fnOpts?.beforeEach?.call(task, "run"); From 2be6c6e0fac9b65671fdf1f28e8a739ad21db464 Mon Sep 17 00:00:00 2001 From: alexander-akait Date: Wed, 1 Oct 2025 19:10:11 +0300 Subject: [PATCH 3/3] test: debug --- test/BenchmarkTestCases.benchmark.mjs | 63 +++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/test/BenchmarkTestCases.benchmark.mjs b/test/BenchmarkTestCases.benchmark.mjs index f7f94a630..847506431 100644 --- a/test/BenchmarkTestCases.benchmark.mjs +++ b/test/BenchmarkTestCases.benchmark.mjs @@ -489,6 +489,59 @@ const withCodSpeed = async (/** @type {import("tinybench").Bench} */ bench) => { return result; }; + function printStatus(fn) { + const status = eval("%GetOptimizationStatus(fn)"); + console.log(status.toString(2).padStart(12, "0")); + + if (status & (1 << 0)) { + console.log("is function"); + } + + if (status & (1 << 1)) { + console.log("is never optimized"); + } + + if (status & (1 << 2)) { + console.log("is always optimized"); + } + + if (status & (1 << 3)) { + console.log("is maybe deoptimized"); + } + + if (status & (1 << 4)) { + console.log("is optimized"); + } + + if (status & (1 << 5)) { + console.log("is optimized by TurboFan"); + } + + if (status & (1 << 6)) { + console.log("is interpreted"); + } + + if (status & (1 << 7)) { + console.log("is marked for optimization"); + } + + if (status & (1 << 8)) { + console.log("is marked for concurrent optimization"); + } + + if (status & (1 << 9)) { + console.log("is optimizing concurrently"); + } + + if (status & (1 << 10)) { + console.log("is executing"); + } + + if (status & (1 << 11)) { + console.log("topmost frame is turbo fanned"); + } + } + const runTaskAsync = async (task, uri) => { const { fnOpts, fn } = task; @@ -503,11 +556,15 @@ const withCodSpeed = async (/** @type {import("tinybench").Bench} */ bench) => { samples.push(await iterationAsync(task)); } - await optimizeFunction(async () => { + const toRun = async () => { await fnOpts?.beforeEach?.call(task, "run"); await fn(); await fnOpts?.afterEach?.call(task, "run"); - }); + }; + + await optimizeFunction(toRun); + + console.log("optimize", printStatus(toRun)); await fnOpts?.beforeEach?.call(task, "run"); await mongoMeasurement.start(uri); @@ -604,7 +661,7 @@ const withCodSpeed = async (/** @type {import("tinybench").Bench} */ bench) => { return finalizeSyncRun(); }; } else if (codspeedRunnerMode === "walltime") { - // TODO + // We don't need it } return bench;