Merge 44db038779 into e1afcd4cc2

.github/workflows/test.yml

@@ -97,7 +97,7 @@ jobs:
       - run: yarn link webpack --frozen-lockfile
 
       - name: Run benchmarks
-        uses: CodSpeedHQ/action@3959e9e296ef25296e93e32afcc97196f966e57f # v4.1.0
+        uses: CodSpeedHQ/action@653fdc30e6c40ffd9739e40c8a0576f4f4523ca1 # v4.0.1
         with:
           run: yarn benchmark --ci
           mode: "instrumentation"

cspell.json

@@ -284,7 +284,6 @@
     "url's",
     "valign",
     "valtype",
-    "walltime",
     "wasi",
     "wasm",
     "watchings",
@@ -305,6 +304,7 @@
     "commithash",
     "formaters",
     "akait",
+    "Akait",
     "evenstensberg",
     "Stensberg",
     "ovflowd",

declarations/WebpackOptions.d.ts

@@ -3349,6 +3349,7 @@ export interface JavascriptParserOptions {
 	 * Set the inner regular expression for partial dynamic dependencies.
 	 */
 	wrappedContextRegExp?: RegExp;
+	[k: string]: any;
 }
 /**
  * Generator options for json modules.

package.json

@@ -110,7 +110,7 @@
   "devDependencies": {
     "@babel/core": "^7.27.1",
     "@babel/preset-react": "^7.27.1",
-    "@codspeed/core": "^5.0.1",
+    "@codspeed/core": "^4.0.1",
     "@eslint/js": "^9.36.0",
     "@eslint/markdown": "^7.3.0",
     "@stylistic/eslint-plugin": "^5.4.0",

schemas/WebpackOptions.json

@@ -1786,7 +1786,7 @@
     "JavascriptParserOptions": {
       "description": "Parser options for javascript modules.",
       "type": "object",
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "amd": {
           "$ref": "#/definitions/Amd"

test/BenchmarkTestCases.benchmark.mjs

@@ -3,15 +3,6 @@ import fs from "fs/promises";
 import { Session } from "inspector";
 import path from "path";
 import { fileURLToPath, pathToFileURL } from "url";
-import {
-	InstrumentHooks,
-	getCodspeedRunnerMode,
-	getGitDir,
-	getV8Flags,
-	mongoMeasurement,
-	setupCore,
-	teardownCore
-} from "@codspeed/core";
 import { simpleGit } from "simple-git";
 import { Bench, hrtimeNow } from "tinybench";
 
@@ -21,6 +12,32 @@ const git = simpleGit(rootPath);
 
 const REV_LIST_REGEXP = /^([a-f0-9]+)\s*([a-f0-9]+)\s*([a-f0-9]+)?\s*$/;
 
+const getV8Flags = () => {
+	const nodeVersionMajor = Number.parseInt(
+		process.version.slice(1).split(".")[0],
+		10
+	);
+	const flags = [
+		"--hash-seed=1",
+		"--random-seed=1",
+		"--no-opt",
+		"--predictable",
+		"--predictable-gc-schedule",
+		"--interpreted-frames-native-stack",
+		"--allow-natives-syntax",
+		"--expose-gc",
+		"--no-concurrent-sweeping",
+		"--max-old-space-size=4096"
+	];
+	if (nodeVersionMajor < 18) {
+		flags.push("--no-randomize-hashes");
+	}
+	if (nodeVersionMajor < 20) {
+		flags.push("--no-scavenge-task");
+	}
+	return flags;
+};
+
 const checkV8Flags = () => {
 	const requiredFlags = getV8Flags();
 	const actualFlags = process.execArgv;
@@ -231,8 +248,6 @@ for (const baselineInfo of baselineRevisions) {
 	}
 }
 
-const baseOutputPath = path.join(__dirname, "js", "benchmark");
-
 function buildConfiguration(
 	test,
 	baseline,
@@ -370,239 +385,105 @@ const scenarios = [
 	}
 ];
 
-function getStackTrace(belowFn) {
+const baseOutputPath = path.join(__dirname, "js", "benchmark");
-	const oldLimit = Error.stackTraceLimit;
-	Error.stackTraceLimit = Infinity;
-	const dummyObject = {};
-	const v8Handler = Error.prepareStackTrace;
-	Error.prepareStackTrace = (dummyObject, v8StackTrace) => v8StackTrace;
-	Error.captureStackTrace(dummyObject, belowFn || getStackTrace);
-	const v8StackTrace = dummyObject.stack;
-	Error.prepareStackTrace = v8Handler;
-	Error.stackTraceLimit = oldLimit;
-	return v8StackTrace;
-}
-
-function getCallingFile() {
-	const stack = getStackTrace();
-	let callingFile = stack[2].getFileName(); // [here, withCodSpeed, actual caller]
-	const gitDir = getGitDir(callingFile);
-	if (gitDir === undefined) {
-		throw new Error("Could not find a git repository");
-	}
-	if (callingFile.startsWith("file://")) {
-		callingFile = fileURLToPath(callingFile);
-	}
-	return path.relative(gitDir, callingFile);
-}
-
-const taskUriMap = new WeakMap();
-
-function getOrCreateUriMap(bench) {
-	let uriMap = taskUriMap.get(bench);
-	if (!uriMap) {
-		uriMap = new Map();
-		taskUriMap.set(bench, uriMap);
-	}
-	return uriMap;
-}
-
-function getTaskUri(bench, taskName, rootCallingFile) {
-	const uriMap = taskUriMap.get(bench);
-	return uriMap?.get(taskName) || `${rootCallingFile}::${taskName}`;
-}
 
 const withCodSpeed = async (/** @type {import("tinybench").Bench} */ bench) => {
-	const codspeedRunnerMode = getCodspeedRunnerMode();
+	const { Measurement, getGitDir, mongoMeasurement, setupCore, teardownCore } =
+		await import("@codspeed/core");
 
-	if (codspeedRunnerMode === "disabled") {
+	if (!Measurement.isInstrumented()) {
+		const rawRun = bench.run;
+		bench.run = async () => {
+			console.warn(
+				`[CodSpeed] ${bench.tasks.length} benches detected but no instrumentation found, falling back to tinybench`
+			);
+			return await rawRun.bind(bench)();
+		};
 		return bench;
 	}
 
+	const getStackTrace = (belowFn) => {
+		const oldLimit = Error.stackTraceLimit;
+		Error.stackTraceLimit = Infinity;
+		const dummyObject = {};
+		const v8Handler = Error.prepareStackTrace;
+		Error.prepareStackTrace = (dummyObject, v8StackTrace) => v8StackTrace;
+		Error.captureStackTrace(dummyObject, belowFn || getStackTrace);
+		const v8StackTrace = dummyObject.stack;
+		Error.prepareStackTrace = v8Handler;
+		Error.stackTraceLimit = oldLimit;
+		return v8StackTrace;
+	};
+
+	const getCallingFile = () => {
+		const stack = getStackTrace();
+		let callingFile = stack[2].getFileName(); // [here, withCodSpeed, actual caller]
+		const gitDir = getGitDir(callingFile);
+		if (gitDir === undefined) {
+			throw new Error("Could not find a git repository");
+		}
+		if (callingFile.startsWith("file://")) {
+			callingFile = fileURLToPath(callingFile);
+		}
+		return path.relative(gitDir, callingFile);
+	};
+
 	const rawAdd = bench.add;
-	const uriMap = getOrCreateUriMap(bench);
 	bench.add = (name, fn, opts) => {
 		const callingFile = getCallingFile();
-		let uri = callingFile;
+		const uri = `${callingFile}::${name}`;
-		if (bench.name !== undefined) {
+		const options = { ...opts, uri };
-			uri += `::${bench.name}`;
+		return rawAdd.bind(bench)(name, fn, options);
-		}
-		uri += `::${name}`;
-		uriMap.set(name, uri);
-		return rawAdd.bind(bench)(name, fn, opts);
 	};
 	const rootCallingFile = getCallingFile();
-
+	bench.run = async function run() {
-	if (codspeedRunnerMode === "instrumented") {
+		const iterations = bench.opts.iterations - 1;
-		const setupBenchRun = () => {
+		console.log("[CodSpeed] running");
-			setupCore();
+		setupCore();
-			console.log(
+		for (const task of bench.tasks) {
-				"[CodSpeed] running with @codspeed/tinybench (instrumented mode)"
-			);
-		};
-		const finalizeBenchRun = () => {
-			teardownCore();
-			console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
-			return bench.tasks;
-		};
-
-		const wrapFunctionWithFrame = (fn, isAsync) => {
-			if (isAsync) {
-				return async function __codspeed_root_frame__() {
-					await fn();
-				};
-			}
-
-			return function __codspeed_root_frame__() {
-				fn();
-			};
-		};
-
-		const logTaskCompletion = (uri, status) => {
-			console.log(`[CodSpeed] ${status} ${uri}`);
-		};
-
-		const taskCompletionMessage = () =>
-			InstrumentHooks.isInstrumented() ? "Measured" : "Checked";
-
-		const iterationAsync = async (task) => {
-			try {
-				await task.fnOpts.beforeEach?.call(task, "run");
-				const start = bench.opts.now();
-				await task.fn();
-				const end = bench.opts.now() - start || 0;
-				await task.fnOpts.afterEach?.call(this, "run");
-				return [start, end];
-			} catch (err) {
-				if (bench.opts.throws) {
-					throw err;
-				}
-			}
-		};
-
-		const wrapWithInstrumentHooksAsync = async (fn, uri) => {
-			InstrumentHooks.startBenchmark();
-			const result = await fn();
-			InstrumentHooks.stopBenchmark();
-			InstrumentHooks.setExecutedBenchmark(process.pid, uri);
-			return result;
-		};
-
-		const runTaskAsync = async (task, uri) => {
-			const { fnOpts, fn } = task;
-
-			// Custom setup
 			await bench.opts.setup?.(task, "run");
-
+			await task.fnOpts.beforeAll?.call(task);
-			await fnOpts?.beforeAll?.call(task, "run");
-
-			// Custom warmup
-			// We don't run `optimizeFunction` because our function is never optimized, instead we just warmup webpack
 			const samples = [];
-
+			async function iteration() {
-			while (samples.length < bench.opts.iterations - 1) {
+				try {
-				samples.push(await iterationAsync(task));
+					await task.fnOpts.beforeEach?.call(task, "run");
-			}
+					const start = bench.opts.now();
-
+					await task.fn();
-			await fnOpts?.beforeEach?.call(task, "run");
+					samples.push(bench.opts.now() - start || 0);
-			await mongoMeasurement.start(uri);
+					await task.fnOpts.afterEach?.call(this, "run");
-			global.gc?.();
+				} catch (err) {
-			await wrapWithInstrumentHooksAsync(wrapFunctionWithFrame(fn, true), uri);
+					if (bench.opts.throws) {
-			await mongoMeasurement.stop(uri);
+						throw err;
-			await fnOpts?.afterEach?.call(task, "run");
+					}
-			console.log(`[Codspeed] ✔ Measured ${uri}`);
-			await fnOpts?.afterAll?.call(task, "run");
-
-			// Custom teardown
-			await bench.opts.teardown?.(task, "run");
-
-			logTaskCompletion(uri, taskCompletionMessage());
-		};
-
-		const iteration = (task) => {
-			try {
-				task.fnOpts.beforeEach?.call(task, "run");
-				const start = bench.opts.now();
-				task.fn();
-				const end = bench.opts.now() - start || 0;
-				task.fnOpts.afterEach?.call(this, "run");
-				return [start, end];
-			} catch (err) {
-				if (bench.opts.throws) {
-					throw err;
 				}
 			}
-		};
+			while (samples.length < iterations) {
-
+				await iteration();
-		const wrapWithInstrumentHooks = (fn, uri) => {
-			InstrumentHooks.startBenchmark();
-			const result = fn();
-			InstrumentHooks.stopBenchmark();
-			InstrumentHooks.setExecutedBenchmark(process.pid, uri);
-			return result;
-		};
-
-		const runTaskSync = (task, uri) => {
-			const { fnOpts, fn } = task;
-
-			// Custom setup
-			bench.opts.setup?.(task, "run");
-
-			fnOpts?.beforeAll?.call(task, "run");
-
-			// Custom warmup
-			const samples = [];
-
-			while (samples.length < bench.opts.iterations - 1) {
-				samples.push(iteration(task));
 			}
-
+			// Codspeed Measure
-			fnOpts?.beforeEach?.call(task, "run");
+			const uri =
-
+				task.opts && "uri" in task.options
-			wrapWithInstrumentHooks(wrapFunctionWithFrame(fn, false), uri);
+					? task.opts.uri
-
+					: `${rootCallingFile}::${task.name}`;
-			fnOpts?.afterEach?.call(task, "run");
+			await task.fnOpts.beforeEach?.call(task);
+			await mongoMeasurement.start(uri);
+			await (async function __codspeed_root_frame__() {
+				Measurement.startInstrumentation();
+				await task.fn();
+				Measurement.stopInstrumentation(uri);
+			})();
+			await mongoMeasurement.stop(uri);
+			await task.fnOpts.afterEach?.call(task);
 			console.log(`[Codspeed] ✔ Measured ${uri}`);
-			fnOpts?.afterAll?.call(task, "run");
+			await task.fnOpts.afterAll?.call(task);
-
-			// Custom teardown
-			bench.opts.teardown?.(task, "run");
-
-			logTaskCompletion(uri, taskCompletionMessage());
-		};
-
-		const finalizeAsyncRun = () => {
-			finalizeBenchRun();
-		};
-		const finalizeSyncRun = () => {
-			finalizeBenchRun();
-		};
-
-		bench.run = async () => {
-			setupBenchRun();
-
-			for (const task of bench.tasks) {
-				const uri = getTaskUri(task.bench, task.name, rootCallingFile);
-				await runTaskAsync(task, uri);
-			}
-
-			return finalizeAsyncRun();
-		};
-
-		bench.runSync = () => {
-			setupBenchRun();
-
-			for (const task of bench.tasks) {
-				const uri = getTaskUri(task.bench, task.name, rootCallingFile);
-				runTaskSync(task, uri);
-			}
-
-			return finalizeSyncRun();
-		};
-	} else if (codspeedRunnerMode === "walltime") {
-		// We don't need it
-	}
 
+			await bench.opts.teardown?.(task, "run");
+			task.processRunResult({ latencySamples: samples });
+		}
+		teardownCore();
+		console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
+		return bench.tasks;
+	};
 	return bench;
 };
 
@@ -614,6 +495,7 @@ const bench = await withCodSpeed(
 		warmupIterations: 2,
 		iterations: 8,
 		setup(task, mode) {
+			global.gc();
 			console.log(`Setup (${mode} mode): ${task.name}`);
 		},
 		teardown(task, mode) {

types.d.ts

@@ -8195,6 +8195,8 @@ declare class JavascriptParser extends ParserClass {
  * Parser options for javascript modules.
  */
 declare interface JavascriptParserOptions {
+	[index: string]: any;
+
 	/**
 	 * Set the value of `require.amd` and `define.amd`. Or disable AMD support.
 	 */

yarn.lock

@@ -332,14 +332,14 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@codspeed/core@^5.0.1":
+"@codspeed/core@^4.0.1":
-  version "5.0.1"
+  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/@codspeed/core/-/core-5.0.1.tgz#6145c898a86a6d56a169611c3e9657a8b97c7642"
+  resolved "https://registry.yarnpkg.com/@codspeed/core/-/core-4.0.1.tgz#91049cce17b8c1d1b4b6cbc481f5ddc1145d6e1e"
-  integrity sha512-4g5ZyFAin8QywK4+0FK1uXG3GLRPu0oc3xbP+7OUhhFxbwpzFuaJtKmnTofMqLy9/pHH6Bl/7H0/DTVH3cpFkA==
+  integrity sha512-fJ53arfgtzCDZa8DuGJhpTZ3Ll9A1uW5nQ2jSJnfO4Hl5MRD2cP8P4vPvIUAGbdbjwCxR1jat6cW8OloMJkJXw==
   dependencies:
     axios "^1.4.0"
     find-up "^6.3.0"
-    form-data "^4.0.4"
+    form-data "^4.0.0"
     node-gyp-build "^4.6.0"
 
 "@cspell/cspell-bundled-dicts@9.1.3":
@@ -3878,7 +3878,7 @@ fork-ts-checker-webpack-plugin@^9.0.2:
     semver "^7.3.5"
     tapable "^2.2.1"
 
-form-data@^4.0.4:
+form-data@^4.0.0, form-data@^4.0.4:
   version "4.0.4"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.4.tgz#784cdcce0669a9d68e94d11ac4eea98088edd2c4"
   integrity sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==