chore: add more benchmark cases (#19481)

.github/workflows/test.yml

@@ -60,6 +60,10 @@ jobs:
       - run: node -e "const content = require('./package.json');delete content.devDependencies;require('fs').writeFileSync('package.json', JSON.stringify(content, null, 2));"
       - run: yarn install --production --frozen-lockfile
   benchmark:
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1/4, 2/4, 3/4, 4/4]
     runs-on: ubuntu-latest
     permissions:
       issues: write
@@ -84,6 +88,7 @@ jobs:
           token: ${{ secrets.CODSPEED_TOKEN }}
         env:
           CODSPEED: 1
+          SHARD: ${{ matrix.shard }}
   basic:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -5,8 +5,7 @@
 /test/temp
 /test/ChangesAndRemovals
 /test/**/dev-defaults.webpack.lock
-/benchmark/js
-/benchmark/fixtures
+/test/**/generated/**
 /examples/**/dist
 /examples/nodejs-addons/build/**
 /assembly/**/*.wat

.prettierignore

@@ -14,6 +14,8 @@ test/**/*.*
 !test/**/infrastructure-log.js
 !test/*.md
 !test/helpers/*.*
+!test/benchmarkCases/**/*.mjs
+!test/_helpers/**/*.mjs
 test/js/**/*.*
 
 # Ignore some folders

benchmark/benchmark.js

@@ -1,239 +0,0 @@
-const path = require("path");
-const fs = require("fs");
-const Benchmark = require("benchmark");
-const webpack = require("../");
-const fixtures = path.join(__dirname, "fixtures");
-const outputPath = path.join(__dirname, "js");
-
-const benchmarkOptions = {
-	defer: true,
-	onCycle: function () {
-		process.stderr.write(".");
-	},
-	minSamples: 10
-};
-
-function runTimes(compiler, times, deferred) {
-	fs.writeFileSync(
-		path.join(fixtures, "0.js"),
-		"module.exports = " + Math.random(),
-		"utf-8"
-	);
-	compiler.run(err => {
-		if (err) throw err;
-		if (times === 1) deferred.resolve();
-		else runTimes(compiler, times - 1, deferred);
-	});
-}
-
-const tests = {
-	"normal build": [
-		[0, 1, 5, 10, 50, 100, 200],
-		(size, deferred) => {
-			webpack(
-				{
-					context: fixtures,
-					entry: `./${size}.js`,
-					output: {
-						path: outputPath,
-						filename: "bundle.js"
-					}
-				},
-				err => {
-					if (err) throw err;
-					deferred.resolve();
-				}
-			);
-		}
-	],
-	"eval dev build": [
-		[0, 1, 2, 5, 10, 15],
-		(size, deferred) => {
-			webpack(
-				{
-					context: fixtures,
-					entry: `./${size}.big.js`,
-					output: {
-						path: outputPath,
-						filename: "bundle.js"
-					},
-					devtool: "eval"
-				},
-				err => {
-					if (err) throw err;
-					deferred.resolve();
-				}
-			);
-		}
-	],
-	"sourcemap build": [
-		[0, 1, 2, 5, 10, 15],
-		(size, deferred) => {
-			webpack(
-				{
-					context: fixtures,
-					entry: `./${size}.big.js`,
-					output: {
-						path: outputPath,
-						filename: "bundle.js"
-					},
-					devtool: "source-map"
-				},
-				err => {
-					if (err) throw err;
-					deferred.resolve();
-				}
-			);
-		}
-	],
-	"cheap sourcemap build": [
-		[0, 1, 2, 5, 10, 15],
-		(size, deferred) => {
-			webpack(
-				{
-					context: fixtures,
-					entry: `./${size}.big.js`,
-					output: {
-						path: outputPath,
-						filename: "bundle.js"
-					},
-					devtool: "cheap-source-map"
-				},
-				err => {
-					if (err) throw err;
-					deferred.resolve();
-				}
-			);
-		}
-	],
-	"build w/ chunks": [
-		[0, 1, 5, 10, 50, 100, 200],
-		(size, deferred) => {
-			webpack(
-				{
-					context: fixtures,
-					entry: `./${size}.async.js`,
-					output: {
-						path: outputPath,
-						filename: "bundle.js"
-					}
-				},
-				err => {
-					if (err) throw err;
-					deferred.resolve();
-				}
-			);
-		}
-	],
-	incremental: [
-		[0, 1, 5, 10, 50, 100, 200],
-		(size, deferred) => {
-			var compiler = webpack({
-				cache: true,
-				context: fixtures,
-				entry: "./" + size + ".js",
-				output: {
-					path: outputPath,
-					filename: "bundle.js"
-				}
-			});
-			runTimes(compiler, 2, deferred);
-		}
-	],
-	"incremental cheap sourcemap": [
-		[1, 2, 3, 4, 5, 6],
-		(size, deferred) => {
-			var compiler = webpack({
-				cache: true,
-				context: fixtures,
-				entry: "./200.js",
-				output: {
-					path: outputPath,
-					filename: "bundle.js"
-				},
-				devtool: "cheap-source-map"
-			});
-			runTimes(compiler, size, deferred);
-		}
-	],
-	incremental2: [
-		[0, 1, 5, 10, 50, 100, 200],
-		(size, deferred) => {
-			var compiler = webpack({
-				cache: true,
-				context: fixtures,
-				entry: `./${size}.js`,
-				output: {
-					path: outputPath,
-					filename: "bundle.js"
-				}
-			});
-			runTimes(compiler, 3, deferred);
-		}
-	],
-	incremental4: [
-		[0, 1, 5, 10, 50, 100, 200],
-		(size, deferred) => {
-			var compiler = webpack({
-				cache: true,
-				context: fixtures,
-				entry: `./${size}.js`,
-				output: {
-					path: outputPath,
-					filename: "bundle.js"
-				}
-			});
-			runTimes(compiler, 5, deferred);
-		}
-	],
-	incremental16: [
-		[0, 1, 5, 10, 50, 100, 200],
-		(size, deferred) => {
-			var compiler = webpack({
-				cache: true,
-				context: fixtures,
-				entry: `./${size}.js`,
-				output: {
-					path: outputPath,
-					filename: "bundle.js"
-				}
-			});
-			runTimes(compiler, 17, deferred);
-		}
-	]
-};
-
-const suite = new Benchmark.Suite();
-
-Object.keys(tests)
-	.filter(name =>
-		process.argv.length > 2 ? name.includes(process.argv[2]) : true
-	)
-	.forEach(name => {
-		const test = tests[name];
-		test[0].forEach(size => {
-			suite.add(
-				`${name} ${size}`,
-				deferred => {
-					test[1](size, deferred);
-				},
-				benchmarkOptions
-			);
-		});
-	});
-
-suite.on("cycle", event => {
-	process.stderr.write("\n");
-	const b = event.target;
-	console.log(
-		b.name +
-			"\t" +
-			Math.floor(1000 * (b.stats.mean - b.stats.moe)) +
-			"\t" +
-			Math.floor(1000 * (b.stats.mean + b.stats.moe))
-	);
-});
-
-suite.run({
-	async: true
-});

benchmark/createBenchmark.js

@@ -1,30 +0,0 @@
-const webpack = require("../");
-const path = require("path");
-
-webpack(
-	{
-		context: __dirname,
-		entry: "./createBenchmark/entry.js",
-		output: {
-			path: __dirname,
-			filename: "benchmark-bundle.js"
-		},
-		target: "node",
-		node: {
-			__dirname: false
-		},
-		optimization: {
-			moduleIds: "named"
-		},
-		plugins: [
-			new webpack.IgnorePlugin(/^(fsevents|terser)$/),
-			new webpack.NormalModuleReplacementPlugin(
-				/^.\/loadLoader$/,
-				path.resolve(__dirname, "./createBenchmark/loadLoader")
-			)
-		]
-	},
-	(err, stats) => {
-		console.log(stats.toString());
-	}
-);

benchmark/createBenchmark/entry.js

@@ -1,26 +0,0 @@
-const webpack = require("../../");
-const path = require("path");
-
-const testCase = process.argv[2];
-
-const config = {
-	context: __dirname,
-	entry: `./${testCase}`,
-	output: {
-		path: path.resolve(__dirname, "output-" + testCase)
-	},
-	devtool: process.argv[3]
-};
-
-const compiler = webpack(config);
-compiler.run((err, stats) => {
-	if (err) {
-		console.error(err);
-	} else {
-		console.log(
-			stats.toString({
-				errorDetails: true
-			})
-		);
-	}
-});

benchmark/createBenchmark/loadLoader.js

@@ -1,3 +0,0 @@
-module.exports = (loader, callback) => {
-	callback(new Error("Loaders are not supported"));
-};

benchmark/createFixtures.js

@@ -1,59 +0,0 @@
-const path = require("path");
-const fs = require("fs");
-
-const fixtures = path.join(__dirname, "fixtures");
-
-try {
-	fs.mkdirSync(fixtures);
-} catch (e) {
-	// The directory already exists
-}
-
-function generateRequireString(conditional, suffix) {
-	const prefixedSuffix = suffix ? `.${suffix}` : "";
-	return `require(${JSON.stringify(`./${conditional}${prefixedSuffix}.js`)});`;
-}
-
-for (let i = 0; i < 10000; i++) {
-	const source = [];
-	if (i > 8) source.push(generateRequireString((i / 8) | 0));
-	if (i > 4) source.push(generateRequireString((i / 4) | 0));
-	if (i > 2) source.push(generateRequireString((i / 2) | 0));
-	if (i > 0) source.push(generateRequireString(i - 1));
-	source.push("module.exports = " + i + ";");
-	fs.writeFileSync(path.join(fixtures, i + ".js"), source.join("\n"), "utf-8");
-}
-
-for (let i = 0; i < 10000; i++) {
-	const source = [];
-	source.push("require.ensure([], function(require) {");
-	if (i > 8) source.push(generateRequireString((i / 8) | 0, "async"));
-	if (i > 4) source.push(generateRequireString((i / 4) | 0, "async"));
-	if (i > 2) source.push(generateRequireString((i / 2) | 0, "async"));
-	if (i > 0) source.push(generateRequireString(i - 1, "async"));
-	source.push("});");
-	source.push("module.exports = " + i + ";");
-	fs.writeFileSync(
-		path.join(fixtures, i + ".async.js"),
-		source.join("\n"),
-		"utf-8"
-	);
-}
-
-for (let i = 0; i < 100; i++) {
-	const source = [];
-	if (i > 8) source.push(generateRequireString((i / 8) | 0, "big"));
-	if (i > 4) source.push(generateRequireString((i / 4) | 0, "big"));
-	if (i > 2) source.push(generateRequireString((i / 2) | 0, "big"));
-	if (i > 0) source.push(generateRequireString(i - 1, "big"));
-	for (let j = 0; j < 300; j++)
-		source.push(
-			"if(Math.random())hello.world();test.a.b.c.d();x(1,2,3,4);var a,b,c,d,e,f;"
-		);
-	source.push("module.exports = " + i + ";");
-	fs.writeFileSync(
-		path.join(fixtures, i + ".big.js"),
-		source.join("\n"),
-		"utf-8"
-	);
-}

benchmark/createFixtures2.js

@@ -1,69 +0,0 @@
-const path = require("path");
-const fs = require("fs");
-
-const fixtures = path.join(__dirname, "fixtures");
-
-try {
-	fs.mkdirSync(fixtures);
-} catch (e) {
-	// The directory already exists
-}
-
-function genModule(prefix, depth, asyncDepth, multiplex, r, circular) {
-	const source = [];
-	const isAsync = depth >= asyncDepth;
-	if (!isAsync) circular.push(path.resolve(fixtures, prefix + "/index.js"));
-	source.push("(function() {");
-	const m = (r % multiplex) + 1;
-	let sum = 1;
-	let item;
-	try {
-		fs.mkdirSync(path.resolve(fixtures, prefix));
-	} catch (e) {
-		// The directory already exists
-	}
-	if (depth > 0) {
-		for (let i = 0; i < m; i++) {
-			sum += genModule(
-				prefix + "/" + i,
-				depth - 1,
-				asyncDepth,
-				multiplex,
-				(r + i + depth) * m + i + depth,
-				circular
-			);
-			source.push("require(" + JSON.stringify("./" + i) + ");");
-			if (i === 0) {
-				if (isAsync) source.push("}); require.ensure([], function() {");
-			}
-		}
-		item = circular[r % circular.length];
-	}
-	source.push("}, " + JSON.stringify(prefix) + ");");
-	if (item) source.push("require(" + JSON.stringify(item) + ");");
-	source.push("module.exports = " + JSON.stringify(prefix) + ";");
-	fs.writeFileSync(
-		path.resolve(fixtures, prefix + "/index.js"),
-		source.join("\n"),
-		"utf-8"
-	);
-	return sum;
-}
-
-for (let i = 2; i < 14; i++) {
-	const count = genModule("tree-" + i, 6, 100, i, 0, []);
-	console.log("generated tree", i, count);
-}
-
-for (let i = 2; i < 14; i++) {
-	const count = genModule("async-tree-" + i, 6, 1, i, 0, []);
-	console.log("generated async tree", i, count);
-}
-
-const a = genModule("module-async", 7, 1, 3, 2, []);
-
-const b = genModule("module-big-async", 5, 2, 9, 2, []);
-
-const c = genModule("module-broad-async", 3, 3, 20, 10, []);
-
-console.log("generated modules", a, b, c);

benchmark/createTestCases.js

@@ -1,83 +0,0 @@
-let avgJs = `
-const str = "we" + "do" + "some" + "ops";
-for(const x of str.split("")) {
-	if(x.charCodeAt(0) > 40) {
-		console.log("omg");
-	} else {
-		console.log(Math.random() * 2 + 3 * 2);
-	}
-}
-
-// Some comment
-switch(a.b.c.d.f.e.g.h.i) {
-	case true:
-		break;
-	case "magic":
-		throw new Error("Error!");
-	case 9:
-		(function() {
-			// extra scope
-			var x = 123;
-			var y = 456;
-			var z = x + z * x / y;
-			x && y && (z = x ? y : x);
-		}())
-}
-
-function a() {}
-function b() {}
-function c() {}
-function d() {}
-function e() {}
-function f() {}
-`;
-
-for (let i = 0; i < 2; i++) {
-	avgJs += `(function() {${avgJs}}());`;
-}
-
-const fs = require("fs");
-const root = __dirname;
-
-createTree(fs, 100, `${root}/modules-100`);
-createTree(fs, 500, `${root}/modules-500`);
-createTree(fs, 1000, `${root}/modules-1000`);
-createTree(fs, 3000, `${root}/modules-3000`);
-createTree(fs, 5000, `${root}/modules-5000`);
-
-function createTree(fs, count, folder) {
-	fs.mkdirSync(folder);
-	let remaining = count - 1;
-
-	function make(prefix, count, depth) {
-		if (count === 0) {
-			fs.writeFileSync(`${folder}/${prefix}.js`, `export default 1;\n${avgJs}`);
-		} else {
-			const list = [];
-			for (let i = 0; i < count; i++) {
-				if (remaining-- <= 0) break;
-				if (depth <= 4 && i >= 3 && i <= 4) {
-					list.push(
-						`const module${i} = import("./${prefix}-${i}");\ncounter += module${i};`
-					);
-				} else {
-					list.push(
-						`import module${i} from "./${prefix}-${i}";\ncounter += module${i};`
-					);
-				}
-				make(
-					`${prefix}-${i}`,
-					depth > 4 || count > 30 ? 0 : count + depth + Math.pow(i, 2),
-					depth + 1
-				);
-			}
-			fs.writeFileSync(
-				`${folder}/${prefix}.js`,
-				`let counter = 0;\n${list.join(
-					"\n"
-				)};\nexport default counter;\n${avgJs}`
-			);
-		}
-	}
-	make("index", 2, 0);
-}

eslint.config.mjs

@@ -27,6 +27,8 @@ export default defineConfig([
 		"!test/**/deprecations.js",
 		"!test/**/infrastructure-log.js",
 		"!test/helpers/*.*",
+		"!test/benchmarkCases/**/*.mjs",
+		"!test/_helpers/**/*.mjs",
 		"test/js/**/*.*",
 
 		// Ignore some folders

package.json

@@ -103,6 +103,7 @@
     "strip-ansi": "^6.0.0",
     "style-loader": "^4.0.0",
     "terser": "^5.38.1",
+    "three": "^0.176.0",
     "toml": "^3.0.0",
     "tooling": "webpack/tooling#v1.23.7",
     "ts-loader": "^9.5.1",

test/BenchmarkTestCases.benchmark.mjs

@@ -4,9 +4,8 @@ import path from "path";
 import fs from "fs/promises";
 import { constants } from "fs";
 import Benchmark from "benchmark";
-import { remove } from "./helpers/remove.js";
 import { dirname } from "path";
-import { fileURLToPath } from "url";
+import { fileURLToPath, pathToFileURL } from "url";
 import simpleGit from "simple-git";
 import { withCodSpeed } from "@codspeed/benchmark.js-plugin";
 
@@ -65,18 +64,21 @@ async function getHead(revList) {
 		return process.env.HEAD;
 	}
 
+	// On CI we take the latest commit `merge commit` as a head
 	if (revList[3]) {
 		return revList[3];
 	}
 
+	// Otherwise we take the latest commit
 	return revList[1];
 }
 
 /**
+ * @param {string} head head
  * @param {(string | undefined)[]} revList rev list
  * @returns {Promise<string>} base
  */
-async function getBase(revList) {
+async function getBase(head, revList) {
 	if (typeof process.env.BASE !== "undefined") {
 		return process.env.BASE;
 	}
@@ -102,6 +104,10 @@ async function getBase(revList) {
 			throw new Error("No parent commit found");
 		}
 
+		if (head === revList[1]) {
+			return revList[2];
+		}
+
 		return revList[1];
 	}
 
@@ -134,7 +140,7 @@ async function getBaselineRevs() {
 		];
 	}
 
-	const base = await getBase(revList);
+	const base = await getBase(head, revList);
 
 	if (!head || !base) {
 		throw new Error("No baseline found");
@@ -200,29 +206,13 @@ for (const baselineInfo of baselineRevisions) {
 		baselines.push({
 			name: baselineInfo.name,
 			rev: baselineRevision,
-			webpack: async config => {
+			webpack: async () => {
 				const webpack = (
 					await import(
-						path.resolve(
-							baselinePath,
-							`./lib/index.js?nocache=${Math.random()}`
-						)
+						pathToFileURL(path.resolve(baselinePath, `./lib/index.js`))
 					)
 				).default;
 
-				await new Promise((resolve, reject) => {
-					const warmupCompiler = webpack(config, (err, _stats) => {
-						if (err) {
-							reject(err);
-							return;
-						}
-
-						warmupCompiler.purgeInputFileSystem();
-
-						resolve();
-					});
-				});
-
 				return webpack;
 			}
 		});
@@ -257,52 +247,185 @@ for (const baselineInfo of baselineRevisions) {
 	}
 }
 
-async function registerBenchmarks(suite, test, baselines) {
-	for (const baseline of baselines) {
-		const outputDirectory = path.join(
-			__dirname,
-			"js",
-			"benchmark",
-			`baseline-${baseline.name}`,
-			test
-		);
-		const testDirectory = path.join(casesPath, test);
-		const config =
-			(
-				await import(
-					path.join(testDirectory, `webpack.config.js?nocache=${Math.random()}`)
-				)
-			).default || {};
+const scenarios = [
+	{
+		name: "mode-development",
+		mode: "development"
+	},
+	{
+		name: "mode-development-rebuild",
+		mode: "development",
+		watch: true
+	},
+	{
+		name: "mode-production",
+		mode: "production"
+	}
+];
 
-		config.mode = config.mode || "production";
+function buildConfiguration(
+	test,
+	baseline,
+	realConfig,
+	scenario,
+	testDirectory
+) {
+	const { watch, ...rest } = scenario;
+	const config = structuredClone({ ...realConfig, ...rest });
+
+	config.entry = config.entry || "./index.js";
+	config.devtool = config.devtool || false;
+	config.name = `${test}-${baseline.name}-${scenario.name}`;
+	config.context = testDirectory;
+	config.performance = false;
 	config.output = config.output || {};
+	config.output.path = path.join(
+		baseOutputPath,
+		test,
+		`scenario-${scenario.name}`,
+		`baseline-${baseline.name}`
+	);
 
-		if (!config.context) config.context = testDirectory;
-		if (!config.output.path) config.output.path = outputDirectory;
-
-		const suiteName = `benchmark "${test}"${CODSPEED ? "" : ` ${baseline.name} (${baseline.rev})`}`;
-		const webpack = await baseline.webpack(config);
-
-		suite.add(suiteName, {
-			baseTestName: test,
-			defer: true,
-			fn(deferred) {
-				const compiler = webpack(config, (err, stats) => {
-					compiler.purgeInputFileSystem();
-
-					if (err) {
-						throw err;
+	if (config.cache) {
+		config.cache.cacheDirectory = path.resolve(config.output.path, ".cache");
 	}
 
-					if (stats.hasErrors()) {
-						throw new Error(stats.toString());
+	return config;
+}
+
+function warmupCompiler(compiler) {
+	return new Promise((resolve, reject) => {
+		compiler.run((err, stats) => {
+			if (err) {
+				reject(err);
+			}
+
+			if (stats.hasWarnings() || stats.hasErrors()) {
+				reject(new Error(stats.toString()));
+			}
+
+			compiler.close(closeErr => {
+				if (closeErr) {
+					reject(closeErr);
+				}
+
+				resolve();
+			});
+		});
+	});
+}
+
+function runWatch(compiler) {
+	return new Promise((resolve, reject) => {
+		const watching = compiler.watch({}, (err, stats) => {
+			if (err) {
+				reject(err);
+			}
+
+			if (stats.hasWarnings() || stats.hasErrors()) {
+				reject(new Error(stats.toString()));
+			}
+
+			resolve(watching);
+		});
+	});
+}
+
+const baseOutputPath = path.join(__dirname, "js", "benchmark");
+
+async function registerSuite(suite, test, baselines) {
+	const testDirectory = path.join(casesPath, test);
+	const setupPath = path.resolve(testDirectory, "setup.mjs");
+
+	let needSetup = true;
+
+	try {
+		await fs.access(setupPath, constants.R_OK);
+	} catch (_err) {
+		needSetup = false;
+	}
+
+	if (needSetup) {
+		await import(`${pathToFileURL(setupPath)}?date=${Date.now()}`);
+	}
+
+	const realConfig = (
+		await import(
+			`${pathToFileURL(path.join(testDirectory, `webpack.config.js`))}`
+		)
+	).default;
+
+	await Promise.all(
+		baselines.map(async baseline => {
+			const webpack = await baseline.webpack();
+
+			await Promise.all(
+				scenarios.map(async scenario => {
+					const config = buildConfiguration(
+						test,
+						baseline,
+						realConfig,
+						scenario,
+						testDirectory
+					);
+
+					// Warmup and also prebuild cache
+					await warmupCompiler(webpack(config));
+
+					// Make an extra run for watching tests
+					let watching;
+
+					if (scenario.watch) {
+						watching = await runWatch(webpack(config));
+					}
+
+					const stringifiedScenario = JSON.stringify(scenario);
+					const suiteName = `benchmark "${test}", scenario '${stringifiedScenario}'${CODSPEED ? "" : ` ${baseline.name} (${baseline.rev})`}`;
+					const fullSuiteName = `benchmark "${test}", scenario '${stringifiedScenario}' ${baseline.name} (${baseline.rev})`;
+
+					console.log(`Register: ${fullSuiteName}`);
+
+					suite.add(suiteName, {
+						collectBy: `${test}, scenario '${stringifiedScenario}'`,
+						defer: true,
+						fn(deferred) {
+							if (watching) {
+								watching.invalidate(() => {
+									watching.close(closeErr => {
+										if (closeErr) {
+											throw closeErr;
 										}
 
 										deferred.resolve();
 									});
+								});
+							} else {
+								const baseCompiler = webpack(config);
+
+								baseCompiler.run((err, stats) => {
+									if (err) {
+										throw err;
 									}
+
+									if (stats.hasWarnings() || stats.hasErrors()) {
+										throw new Error(stats.toString());
+									}
+
+									baseCompiler.close(closeErr => {
+										if (closeErr) {
+											throw closeErr;
+										}
+
+										deferred.resolve();
+									});
 								});
 							}
+						}
+					});
+				})
+			);
+		})
+	);
 }
 
 const suite = withCodSpeed(
@@ -315,31 +438,62 @@ const suite = withCodSpeed(
 	})
 );
 
+await fs.rm(baseOutputPath, { recursive: true, force: true });
+
 const casesPath = path.join(__dirname, "benchmarkCases");
+const allBenchmarks = (await fs.readdir(casesPath))
+	.filter(item => !item.includes("_"))
+	.sort((a, b) => a.localeCompare(b));
 
-const tests = [];
+const benchmarks = allBenchmarks.filter(item => !item.includes("long"));
+const longBenchmarks = allBenchmarks.filter(item => item.includes("long"));
+const i = Math.floor(benchmarks.length / longBenchmarks.length);
 
-for (const folder of await fs.readdir(casesPath)) {
-	if (folder.includes("_")) {
-		continue;
-	}
+for (const [index, value] of longBenchmarks.entries()) {
+	benchmarks.splice(index * i, 0, value);
+}
 
-	try {
-		await fs.access(
-			path.resolve(casesPath, folder, "webpack.config.js"),
-			constants.R_OK
+const shard =
+	typeof process.env.SHARD !== "undefined"
+		? process.env.SHARD.split("/").map(item => Number.parseInt(item, 10))
+		: [1, 1];
+
+if (
+	typeof shard[0] === "undefined" ||
+	typeof shard[1] === "undefined" ||
+	shard[0] > shard[1] ||
+	shard[0] <= 0 ||
+	shard[1] <= 0
+) {
+	throw new Error(
+		`Invalid \`SHARD\` value - it should be less then a part and more than zero, shard part is ${shard[0]}, count of shards is ${shard[1]}`
 	);
-	} catch (_err) {
-		continue;
+}
+
+function splitToNChunks(array, n) {
+	const result = [];
+
+	for (let i = n; i > 0; i--) {
+		result.push(array.splice(0, Math.ceil(array.length / i)));
 	}
 
-	tests.push(folder);
+	return result;
 }
 
-for (const test of tests) {
-	await registerBenchmarks(suite, test, baselines);
+const countOfBenchmarks = benchmarks.length;
+
+if (countOfBenchmarks < shard[1]) {
+	throw new Error(
+		`Shard upper limit is more than count of benchmarks, count of benchmarks is ${countOfBenchmarks}, shard is ${shard[1]}`
+	);
 }
 
+await Promise.all(
+	splitToNChunks(benchmarks, shard[1])[shard[0] - 1].map(benchmark =>
+		registerSuite(suite, benchmark, baselines)
+	)
+);
+
 const statsByTests = new Map();
 
 suite.on("cycle", event => {
@@ -349,21 +503,25 @@ suite.on("cycle", event => {
 	const nSqrt = Math.sqrt(n);
 	const z = tDistribution(n - 1);
 
-	stats.sampleCount = stats.sample.length;
+	const sampleCount = stats.sample.length;
 	stats.minConfidence = stats.mean - (z * stats.deviation) / nSqrt;
 	stats.maxConfidence = stats.mean + (z * stats.deviation) / nSqrt;
-	stats.text = `${target.name} ${Math.round(stats.mean * 1000)} ms ± ${Math.round(
+	const confidence = `${Math.round(stats.mean * 1000)} ms ± ${Math.round(
 		stats.deviation * 1000
 	)} ms [${Math.round(stats.minConfidence * 1000)} ms; ${Math.round(
 		stats.maxConfidence * 1000
 	)} ms]`;
+	stats.text = `${target.name} ${confidence}`;
 
-	const baseTestName = target.baseTestName;
-	const allStats = statsByTests.get(baseTestName);
+	const collectBy = target.collectBy;
+	const allStats = statsByTests.get(collectBy);
+
+	console.log(
+		`Done: ${target.name} ${confidence} (${sampleCount} runs sampled)`
+	);
 
 	if (!allStats) {
-		console.log(String(target));
-		statsByTests.set(baseTestName, [stats]);
+		statsByTests.set(collectBy, [stats]);
 		return;
 	}
 
@@ -373,14 +531,10 @@ suite.on("cycle", event => {
 	const baselineStats = allStats[1];
 
 	console.log(
-		`Benchmark "${baseTestName}" result: ${headStats.text} is ${Math.round(
+		`Result: ${headStats.text} is ${Math.round(
 			(baselineStats.mean / headStats.mean) * 100 - 100
 		)}% ${baselineStats.maxConfidence < headStats.minConfidence ? "slower than" : baselineStats.minConfidence > headStats.maxConfidence ? "faster than" : "the same as"} ${baselineStats.text}`
 	);
 });
 
 suite.run({ async: true });
-
-suite.on("complete", () => {
-	remove(baselinesPath);
-});

test/benchmarkCases/_helpers/create-tree.mjs

@@ -0,0 +1,103 @@
+import fs from "fs/promises";
+
+const avgCode = `
+const str = "we" + "do" + "some" + "ops";
+for(const x of str.split("")) {
+	if(x.charCodeAt(0) > 40) {
+		console.log("omg");
+	} else {
+		console.log(Math.random() * 2 + 3 * 2);
+	}
+}
+
+// Some comment
+const value = 9;
+
+switch(value) {
+	case true:
+		break;
+	case "magic":
+		throw new Error("Error!");
+	case 9:
+		(function() {
+			// extra scope
+			var x = 123;
+			var y = 456;
+			var z = x + z * x / y;
+			x && y && (z = x ? y : x);
+		}())
+}
+
+function a() {}
+function b() {}
+function c() {}
+function d() {}
+function e() {}
+function f() {}
+`;
+
+async function createTree(
+	folder,
+	useRequire = false,
+	count = 50,
+	async = undefined
+) {
+	await fs.mkdir(folder, { recursive: true });
+
+	let remaining = count - 1;
+
+	async function make(prefix, count, depth) {
+		if (count === 0) {
+			await fs.writeFile(
+				`${folder}/${prefix}.js`,
+				useRequire
+					? `module.exports = 1;\n${avgCode}`
+					: `export default 1;\n${avgCode}`
+			);
+		} else {
+			const list = [];
+
+			for (let i = 0; i < count; i++) {
+				if (remaining-- <= 0) break;
+				const isAsync =
+					typeof async !== "undefined"
+						? depth >= async
+						: depth <= 4 && i >= 3 && i <= 4;
+				const module = `${prefix}-${i}`;
+
+				if (isAsync) {
+					list.push(
+						useRequire
+							? `require.ensure([], function() { const module${i} = require(${JSON.stringify(`./${module}.js`)}); });\n`
+							: `const module${i} = await import(${JSON.stringify(`./${module}.js`)});\n`
+					);
+					list.push(`counter += module${i};\nconsole.log(counter)`);
+				} else {
+					list.push(
+						useRequire
+							? `const module${i} = require(${JSON.stringify(`./${module}.js`)});\n`
+							: `import module${i} from ${JSON.stringify(`./${module}.js`)};\n`
+					);
+					list.push(`counter += module${i};\nconsole.log(counter)`);
+				}
+
+				await make(
+					module,
+					depth > 4 || count > 30 ? 0 : count + depth + i ** 2,
+					depth + 1
+				);
+			}
+
+			await fs.writeFile(
+				`${folder}/${prefix}.js`,
+				`let counter = 0;\n${list.join(
+					"\n"
+				)};\n${useRequire ? `module.exports = counter;\n${avgCode}` : `export default counter;\n${avgCode}`}`
+			);
+		}
+	}
+
+	await make("module", 2, 0);
+}
+
+export default createTree;

test/benchmarkCases/cache-filesystem/index.js

@@ -0,0 +1,2 @@
+import * as mod from "three/src/Three.js";
+export { mod };

test/benchmarkCases/cache-filesystem/webpack.config.js

@@ -0,0 +1,7 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index",
+	cache: {
+		type: "filesystem"
+	}
+};

test/benchmarkCases/context-commonjs/index.js

@@ -0,0 +1,6 @@
+async function run() {
+	const mod = "module.js"
+
+	require(`./generated/${mod}`);
+}
+

test/benchmarkCases/context-commonjs/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated, true, 25);

test/benchmarkCases/context-esm/index.js

@@ -0,0 +1,5 @@
+async function run() {
+	const mod = "module.js"
+
+	await import(`./generated/${mod}`);
+}

test/benchmarkCases/context-esm/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated, false, 25);

test/benchmarkCases/devtool-eval-source-map/index.js

@@ -0,0 +1,2 @@
+import * as mod from "./generated/module.js";
+export { mod };

test/benchmarkCases/devtool-eval-source-map/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated);

test/benchmarkCases/devtool-eval-source-map/webpack.config.js

@@ -0,0 +1,5 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index",
+	devtool: "eval-source-map"
+};

test/benchmarkCases/devtool-eval/index.js

@@ -0,0 +1,2 @@
+import * as mod from "./generated/module.js";
+export { mod };

test/benchmarkCases/devtool-eval/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated);

test/benchmarkCases/devtool-eval/webpack.config.js

@@ -1,4 +1,5 @@
 /** @type {import("../../../").Configuration} */
 module.exports = {
-	entry: ["react", "react-dom", "lodash"]
+	entry: "./index",
+	devtool: "eval"
 };

test/benchmarkCases/devtool-source-map/a.js

@@ -1,10 +0,0 @@
-require("./b?0" + __resourceQuery);
-require("./b?1" + __resourceQuery);
-require("./b?2" + __resourceQuery);
-require("./b?3" + __resourceQuery);
-require("./b?4" + __resourceQuery);
-require("./b?5" + __resourceQuery);
-require("./b?6" + __resourceQuery);
-require("./b?7" + __resourceQuery);
-require("./b?8" + __resourceQuery);
-require("./b?9" + __resourceQuery);

test/benchmarkCases/devtool-source-map/b.js

@@ -1,11 +0,0 @@
-require("./c?0" + __resourceQuery);
-require("./c?1" + __resourceQuery);
-require("./c?2" + __resourceQuery);
-require("./c?3" + __resourceQuery);
-require("./c?4" + __resourceQuery);
-require("./c?5" + __resourceQuery);
-require("./c?6" + __resourceQuery);
-require("./c?7" + __resourceQuery);
-require("./c?8" + __resourceQuery);
-require("./c?9" + __resourceQuery);
-require("./a" + __resourceQuery.slice(0, 2));

test/benchmarkCases/devtool-source-map/c.js

@@ -1 +0,0 @@
-// content

test/benchmarkCases/devtool-source-map/index.js

@@ -1,10 +1,2 @@
-import "./a?0";
-import "./a?1";
-import "./a?2";
-import "./a?3";
-import "./a?4";
-import "./a?5";
-import "./a?6";
-import "./a?7";
-import "./a?8";
-import "./a?9";
+import * as mod from "./generated/module.js";
+export { mod };

test/benchmarkCases/devtool-source-map/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated);

test/benchmarkCases/future-defaults/index.js

@@ -0,0 +1,2 @@
+import * as mod from "./generated/module.js";
+export { mod }

test/benchmarkCases/future-defaults/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated, false, 75);

test/benchmarkCases/future-defaults/webpack.config.js

@@ -0,0 +1,7 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index",
+	experiments: {
+		futureDefaults: true
+	}
+};

test/benchmarkCases/many-chunks-commonjs/index.js

@@ -0,0 +1,2 @@
+const result = require("./generated/module.js");
+module.exports = result;

test/benchmarkCases/many-chunks-commonjs/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated, true, 50, 0);

test/benchmarkCases/many-chunks-commonjs/webpack.config.js

@@ -0,0 +1,4 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index"
+};

test/benchmarkCases/many-chunks-esm/index.js

@@ -0,0 +1,2 @@
+import * as mod from "./generated/module.js";
+export { mod };

test/benchmarkCases/many-chunks-esm/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated, false, 50, 0);

test/benchmarkCases/many-chunks-esm/webpack.config.js

@@ -0,0 +1,4 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index"
+};

test/benchmarkCases/many-chunks/a.js

@@ -1,10 +0,0 @@
-import("./b?0" + __resourceQuery);
-import("./b?1" + __resourceQuery);
-import("./b?2" + __resourceQuery);
-import("./b?3" + __resourceQuery);
-import("./b?4" + __resourceQuery);
-import("./b?5" + __resourceQuery);
-import("./b?6" + __resourceQuery);
-import("./b?7" + __resourceQuery);
-import("./b?8" + __resourceQuery);
-import("./b?9" + __resourceQuery);

test/benchmarkCases/many-chunks/b.js

@@ -1,6 +0,0 @@
-import("./c?0" + __resourceQuery);
-import("./c?1" + __resourceQuery);
-import("./c?2" + __resourceQuery);
-import("./c?3" + __resourceQuery);
-import("./c?4" + __resourceQuery);
-import("./a" + __resourceQuery.slice(0, 2));

test/benchmarkCases/many-chunks/c.js

@@ -1 +0,0 @@
-// content

test/benchmarkCases/many-chunks/index.js

@@ -1,10 +0,0 @@
-import("./a?0");
-import("./a?1");
-import("./a?2");
-import("./a?3");
-import("./a?4");
-import("./a?5");
-import("./a?6");
-import("./a?7");
-import("./a?8");
-import("./a?9");

test/benchmarkCases/many-modules-commonjs/index.js

@@ -0,0 +1,2 @@
+const result = require("./generated/module.js");
+module.exports = result;

test/benchmarkCases/many-modules-commonjs/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated, true, 75);

test/benchmarkCases/many-modules-commonjs/webpack.config.js

@@ -0,0 +1,4 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index"
+};

test/benchmarkCases/many-modules-esm/index.js

@@ -0,0 +1,2 @@
+import * as mod from "./generated/module.js";
+export { mod };

test/benchmarkCases/many-modules-esm/setup.mjs

@@ -0,0 +1,11 @@
+import createTree from "../_helpers/create-tree.mjs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import fs from "fs/promises";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const generated = resolve(__dirname, "./generated");
+
+await fs.rm(generated, { recursive: true, force: true });
+
+await createTree(generated, false, 75);

test/benchmarkCases/many-modules-esm/webpack.config.js

@@ -0,0 +1,4 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index"
+};

test/benchmarkCases/many-modules/a.js

@@ -1,10 +0,0 @@
-require("./b?0" + __resourceQuery);
-require("./b?1" + __resourceQuery);
-require("./b?2" + __resourceQuery);
-require("./b?3" + __resourceQuery);
-require("./b?4" + __resourceQuery);
-require("./b?5" + __resourceQuery);
-require("./b?6" + __resourceQuery);
-require("./b?7" + __resourceQuery);
-require("./b?8" + __resourceQuery);
-require("./b?9" + __resourceQuery);

test/benchmarkCases/many-modules/b.js

@@ -1,11 +0,0 @@
-require("./c?0" + __resourceQuery);
-require("./c?1" + __resourceQuery);
-require("./c?2" + __resourceQuery);
-require("./c?3" + __resourceQuery);
-require("./c?4" + __resourceQuery);
-require("./c?5" + __resourceQuery);
-require("./c?6" + __resourceQuery);
-require("./c?7" + __resourceQuery);
-require("./c?8" + __resourceQuery);
-require("./c?9" + __resourceQuery);
-require("./a" + __resourceQuery.slice(0, 2));

test/benchmarkCases/many-modules/c.js

@@ -1 +0,0 @@
-// content

test/benchmarkCases/many-modules/index.js

@@ -1,10 +0,0 @@
-import "./a?0";
-import "./a?1";
-import "./a?2";
-import "./a?3";
-import "./a?4";
-import "./a?5";
-import "./a?6";
-import "./a?7";
-import "./a?8";
-import "./a?9";

test/benchmarkCases/minimal/index.js

@@ -0,0 +1 @@
+console.log("Hello world");

test/benchmarkCases/minimal/webpack.config.js

@@ -0,0 +1,4 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index"
+};

test/benchmarkCases/popular-libraries-long/index.js

@@ -0,0 +1,6 @@
+import * as mod1 from "react";
+export { mod1 };
+import * as mod2 from "react-dom";
+export { mod2 };
+import * as mod3 from "lodash";
+export { mod3 };

test/benchmarkCases/popular-libraries-long/webpack.config.js

@@ -0,0 +1,4 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index"
+};

test/benchmarkCases/three-long/index.js

@@ -0,0 +1,2 @@
+import * as copy from "three/src/Three.js";
+export { copy };

test/benchmarkCases/three-long/webpack.config.js

@@ -0,0 +1,4 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index"
+};

test/benchmarkCases/typescript-long/index.js

@@ -0,0 +1,2 @@
+import * as mod from "typescript";
+export { mod };

test/benchmarkCases/typescript-long/webpack.config.js

@@ -0,0 +1,11 @@
+/** @type {import("../../../").Configuration} */
+module.exports = {
+	entry: "./index",
+	target: "node",
+	optimization: {
+		avoidEntryIife: false,
+		// terser is very slow on typescript code, need to investigate
+		minimize: false
+	},
+	ignoreWarnings: [/Critical dependency/]
+};

yarn.lock

@@ -6334,6 +6334,11 @@ thingies@^1.20.0:
   resolved "https://registry.yarnpkg.com/thingies/-/thingies-1.21.0.tgz#e80fbe58fd6fdaaab8fad9b67bd0a5c943c445c1"
   integrity sha512-hsqsJsFMsV+aD4s3CWKk85ep/3I9XzYV/IXaSouJMYIoDlgyi11cBhsqYe9/geRfB0YIikBQg6raRaM+nIMP9g==
 
+three@^0.176.0:
+  version "0.176.0"
+  resolved "https://registry.yarnpkg.com/three/-/three-0.176.0.tgz#a30c1974e46db5745e4f96dd9ee2028d71e16ecf"
+  integrity sha512-PWRKYWQo23ojf9oZSlRGH8K09q7nRSWx6LY/HF/UUrMdYgN9i1e2OwJYHoQjwc6HF/4lvvYLC5YC1X8UJL2ZpA==
+
 timers-ext@^0.1.7:
   version "0.1.8"
   resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.8.tgz#b4e442f10b7624a29dd2aa42c295e257150cf16c"