webpack/test/BenchmarkTestCases.benchmark.js

"use strict";

require("should");
const path = require("path");
const fs = require("fs");
const asyncLib = require("async");
var Test = require("mocha/lib/test");

const Benchmark = require("benchmark");

describe("BenchmarkTestCases", function() {
	const casesPath = path.join(__dirname, "benchmarkCases");
	const tests = fs.readdirSync(casesPath).filter(function(folder) {
		return folder.indexOf("_") < 0 && fs.existsSync(path.resolve(casesPath, folder, "webpack.config.js"));
	});

	const baselinesPath = path.join(__dirname, "js", "benchmark-baselines");
	const baselines = [];

	try {
		fs.mkdirSync(path.join(__dirname, "js"));
	} catch(e) {}
	try {
		fs.mkdirSync(baselinesPath);
	} catch(e) {}

	before(function(done) {
		this.timeout(270000);
		const git = require("simple-git");
		const rootPath = path.join(__dirname, "..");
		getBaselineRevs(rootPath, (err, baselineRevisions) => {
			if(err) return done(err);
			asyncLib.eachSeries(baselineRevisions, (baselineInfo, callback) => {
				const baselineRevision = baselineInfo.rev;
				const baselinePath = path.resolve(baselinesPath, baselineRevision);
				if(fs.existsSync(path.resolve(baselinePath, ".git"))) {
					doLoadWebpack();
				} else {
					try {
						fs.mkdirSync(baselinePath);
					} catch(e) {}
					const gitIndex = path.resolve(rootPath, ".git/index");
					const index = fs.readFileSync(gitIndex);
					git(rootPath).raw(["rev-list", "-n", "1", "HEAD"], (err, prevHead) => {
						if(err) return callback(err);
						git(baselinePath).raw(["--git-dir", path.join(rootPath, ".git"), "reset", "--hard", baselineRevision], err => {
							if(err) return callback(err);
							git(rootPath).raw(["reset", "--soft", prevHead.split("\n")[0]], err => {
								if(err) return callback(err);
								fs.writeFileSync(gitIndex, index);
								doLoadWebpack();
							});
						});
					});
				}

				function doLoadWebpack() {
					const baselineWebpack = require(path.resolve(baselinePath, "lib/webpack.js"));
					baselines.push({
						name: baselineInfo.name,
						rev: baselineRevision,
						webpack: baselineWebpack
					});
					callback();
				}
			}, done);
		});
	});

	function getBaselineRevs(rootPath, callback) {
		const git = require("simple-git")(rootPath);
		const lastVersionTag = "v" + require("../package.json").version;
		git.raw(["rev-list", "-n", "1", lastVersionTag], (err, resultVersion) => {
			if(err) return callback(err);
			const matchVersion = /^([a-f0-9]+)\s*$/.exec(resultVersion);
			if(!matchVersion) return callback(new Error("Invalid result from git revparse"));
			const revLastVersion = matchVersion[1];
			git.raw(["rev-list", "--parents", "-n", "1", "HEAD"], (err, resultParents) => {
				if(err) return callback(err);
				const match = /^([a-f0-9]+)\s*([a-f0-9]+)\s*([a-f0-9]+)?\s*$/.exec(resultParents);
				if(!match) return callback(new Error("Invalid result from git rev-list"));
				const head = match[1];
				const parent1 = match[2];
				const parent2 = match[3];
				if(parent2 && parent1) {
					return callback(null, [{
						name: "HEAD",
						rev: head
					}, head !== revLastVersion && {
						name: lastVersionTag,
						rev: revLastVersion
					}, parent1 !== revLastVersion && head !== revLastVersion && {
						name: "base",
						rev: parent1
					}].filter(Boolean));
				} else if(parent1) {
					return callback(null, [{
						name: "HEAD",
						rev: head
					}, head !== revLastVersion && {
						name: lastVersionTag,
						rev: revLastVersion
					}].filter(Boolean));
				} else {
					return callback(new Error("No baseline found"));
				}
			});
		});
	}

	function tDistribution(n) {
		// two-sided, 90%
		// https://en.wikipedia.org/wiki/Student%27s_t-distribution
		if(n <= 30) {
			//            1      2      ...
			const data = [6.314, 2.920, 2.353, 2.132, 2.015, 1.943, 1.895, 1.860, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.740, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697];
			return data[n - 1];
		} else if(n <= 120) {
			//            30     40     50     60     70     80     90     100    110    120
			const data = [1.697, 1.684, 1.676, 1.671, 1.667, 1.664, 1.662, 1.660, 1.659, 1.658];
			var a = data[Math.floor(n / 10) - 3];
			var b = data[Math.ceil(n / 10) - 3];
			var f = n / 10 - Math.floor(n / 10);
			return a * (1 - f) + b * f;
		} else {
			return 1.645;
		}
	}

	function runBenchmark(webpack, config, callback) {
		// warmup
		const warmupCompiler = webpack(config, (err, stats) => {
			warmupCompiler.purgeInputFileSystem();
			const bench = new Benchmark(function(deferred) {
				const compiler = webpack(config, (err, stats) => {
					compiler.purgeInputFileSystem();
					if(err) {
						callback(err);
						return;
					}
					if(stats.hasErrors()) {
						callback(new Error(stats.toJson().errors.join("\n\n")));
						return;
					}
					deferred.resolve();
				});
			}, {
				maxTime: 30,
				defer: true,
				initCount: 1,
				onComplete: function() {
					const stats = bench.stats;
					const n = stats.sample.length;
					const nSqrt = Math.sqrt(n);
					const z = tDistribution(n - 1);
					stats.minConfidence = stats.mean - z * stats.deviation / nSqrt;
					stats.maxConfidence = stats.mean + z * stats.deviation / nSqrt;
					stats.text = `${Math.round(stats.mean * 1000)}ms ± ${Math.round(stats.deviation * 1000)}ms [${Math.round(stats.minConfidence * 1000)}ms; ${Math.round(stats.maxConfidence * 1000)}ms]`;
					callback(null, bench.stats);
				},
				onError: callback
			});
			bench.run({
				async: true
			});
		});
	}

	tests.forEach(testName => {
		const testDirectory = path.join(casesPath, testName);
		let headStats = null;
		const suite = describe(testName, function() {});
		it(`${testName} create benchmarks`, function() {
			baselines.forEach(baseline => {
				let baselineStats = null;

				function it(title, fn) {
					const test = new Test(title, fn);
					suite.addTest(test);
				}
				it(`should benchmark ${baseline.name} (${baseline.rev})`, function(done) {
					this.timeout(180000);
					const outputDirectory = path.join(__dirname, "js", "benchmark", `baseline-${baseline.name}`, testName);
					const config = Object.create(require(path.join(testDirectory, "webpack.config.js")));
					config.output = Object.create(config.output || {});
					if(!config.context) config.context = testDirectory;
					if(!config.output.path) config.output.path = outputDirectory;
					runBenchmark(baseline.webpack, config, (err, stats) => {
						if(err) return done(err);
						console.log(`        ${baseline.name} ${stats.text}`);
						if(baseline.name === "HEAD")
							headStats = stats;
						else
							baselineStats = stats;
						done();
					});
				});

				if(baseline.name !== "HEAD") {
					it(`HEAD should not be slower than ${baseline.name} (${baseline.rev})`, function() {
						if(baselineStats.maxConfidence < headStats.minConfidence) {
							throw new Error(`HEAD (${headStats.text}) is slower than ${baseline.name} (${baselineStats.text}) (90% confidence)`);
						} else if(baselineStats.minConfidence > headStats.maxConfidence) {
							console.log(`======> HEAD is ${Math.round(baselineStats.mean / headStats.mean * 100 - 100)}% faster than ${baseline.name} (90% confidence)!`);
						}
					});
				}
			});
		});
	});
});