mirror of https://github.com/webpack/webpack.git
213 lines
7.2 KiB
JavaScript
213 lines
7.2 KiB
JavaScript
"use strict";
|
|
|
|
const should = require("should");
|
|
const path = require("path");
|
|
const fs = require("fs");
|
|
const async = require("async");
|
|
var Test = require("mocha/lib/test");
|
|
|
|
const webpack = require("../lib/webpack");
|
|
const Benchmark = require("benchmark");
|
|
|
|
describe("BenchmarkTestCases", function() {
|
|
const casesPath = path.join(__dirname, "benchmarkCases");
|
|
const tests = fs.readdirSync(casesPath).filter(function(folder) {
|
|
return folder.indexOf("_") < 0 && fs.existsSync(path.resolve(casesPath, folder, "webpack.config.js"));
|
|
});
|
|
|
|
const baselinesPath = path.join(__dirname, "js", "benchmark-baselines");
|
|
const baselines = [];
|
|
|
|
try {
|
|
fs.mkdirSync(path.join(__dirname, "js"));
|
|
} catch(e) {}
|
|
try {
|
|
fs.mkdirSync(baselinesPath);
|
|
} catch(e) {}
|
|
|
|
before(function(done) {
|
|
this.timeout(270000);
|
|
const git = require("simple-git");
|
|
const rootPath = path.join(__dirname, "..");
|
|
getBaselineRevs(rootPath, (err, baselineRevisions) => {
|
|
if(err) return done(err);
|
|
async.eachSeries(baselineRevisions, (baselineInfo, callback) => {
|
|
const baselineRevision = baselineInfo.rev;
|
|
const baselinePath = path.resolve(baselinesPath, baselineRevision);
|
|
if(fs.existsSync(path.resolve(baselinePath, ".git"))) {
|
|
doLoadWebpack()
|
|
} else {
|
|
try {
|
|
fs.mkdirSync(baselinePath);
|
|
} catch(e) {}
|
|
const gitIndex = path.resolve(rootPath, ".git/index");
|
|
const index = fs.readFileSync(gitIndex);
|
|
git(rootPath).raw(["rev-list", "-n", "1", "HEAD"], (err, prevHead) => {
|
|
if(err) return callback(err);
|
|
git(baselinePath).raw(["--git-dir", path.join(rootPath, ".git"), "reset", "--hard", baselineRevision], err => {
|
|
if(err) return callback(err);
|
|
git(rootPath).raw(["reset", "--soft", prevHead.split("\n")[0]], err => {
|
|
if(err) return callback(err);
|
|
fs.writeFileSync(gitIndex, index);
|
|
doLoadWebpack();
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
function doLoadWebpack() {
|
|
const baselineWebpack = require(path.resolve(baselinePath, "lib/webpack.js"));
|
|
baselines.push({
|
|
name: baselineInfo.name,
|
|
rev: baselineRevision,
|
|
webpack: baselineWebpack
|
|
});
|
|
callback();
|
|
}
|
|
}, done)
|
|
});
|
|
});
|
|
|
|
function getBaselineRevs(rootPath, callback) {
|
|
const git = require("simple-git")(rootPath);
|
|
const lastVersionTag = "v" + require("../package.json").version;
|
|
git.raw(["rev-list", "-n", "1", lastVersionTag], (err, resultVersion) => {
|
|
if(err) return callback(err);
|
|
const matchVersion = /^([a-f0-9]+)\s*$/.exec(resultVersion);
|
|
if(!matchVersion) return callback(new Error("Invalid result from git revparse"));
|
|
const revLastVersion = matchVersion[1];
|
|
git.raw(["rev-list", "--parents", "-n", "1", "HEAD"], (err, resultParents) => {
|
|
if(err) return callback(err);
|
|
const match = /^([a-f0-9]+)\s*([a-f0-9]+)\s*([a-f0-9]+)?\s*$/.exec(resultParents);
|
|
if(!match) return callback(new Error("Invalid result from git rev-list"));
|
|
const head = match[1]
|
|
const parent1 = match[2];
|
|
const parent2 = match[3];
|
|
if(parent2 && parent1) {
|
|
return callback(null, [{
|
|
name: "HEAD",
|
|
rev: head
|
|
}, head !== revLastVersion && {
|
|
name: lastVersionTag,
|
|
rev: revLastVersion
|
|
}, parent1 !== revLastVersion && head !== revLastVersion && {
|
|
name: "base",
|
|
rev: parent1
|
|
}].filter(Boolean));
|
|
} else if(parent1) {
|
|
return callback(null, [{
|
|
name: "HEAD",
|
|
rev: head
|
|
}, head !== revLastVersion && {
|
|
name: lastVersionTag,
|
|
rev: revLastVersion
|
|
}].filter(Boolean));
|
|
} else {
|
|
return callback(new Error("No baseline found"));
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
function tDistribution(n) {
|
|
// two-sided, 90%
|
|
// https://en.wikipedia.org/wiki/Student%27s_t-distribution
|
|
if(n <= 30) {
|
|
// 1 2 ...
|
|
const data = [6.314, 2.920, 2.353, 2.132, 2.015, 1.943, 1.895, 1.860, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.740, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697];
|
|
return data[n - 1];
|
|
} else if(n <= 120) {
|
|
// 30 40 50 60 70 80 90 100 110 120
|
|
const data = [1.697, 1.684, 1.676, 1.671, 1.667, 1.664, 1.662, 1.660, 1.659, 1.658];
|
|
var a = data[Math.floor(n / 10) - 3];
|
|
var b = data[Math.ceil(n / 10) - 3];
|
|
var f = n / 10 - Math.floor(n / 10);
|
|
return a * (1 - f) + b * f;
|
|
} else {
|
|
return 1.645;
|
|
}
|
|
}
|
|
|
|
function runBenchmark(webpack, config, callback) {
|
|
// warmup
|
|
const warmupCompiler = webpack(config, (err, stats) => {
|
|
warmupCompiler.purgeInputFileSystem();
|
|
const bench = new Benchmark(function(deferred) {
|
|
const compiler = webpack(config, (err, stats) => {
|
|
compiler.purgeInputFileSystem();
|
|
if(err) {
|
|
callback(err);
|
|
return;
|
|
}
|
|
if(stats.hasErrors()) {
|
|
callback(new Error(stats.toJson().errors.join("\n\n")));
|
|
return;
|
|
}
|
|
deferred.resolve();
|
|
});
|
|
}, {
|
|
maxTime: 30,
|
|
defer: true,
|
|
initCount: 1,
|
|
onComplete: function() {
|
|
const stats = bench.stats;
|
|
const n = stats.sample.length;
|
|
const nSqrt = Math.sqrt(n);
|
|
const z = tDistribution(n - 1);
|
|
stats.minConfidence = stats.mean - z * stats.deviation / nSqrt;
|
|
stats.maxConfidence = stats.mean + z * stats.deviation / nSqrt;
|
|
stats.text = `${Math.round(stats.mean * 1000)}ms ± ${Math.round(stats.deviation * 1000)}ms [${Math.round(stats.minConfidence * 1000)}ms; ${Math.round(stats.maxConfidence * 1000)}ms]`;
|
|
callback(null, bench.stats);
|
|
},
|
|
onError: callback
|
|
});
|
|
bench.run({
|
|
async: true
|
|
});
|
|
});
|
|
}
|
|
|
|
tests.forEach(testName => {
|
|
const testDirectory = path.join(casesPath, testName);
|
|
let headStats = null;
|
|
const suite = describe(testName, function() {});
|
|
it(`${testName} create benchmarks`, function() {
|
|
baselines.forEach(baseline => {
|
|
let baselineStats = null;
|
|
|
|
function it(title, fn) {
|
|
const test = new Test(title, fn);
|
|
suite.addTest(test);
|
|
}
|
|
it(`should benchmark ${baseline.name} (${baseline.rev})`, function(done) {
|
|
this.timeout(180000);
|
|
const outputDirectory = path.join(__dirname, "js", "benchmark", `baseline-${baseline.name}`, testName);
|
|
const config = Object.create(require(path.join(testDirectory, "webpack.config.js")));
|
|
config.output = Object.create(config.output || {});
|
|
if(!config.context) config.context = testDirectory;
|
|
if(!config.output.path) config.output.path = outputDirectory;
|
|
runBenchmark(baseline.webpack, config, (err, stats) => {
|
|
if(err) return done(err);
|
|
console.log(` ${baseline.name} ${stats.text}`);
|
|
if(baseline.name === "HEAD")
|
|
headStats = stats;
|
|
else
|
|
baselineStats = stats;
|
|
done();
|
|
});
|
|
});
|
|
|
|
if(baseline.name !== "HEAD") {
|
|
it(`HEAD should not be slower than ${baseline.name} (${baseline.rev})`, function() {
|
|
if(baselineStats.maxConfidence < headStats.minConfidence) {
|
|
throw new Error(`HEAD (${headStats.text}) is slower than ${baseline.name} (${baselineStats.text}) (90% confidence)`);
|
|
} else if(baselineStats.minConfidence > headStats.maxConfidence) {
|
|
console.log(`======> HEAD is ${Math.round(baselineStats.mean / headStats.mean * 100 - 100)}% faster than ${baseline.name} (90% confidence)!`);
|
|
}
|
|
});
|
|
}
|
|
});
|
|
});
|
|
});
|
|
});
|