Merge pull request #5502 from webpack/feature/limit-processed-modules

Limit the number of parallel processed modules

lib/Compilation.js

@@ -22,6 +22,7 @@ const Dependency = require("./Dependency");
 const ChunkRenderError = require("./ChunkRenderError");
 const CachedSource = require("webpack-sources").CachedSource;
 const Stats = require("./Stats");
+const Semaphore = require("./util/Semaphore");
 
 function byId(a, b) {
 	if(a.id < b.id) return -1;
@@ -62,6 +63,8 @@ class Compilation extends Tapable {
 		this.hotUpdateChunkTemplate = new HotUpdateChunkTemplate(this.outputOptions);
 		this.moduleTemplate = new ModuleTemplate(this.outputOptions);
 
+		this.semaphore = new Semaphore(options.parallelism || 100);
+
 		this.entries = [];
 		this.preparedChunks = [];
 		this.entrypoints = {};
@@ -229,120 +232,128 @@ class Compilation extends Tapable {
 				callback();
 			};
 
-			const factory = item[0];
-			factory.create({
-				contextInfo: {
-					issuer: module.nameForCondition && module.nameForCondition(),
-					compiler: _this.compiler.name
-				},
-				context: module.context,
-				dependencies: dependencies
-			}, function factoryCallback(err, dependentModule) {
-				let afterFactory;
+			_this.semaphore.acquire(() => {
+				const factory = item[0];
+				factory.create({
+					contextInfo: {
+						issuer: module.nameForCondition && module.nameForCondition(),
+						compiler: _this.compiler.name
+					},
+					context: module.context,
+					dependencies: dependencies
+				}, function factoryCallback(err, dependentModule) {
+					let afterFactory;
 
-				function isOptional() {
-					return dependencies.filter(d => !d.optional).length === 0;
-				}
-
-				function errorOrWarningAndCallback(err) {
-					if(isOptional()) {
-						return warningAndCallback(err);
-					} else {
-						return errorAndCallback(err);
-					}
-				}
-
-				function iterationDependencies(depend) {
-					for(let index = 0; index < depend.length; index++) {
-						const dep = depend[index];
-						dep.module = dependentModule;
-						dependentModule.addReason(module, dep);
-					}
-				}
-
-				if(err) {
-					return errorOrWarningAndCallback(new ModuleNotFoundError(module, err, dependencies));
-				}
-				if(!dependentModule) {
-					return process.nextTick(callback);
-				}
-				if(_this.profile) {
-					if(!dependentModule.profile) {
-						dependentModule.profile = {};
-					}
-					afterFactory = Date.now();
-					dependentModule.profile.factory = afterFactory - start;
-				}
-
-				dependentModule.issuer = module;
-				const newModule = _this.addModule(dependentModule, cacheGroup);
-
-				if(!newModule) { // from cache
-					dependentModule = _this.getModule(dependentModule);
-
-					if(dependentModule.optional) {
-						dependentModule.optional = isOptional();
+					function isOptional() {
+						return dependencies.filter(d => !d.optional).length === 0;
 					}
 
-					iterationDependencies(dependencies);
-
-					if(_this.profile) {
-						if(!module.profile) {
-							module.profile = {};
-						}
-						const time = Date.now() - start;
-						if(!module.profile.dependencies || time > module.profile.dependencies) {
-							module.profile.dependencies = time;
+					function errorOrWarningAndCallback(err) {
+						if(isOptional()) {
+							return warningAndCallback(err);
+						} else {
+							return errorAndCallback(err);
 						}
 					}
 
-					return process.nextTick(callback);
-				}
-
-				if(newModule instanceof Module) {
-					if(_this.profile) {
-						newModule.profile = dependentModule.profile;
+					function iterationDependencies(depend) {
+						for(let index = 0; index < depend.length; index++) {
+							const dep = depend[index];
+							dep.module = dependentModule;
+							dependentModule.addReason(module, dep);
+						}
 					}
 
-					newModule.optional = isOptional();
-					newModule.issuer = dependentModule.issuer;
-					dependentModule = newModule;
-
-					iterationDependencies(dependencies);
-
-					if(_this.profile) {
-						const afterBuilding = Date.now();
-						module.profile.building = afterBuilding - afterFactory;
+					if(err) {
+						_this.semaphore.release();
+						return errorOrWarningAndCallback(new ModuleNotFoundError(module, err, dependencies));
 					}
-
-					if(recursive) {
-						return process.nextTick(_this.processModuleDependencies.bind(_this, dependentModule, callback));
-					} else {
+					if(!dependentModule) {
+						_this.semaphore.release();
 						return process.nextTick(callback);
 					}
-				}
-
-				dependentModule.optional = isOptional();
-
-				iterationDependencies(dependencies);
-
-				_this.buildModule(dependentModule, isOptional(), module, dependencies, err => {
-					if(err) {
-						return errorOrWarningAndCallback(err);
-					}
-
 					if(_this.profile) {
-						const afterBuilding = Date.now();
-						dependentModule.profile.building = afterBuilding - afterFactory;
+						if(!dependentModule.profile) {
+							dependentModule.profile = {};
+						}
+						afterFactory = Date.now();
+						dependentModule.profile.factory = afterFactory - start;
 					}
 
-					if(recursive) {
-						_this.processModuleDependencies(dependentModule, callback);
-					} else {
-						return callback();
+					dependentModule.issuer = module;
+					const newModule = _this.addModule(dependentModule, cacheGroup);
+
+					if(!newModule) { // from cache
+						dependentModule = _this.getModule(dependentModule);
+
+						if(dependentModule.optional) {
+							dependentModule.optional = isOptional();
+						}
+
+						iterationDependencies(dependencies);
+
+						if(_this.profile) {
+							if(!module.profile) {
+								module.profile = {};
+							}
+							const time = Date.now() - start;
+							if(!module.profile.dependencies || time > module.profile.dependencies) {
+								module.profile.dependencies = time;
+							}
+						}
+
+						_this.semaphore.release();
+						return process.nextTick(callback);
 					}
+
+					if(newModule instanceof Module) {
+						if(_this.profile) {
+							newModule.profile = dependentModule.profile;
+						}
+
+						newModule.optional = isOptional();
+						newModule.issuer = dependentModule.issuer;
+						dependentModule = newModule;
+
+						iterationDependencies(dependencies);
+
+						if(_this.profile) {
+							const afterBuilding = Date.now();
+							module.profile.building = afterBuilding - afterFactory;
+						}
+
+						_this.semaphore.release();
+						if(recursive) {
+							return process.nextTick(_this.processModuleDependencies.bind(_this, dependentModule, callback));
+						} else {
+							return process.nextTick(callback);
+						}
+					}
+
+					dependentModule.optional = isOptional();
+
+					iterationDependencies(dependencies);
+
+					_this.buildModule(dependentModule, isOptional(), module, dependencies, err => {
+						if(err) {
+							_this.semaphore.release();
+							return errorOrWarningAndCallback(err);
+						}
+
+						if(_this.profile) {
+							const afterBuilding = Date.now();
+							dependentModule.profile.building = afterBuilding - afterFactory;
+						}
+
+						_this.semaphore.release();
+						if(recursive) {
+							_this.processModuleDependencies(dependentModule, callback);
+						} else {
+							return callback();
+						}
+					});
+
 				});
-
 			});
 		}, function finalCallbackAddModuleDependencies(err) {
 			// In V8, the Error objects keep a reference to the functions on the stack. These warnings &
@@ -379,79 +390,85 @@ class Compilation extends Tapable {
 			throw new Error(`No dependency factory available for this dependency type: ${dependency.constructor.name}`);
 		}
 
-		moduleFactory.create({
-			contextInfo: {
-				issuer: "",
-				compiler: this.compiler.name
-			},
-			context: context,
-			dependencies: [dependency]
-		}, (err, module) => {
-			if(err) {
-				return errorAndCallback(new EntryModuleNotFoundError(err));
-			}
-
-			let afterFactory;
-
-			if(this.profile) {
-				if(!module.profile) {
-					module.profile = {};
-				}
-				afterFactory = Date.now();
-				module.profile.factory = afterFactory - start;
-			}
-
-			const result = this.addModule(module);
-			if(!result) {
-				module = this.getModule(module);
-
-				onModule(module);
-
-				if(this.profile) {
-					const afterBuilding = Date.now();
-					module.profile.building = afterBuilding - afterFactory;
-				}
-
-				return callback(null, module);
-			}
-
-			if(result instanceof Module) {
-				if(this.profile) {
-					result.profile = module.profile;
-				}
-
-				module = result;
-
-				onModule(module);
-
-				moduleReady.call(this);
-				return;
-			}
-
-			onModule(module);
-
-			this.buildModule(module, false, null, null, (err) => {
+		this.semaphore.acquire(() => {
+			moduleFactory.create({
+				contextInfo: {
+					issuer: "",
+					compiler: this.compiler.name
+				},
+				context: context,
+				dependencies: [dependency]
+			}, (err, module) => {
 				if(err) {
-					return errorAndCallback(err);
+					this.semaphore.release();
+					return errorAndCallback(new EntryModuleNotFoundError(err));
 				}
 
+				let afterFactory;
+
 				if(this.profile) {
-					const afterBuilding = Date.now();
-					module.profile.building = afterBuilding - afterFactory;
+					if(!module.profile) {
+						module.profile = {};
+					}
+					afterFactory = Date.now();
+					module.profile.factory = afterFactory - start;
 				}
 
-				moduleReady.call(this);
-			});
+				const result = this.addModule(module);
+				if(!result) {
+					module = this.getModule(module);
 
-			function moduleReady() {
-				this.processModuleDependencies(module, err => {
-					if(err) {
-						return callback(err);
+					onModule(module);
+
+					if(this.profile) {
+						const afterBuilding = Date.now();
+						module.profile.building = afterBuilding - afterFactory;
 					}
 
+					this.semaphore.release();
 					return callback(null, module);
+				}
+
+				if(result instanceof Module) {
+					if(this.profile) {
+						result.profile = module.profile;
+					}
+
+					module = result;
+
+					onModule(module);
+
+					moduleReady.call(this);
+					return;
+				}
+
+				onModule(module);
+
+				this.buildModule(module, false, null, null, (err) => {
+					if(err) {
+						this.semaphore.release();
+						return errorAndCallback(err);
+					}
+
+					if(this.profile) {
+						const afterBuilding = Date.now();
+						module.profile.building = afterBuilding - afterFactory;
+					}
+
+					moduleReady.call(this);
 				});
-			}
+
+				function moduleReady() {
+					this.semaphore.release();
+					this.processModuleDependencies(module, err => {
+						if(err) {
+							return callback(err);
+						}
+
+						return callback(null, module);
+					});
+				}
+			});
 		});
 	}
 

lib/util/Semaphore.js

@@ -0,0 +1,32 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+	Author Tobias Koppers @sokra
+*/
+"use strict";
+
+class Semaphore {
+	constructor(available) {
+		this.available = available;
+		this.waiters = [];
+	}
+
+	acquire(callback) {
+		if(this.available > 0) {
+			this.available--;
+			callback();
+		} else {
+			this.waiters.push(callback);
+		}
+	}
+
+	release() {
+		if(this.waiters.length > 0) {
+			const callback = this.waiters.pop();
+			process.nextTick(callback);
+		} else {
+			this.available++;
+		}
+	}
+}
+
+module.exports = Semaphore;

schemas/webpackOptionsSchema.json

@@ -900,6 +900,11 @@
     "output": {
       "$ref": "#/definitions/output"
     },
+    "parallelism": {
+      "description": "The number of parallel processed modules in the compilation.",
+      "minimum": 1,
+      "type": "number"
+    },
     "performance": {
       "description": "Configuration for web performance recommendations.",
       "anyOf": [

test/Validation.test.js

@@ -146,8 +146,8 @@ describe("Validation", () => {
 		message: [
 			" - configuration has an unknown property 'postcss'. These properties are valid:",
 			"   object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry, externals?, " +
-			"loader?, module?, name?, node?, output?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, " +
-			"recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }",
+			"loader?, module?, name?, node?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, " +
+			"recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }",
 			"   For typos: please correct them.",
 			"   For loader options: webpack 2 no longer allows custom properties in configuration.",
 			"     Loaders should be updated to allow passing options via loader options in module.rules.",