From 5b4cbb5ee041bd96c80f0d657bda5f6fe5bc1450 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 11 Sep 2018 18:47:55 +0200 Subject: [PATCH] add queues to Compilation remove Semaphore and use AsyncQueue instead deprecate Module.needRebuild, add Module.needBuild remove Module.unbuild add Module.invalidateBuild --- lib/Compilation.js | 512 ++++++++++++++--------------- lib/ContextModule.js | 19 +- lib/DelegatedModule.js | 8 +- lib/DllModule.js | 7 +- lib/ExternalModule.js | 7 +- lib/Module.js | 21 +- lib/ModuleProfile.js | 8 +- lib/NormalModule.js | 19 +- lib/RawModule.js | 7 +- lib/dependencies/LoaderPlugin.js | 94 +++--- lib/optimize/ConcatenatedModule.js | 3 +- lib/util/AsyncQueue.js | 224 +++++++++++++ test/NormalModule.unittest.js | 33 +- test/RawModule.unittest.js | 6 - 14 files changed, 601 insertions(+), 367 deletions(-) create mode 100644 lib/util/AsyncQueue.js diff --git a/lib/Compilation.js b/lib/Compilation.js index db5a770f7..b333bd73a 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -36,8 +36,8 @@ const ModuleTemplate = require("./ModuleTemplate"); const RuntimeTemplate = require("./RuntimeTemplate"); const Stats = require("./Stats"); const compareLocations = require("./compareLocations"); +const AsyncQueue = require("./util/AsyncQueue"); const Queue = require("./util/Queue"); -const Semaphore = require("./util/Semaphore"); const SortableSet = require("./util/SortableSet"); const { concatComparators, @@ -63,8 +63,7 @@ const { arrayToSetDeprecation } = require("./util/deprecation"); // TODO use @callback /** @typedef {{[assetName: string]: Source}} CompilationAssets */ -/** @typedef {(err?: Error|null, result?: Module) => void } ModuleCallback */ -/** @typedef {(err?: Error|null, result?: Module) => void } ModuleChainCallback */ +/** @typedef {(err?: WebpackError|null, result?: Module) => void } ModuleCallback */ /** @typedef {(err?: Error|null) => void} Callback */ /** @typedef {(d: Dependency) => any} DepBlockVarDependenciesCallback */ /** @typedef {new (...args: any[]) => Dependency} DepConstructor */ @@ -89,12 +88,6 @@ const { arrayToSetDeprecation } = require("./util/deprecation"); * @property {(data: ModuleFactoryCreateData, callback: ModuleCallback) => any} create */ -/** - * @typedef {Object} SortedDependency - * @property {ModuleFactory} factory - * @property {Dependency[]} dependencies - */ - /** * @typedef {Object} AvailableModulesChunkGroupMapping * @property {ChunkGroup} chunkGroup @@ -203,7 +196,7 @@ class Compilation { buildModule: new SyncHook(["module"]), /** @type {SyncHook} */ rebuildModule: new SyncHook(["module"]), - /** @type {SyncHook} */ + /** @type {SyncHook} */ failedModule: new SyncHook(["module", "error"]), /** @type {SyncHook} */ succeedModule: new SyncHook(["module"]), @@ -381,7 +374,26 @@ class Compilation { this.moduleGraph = new ModuleGraph(); this.chunkGraph = undefined; - this.semaphore = new Semaphore(options.parallelism || 100); + this.factorizeQueue = new AsyncQueue({ + name: "factorize", + parallelism: options.parallelism || 100, + processor: this._factorizeModule.bind(this) + }); + this.buildQueue = new AsyncQueue({ + name: "build", + parallelism: options.parallelism || 100, + processor: this._buildModule.bind(this) + }); + this.rebuildQueue = new AsyncQueue({ + name: "rebuild", + parallelism: options.parallelism || 100, + processor: this._rebuildModule.bind(this) + }); + this.processDependenciesQueue = new AsyncQueue({ + name: "processDependencies", + parallelism: options.parallelism || 100, + processor: this._processModuleDependencies.bind(this) + }); /** @type {Map} */ this.entryDependencies = new Map(); @@ -433,8 +445,6 @@ class Compilation { /** @type {WeakSet} */ this.builtModules = new WeakSet(); /** @private @type {Map} */ - this._buildingModules = new Map(); - /** @private @type {Map} */ this._rebuildingModules = new Map(); } @@ -446,26 +456,19 @@ class Compilation { * @typedef {Object} AddModuleResult * @property {Module} module the added or existing module * @property {boolean} issuer was this the first request for this module - * @property {boolean} build should the module be build - * @property {boolean} dependencies should dependencies be walked */ /** * @param {Module} module module to be added that was created * @param {any=} cacheGroup cacheGroup it is apart of - * @returns {AddModuleResult} returns meta about whether or not the module had built - * had an issuer, or any dependnecies + * @returns {Module} returns the module in the compilation, + * it could be the passed one (if new), or an already existing in the compilation */ addModule(module, cacheGroup) { const identifier = module.identifier(); const alreadyAddedModule = this._modules.get(identifier); if (alreadyAddedModule) { - return { - module: alreadyAddedModule, - issuer: false, - build: false, - dependencies: false - }; + return alreadyAddedModule; } const cacheName = (cacheGroup || "m") + identifier; if (this.cache && this.cache[cacheName]) { @@ -473,32 +476,6 @@ class Compilation { cacheModule.updateCacheModule(module); - let rebuild = true; - if (this.fileTimestamps && this.contextTimestamps) { - rebuild = cacheModule.needRebuild( - this.fileTimestamps, - this.contextTimestamps - ); - } - - if (!rebuild) { - this._modules.set(identifier, cacheModule); - this.modules.add(cacheModule); - for (const err of cacheModule.errors) { - this.errors.push(err); - } - for (const err of cacheModule.warnings) { - this.warnings.push(err); - } - ModuleGraph.setModuleGraphForModule(cacheModule, this.moduleGraph); - return { - module: cacheModule, - issuer: true, - build: false, - dependencies: true - }; - } - cacheModule.unbuild(); module = cacheModule; } this._modules.set(identifier, module); @@ -507,12 +484,7 @@ class Compilation { } this.modules.add(module); ModuleGraph.setModuleGraphForModule(module, this.moduleGraph); - return { - module: module, - issuer: true, - build: true, - dependencies: true - }; + return module; } /** @@ -535,40 +507,48 @@ class Compilation { } /** - * @param {Module} module module with its callback list - * @param {Callback} callback the callback function + * Schedules a build of the module object + * + * @param {Module} module module to be built + * @param {ModuleCallback} callback the callback * @returns {void} */ - waitForBuildingFinished(module, callback) { - let callbackList = this._buildingModules.get(module); - if (callbackList) { - callbackList.push(() => callback()); - } else { - process.nextTick(callback); - } + buildModule(module, callback) { + this.buildQueue.add(module, callback); } /** * Builds the module object * * @param {Module} module module to be built - * @param {TODO} thisCallback the callback + * @param {ModuleCallback} callback the callback * @returns {TODO} returns the callback function with results */ - buildModule(module, thisCallback) { - let callbackList = this._buildingModules.get(module); - if (callbackList) { - callbackList.push(thisCallback); - return; + _buildModule(module, callback) { + const currentProfile = this.profile + ? this.moduleGraph.getProfile(module) + : undefined; + if (currentProfile !== undefined) { + currentProfile.markBuildingStart(); } - this._buildingModules.set(module, (callbackList = [thisCallback])); - const callback = err => { - this._buildingModules.delete(module); - for (const cb of callbackList) { - cb(err); + let rebuild = true; + if (this.fileTimestamps && this.contextTimestamps) { + rebuild = module.needBuild(this.fileTimestamps, this.contextTimestamps); + } + + if (!rebuild) { + for (const err of module.errors) { + this.errors.push(err); } - }; + for (const err of module.warnings) { + this.warnings.push(err); + } + if (currentProfile !== undefined) { + currentProfile.markBuildingEnd(); + } + return callback(); + } this.hooks.buildModule.call(module); this.builtModules.add(module); @@ -577,11 +557,14 @@ class Compilation { this, this.resolverFactory.get("normal", module.resolveOptions), this.inputFileSystem, - error => { + err => { module.dependencies.sort((a, b) => compareLocations(a.loc, b.loc)); - if (error) { - this.hooks.failedModule.call(module, error); - return callback(error); + if (currentProfile !== undefined) { + currentProfile.markBuildingEnd(); + } + if (err) { + this.hooks.failedModule.call(module, err); + return callback(err); } this.hooks.succeedModule.call(module); return callback(); @@ -595,6 +578,15 @@ class Compilation { * @returns {void} */ processModuleDependencies(module, callback) { + this.processDependenciesQueue.add(module, callback); + } + + /** + * @param {Module} module to be processed for deps + * @param {ModuleCallback} callback callback to be triggered + * @returns {void} + */ + _processModuleDependencies(module, callback) { const dependencies = new Map(); let currentBlock = module; @@ -648,91 +640,37 @@ class Compilation { } } - this.addModuleDependencies(module, sortedDependencies, this.bail, callback); - } + // This is nested so we need to allow one additional task + this.processDependenciesQueue.increaseParallelism(); - /** - * @typedef {Object} HandleNewModuleOptions - * @property {Module} newModule - * @property {Module | null} originModule - * @property {Dependency[]} dependencies - * @property {ModuleProfile} currentProfile - * @property {boolean=} bail - */ - - /** - * @param {HandleNewModuleOptions} options options object - * @param {ModuleCallback} callback callback - * @returns {void} - */ - handleNewModule( - { newModule, originModule, dependencies, currentProfile, bail }, - callback - ) { - this.semaphore.acquire(() => { - const moduleGraph = this.moduleGraph; - - const addModuleResult = this.addModule(newModule); - - const module = addModuleResult.module; - - for (let i = 0; i < dependencies.length; i++) { - const dependency = dependencies[i]; - moduleGraph.setResolvedModule(originModule, dependency, module); - } - - if (addModuleResult.issuer) { - if (currentProfile !== undefined) { - moduleGraph.setProfile(module, currentProfile); - } - - if (originModule !== undefined) { - moduleGraph.setIssuer(module, originModule); - } - } else { - if (currentProfile !== undefined) { - currentProfile.mergeInto(moduleGraph.getProfile(module)); - } - } - - const afterBuild = () => { - if (addModuleResult.dependencies) { - this.processModuleDependencies(module, err => { - if (err) return callback(err); - callback(null, module); - }); - } else { - return callback(null, module); - } - }; - - if (addModuleResult.build) { - if (currentProfile !== undefined) { - currentProfile.markBuildingStart(); - } - this.buildModule(module, err => { - if (err) { - if (!err.module) { - err.module = module; + asyncLib.forEach( + sortedDependencies, + (item, callback) => { + this.handleModuleCreation( + { + factory: item.factory, + dependencies: item.dependencies, + originModule: module + }, + err => { + // In V8, the Error objects keep a reference to the functions on the stack. These warnings & + // errors are created inside closures that keep a reference to the Compilation, so errors are + // leaking the Compilation object. + if (err && this.bail) { + // eslint-disable-next-line no-self-assign + err.stack = err.stack; + return callback(err); } - this.errors.push(err); - this.semaphore.release(); - if (bail) return callback(err); - return callback(); + callback(); } + ); + }, + err => { + this.processDependenciesQueue.decreaseParallelism(); - if (currentProfile !== undefined) { - currentProfile.markBuildingEnd(); - } - - this.semaphore.release(); - afterBuild(); - }); - } else { - this.semaphore.release(); - this.waitForBuildingFinished(module, afterBuild); + return callback(err); } - }); + ); } /** @@ -741,7 +679,6 @@ class Compilation { * @property {Dependency[]} dependencies * @property {Module | null} originModule * @property {string=} context - * @property {boolean=} bail */ /** @@ -750,99 +687,138 @@ class Compilation { * @returns {void} */ handleModuleCreation( - { factory, dependencies, originModule, context, bail }, + { factory, dependencies, originModule, context }, callback ) { - const semaphore = this.semaphore; - semaphore.acquire(() => { - const currentProfile = this.profile ? new ModuleProfile() : undefined; - factory.create( - { - contextInfo: { - issuer: originModule ? originModule.nameForCondition() : "", - compiler: this.compiler.name - }, - resolveOptions: originModule - ? originModule.resolveOptions - : undefined, - context: context - ? context - : originModule - ? originModule.context - : this.compiler.context, - dependencies: dependencies - }, - (err, newModule) => { - semaphore.release(); - if (err) { - const notFoundError = new ModuleNotFoundError( - originModule, - err, - dependencies.map(d => d.loc).filter(Boolean)[0] - ); - if (dependencies.every(d => d.optional)) { - this.warnings.push(notFoundError); - } else { - this.errors.push(notFoundError); - } - if (bail) return callback(notFoundError); - return callback(); - } - if (!newModule) { - return process.nextTick(callback); - } - if (currentProfile !== undefined) { - currentProfile.markFactoryEnd(); - } + const moduleGraph = this.moduleGraph; - this.handleNewModule( - { - newModule, - originModule, - dependencies, - currentProfile, - bail - }, - callback - ); - } - ); - }); - } - - /** - * @param {Module} module module to add deps to - * @param {SortedDependency[]} dependencies set of sorted dependencies to iterate through - * @param {(boolean|null)=} bail whether to bail or not - * @param {function} callback callback for when dependencies are finished being added - * @returns {void} - */ - addModuleDependencies(module, dependencies, bail, callback) { - asyncLib.forEach( - dependencies, - (item, callback) => { - this.handleModuleCreation( - { - factory: item.factory, - dependencies: item.dependencies, - originModule: module, - bail - }, - callback - ); - }, - err => { - // In V8, the Error objects keep a reference to the functions on the stack. These warnings & - // errors are created inside closures that keep a reference to the Compilation, so errors are - // leaking the Compilation object. + const currentProfile = this.profile ? new ModuleProfile() : undefined; + this.factorizeModule( + { currentProfile, factory, dependencies, originModule, context }, + (err, newModule) => { if (err) { - // eslint-disable-next-line no-self-assign - err.stack = err.stack; + if (dependencies.every(d => d.optional)) { + this.warnings.push(err); + } else { + this.errors.push(err); + } return callback(err); } - return process.nextTick(callback); + if (!newModule) { + return callback(); + } + + const module = this.addModule(newModule); + + for (let i = 0; i < dependencies.length; i++) { + const dependency = dependencies[i]; + moduleGraph.setResolvedModule(originModule, dependency, module); + } + + if (module === newModule) { + if (currentProfile !== undefined) { + moduleGraph.setProfile(module, currentProfile); + } + + if (originModule !== undefined) { + moduleGraph.setIssuer(module, originModule); + } + } else { + if (currentProfile !== undefined) { + currentProfile.mergeInto(moduleGraph.getProfile(module)); + } + } + + this.buildModule(module, error => { + if (error) { + const err = /** @type {WebpackError} */ (error); + if (!err.module) { + err.module = module; + } + this.errors.push(err); + + return callback(err); + } + + // This avoids deadlocks for circular dependencies + if (this.processDependenciesQueue.isProcessing(module)) { + return callback(); + } + + this.processModuleDependencies(module, err => { + if (err) { + return callback(err); + } + callback(null, module); + }); + }); + } + ); + } + + /** + * @typedef {Object} FactorizeModuleOptions + * @property {ModuleProfile} currentProfile + * @property {ModuleFactory} factory + * @property {Dependency[]} dependencies + * @property {Module | null} originModule + * @property {string=} context + */ + + /** + * @param {FactorizeModuleOptions} options options object + * @param {ModuleCallback} callback callback + * @returns {void} + */ + factorizeModule(options, callback) { + this.factorizeQueue.add(options, callback); + } + + /** + * @param {FactorizeModuleOptions} options options object + * @param {ModuleCallback} callback callback + * @returns {void} + */ + _factorizeModule( + { currentProfile, factory, dependencies, originModule, context }, + callback + ) { + if (currentProfile !== undefined) { + currentProfile.markFactoryStart(); + } + factory.create( + { + contextInfo: { + issuer: originModule ? originModule.nameForCondition() : "", + compiler: this.compiler.name + }, + resolveOptions: originModule ? originModule.resolveOptions : undefined, + context: context + ? context + : originModule + ? originModule.context + : this.compiler.context, + dependencies: dependencies + }, + (err, newModule) => { + if (err) { + const notFoundError = new ModuleNotFoundError( + originModule, + err, + dependencies.map(d => d.loc).filter(Boolean)[0] + ); + return callback(notFoundError); + } + if (!newModule) { + return callback(); + } + if (currentProfile !== undefined) { + currentProfile.markFactoryEnd(); + } + + callback(null, newModule); } ); } @@ -851,7 +827,7 @@ class Compilation { * * @param {string} context context string path * @param {Dependency} dependency dependency used to create Module chain - * @param {ModuleChainCallback} callback callback for when module chain is complete + * @param {ModuleCallback} callback callback for when module chain is complete * @returns {void} will throw if dependency instance is not a valid Dependency */ addModuleChain(context, dependency, callback) { @@ -877,10 +853,18 @@ class Compilation { factory: moduleFactory, dependencies: [dependency], originModule: null, - context, - bail: this.bail + context }, - callback + err => { + if (this.bail) { + this.buildQueue.stop(); + this.rebuildQueue.stop(); + this.processDependenciesQueue.stop(); + this.factorizeQueue.stop(); + return callback(err); + } + return callback(); + } ); } @@ -905,28 +889,24 @@ class Compilation { /** * @param {Module} module module to be rebuilt - * @param {Callback} thisCallback callback when module finishes rebuilding + * @param {ModuleCallback} callback callback when module finishes rebuilding * @returns {void} */ - rebuildModule(module, thisCallback) { - let callbackList = this._rebuildingModules.get(module); - if (callbackList) { - callbackList.push(thisCallback); - return; - } - this._rebuildingModules.set(module, (callbackList = [thisCallback])); - - const callback = err => { - this._rebuildingModules.delete(module); - for (const cb of callbackList) { - cb(err); - } - }; + rebuildModule(module, callback) { + this.rebuildQueue.add(module, callback); + } + /** + * @param {Module} module module to be rebuilt + * @param {ModuleCallback} callback callback when module finishes rebuilding + * @returns {void} + */ + _rebuildModule(module, callback) { this.hooks.rebuildModule.call(module); const oldDependencies = module.dependencies.slice(); const oldBlocks = module.blocks.slice(); - module.unbuild(); + module.invalidateBuild(); + this.buildQueue.invalidate(module); this.buildModule(module, err => { if (err) { this.hooks.finishRebuildingModule.call(module); diff --git a/lib/ContextModule.js b/lib/ContextModule.js index 98c7d733a..382017641 100644 --- a/lib/ContextModule.js +++ b/lib/ContextModule.js @@ -9,6 +9,7 @@ const { OriginalSource, RawSource } = require("webpack-sources"); const AsyncDependenciesBlock = require("./AsyncDependenciesBlock"); const Module = require("./Module"); const Template = require("./Template"); +const WebpackError = require("./WebpackError"); const { compareModulesById } = require("./util/comparators"); const contextify = require("./util/identifier").contextify; @@ -79,6 +80,7 @@ class ContextModule extends Module { } this._identifier = this._createIdentifier(); + this._forceBuild = true; } /** @@ -215,12 +217,20 @@ class ContextModule extends Module { return identifier; } + /** + * @returns {void} + */ + invalidateBuild() { + this._forceBuild = true; + } + /** * @param {TODO} fileTimestamps timestamps of files * @param {TODO} contextTimestamps timestamps of directories * @returns {boolean} true, if the module needs a rebuild */ - needRebuild(fileTimestamps, contextTimestamps) { + needBuild(fileTimestamps, contextTimestamps) { + if (this._forceBuild) return true; const ts = contextTimestamps.get(this.context); if (!ts) { return true; @@ -234,15 +244,18 @@ class ContextModule extends Module { * @param {Compilation} compilation the compilation * @param {TODO} resolver TODO * @param {TODO} fs the file system - * @param {function(Error=): void} callback callback function + * @param {function(WebpackError=): void} callback callback function * @returns {void} */ build(options, compilation, resolver, fs, callback) { + this._forceBuild = false; this.buildMeta = {}; this.buildInfo = { builtTime: Date.now(), contextDependencies: this._contextDependencies }; + this.dependencies.length = 0; + this.blocks.length = 0; this.resolveDependencies(fs, this.options, (err, dependencies) => { if (err) return callback(err); @@ -318,7 +331,7 @@ class ContextModule extends Module { } } else { callback( - new Error(`Unsupported mode "${this.options.mode}" in context`) + new WebpackError(`Unsupported mode "${this.options.mode}" in context`) ); return; } diff --git a/lib/DelegatedModule.js b/lib/DelegatedModule.js index 9b7cd0157..ed763ccb8 100644 --- a/lib/DelegatedModule.js +++ b/lib/DelegatedModule.js @@ -19,6 +19,7 @@ const DelegatedSourceDependency = require("./dependencies/DelegatedSourceDepende /** @typedef {import("./Module").SourceContext} SourceContext */ /** @typedef {import("./RequestShortener")} RequestShortener */ /** @typedef {import("./RuntimeTemplate")} RuntimeTemplate */ +/** @typedef {import("./WebpackError")} WebpackError */ /** @typedef {import("./dependencies/ModuleDependency")} ModuleDependency */ /** @typedef {import("./util/createHash").Hash} Hash */ @@ -70,8 +71,8 @@ class DelegatedModule extends Module { * @param {TODO} contextTimestamps timestamps of directories * @returns {boolean} true, if the module needs a rebuild */ - needRebuild(fileTimestamps, contextTimestamps) { - return false; + needBuild(fileTimestamps, contextTimestamps) { + return !this.buildMeta; } /** @@ -79,12 +80,13 @@ class DelegatedModule extends Module { * @param {Compilation} compilation the compilation * @param {TODO} resolver TODO * @param {TODO} fs the file system - * @param {function(Error=): void} callback callback function + * @param {function(WebpackError=): void} callback callback function * @returns {void} */ build(options, compilation, resolver, fs, callback) { this.buildMeta = Object.assign({}, this.delegateData.buildMeta); this.buildInfo = {}; + this.dependencies.length = 0; this.delegatedSourceDependency = new DelegatedSourceDependency( this.sourceRequest ); diff --git a/lib/DllModule.js b/lib/DllModule.js index 71609a535..bd11e9445 100644 --- a/lib/DllModule.js +++ b/lib/DllModule.js @@ -15,6 +15,7 @@ const Module = require("./Module"); /** @typedef {import("./Module").SourceContext} SourceContext */ /** @typedef {import("./RequestShortener")} RequestShortener */ /** @typedef {import("./RuntimeTemplate")} RuntimeTemplate */ +/** @typedef {import("./WebpackError")} WebpackError */ /** @typedef {import("./util/createHash").Hash} Hash */ class DllModule extends Module { @@ -46,7 +47,7 @@ class DllModule extends Module { * @param {Compilation} compilation the compilation * @param {TODO} resolver TODO * @param {TODO} fs the file system - * @param {function(Error=): void} callback callback function + * @param {function(WebpackError=): void} callback callback function * @returns {void} */ build(options, compilation, resolver, fs, callback) { @@ -68,8 +69,8 @@ class DllModule extends Module { * @param {TODO} contextTimestamps timestamps of directories * @returns {boolean} true, if the module needs a rebuild */ - needRebuild(fileTimestamps, contextTimestamps) { - return false; + needBuild(fileTimestamps, contextTimestamps) { + return !this.buildMeta; } /** diff --git a/lib/ExternalModule.js b/lib/ExternalModule.js index ba4ca6324..3ee9aef24 100644 --- a/lib/ExternalModule.js +++ b/lib/ExternalModule.js @@ -18,6 +18,7 @@ const Template = require("./Template"); /** @typedef {import("./Module").SourceContext} SourceContext */ /** @typedef {import("./RequestShortener")} RequestShortener */ /** @typedef {import("./RuntimeTemplate")} RuntimeTemplate */ +/** @typedef {import("./WebpackError")} WebpackError */ /** @typedef {import("./util/createHash").Hash} Hash */ /** @@ -151,8 +152,8 @@ class ExternalModule extends Module { * @param {TODO} contextTimestamps timestamps of directories * @returns {boolean} true, if the module needs a rebuild */ - needRebuild(fileTimestamps, contextTimestamps) { - return false; + needBuild(fileTimestamps, contextTimestamps) { + return !this.buildMeta; } /** @@ -160,7 +161,7 @@ class ExternalModule extends Module { * @param {Compilation} compilation the compilation * @param {TODO} resolver TODO * @param {TODO} fs the file system - * @param {function(Error=): void} callback callback function + * @param {function(WebpackError=): void} callback callback function * @returns {void} */ build(options, compilation, resolver, fs, callback) { diff --git a/lib/Module.js b/lib/Module.js index 59c92c679..62cd70bc2 100644 --- a/lib/Module.js +++ b/lib/Module.js @@ -450,6 +450,18 @@ class Module extends DependenciesBlock { * @param {TODO} contextTimestamps timestamps of directories * @returns {boolean} true, if the module needs a rebuild */ + needBuild(fileTimestamps, contextTimestamps) { + return ( + !this.buildMeta || this.needRebuild(fileTimestamps, contextTimestamps) + ); + } + + /** + * @deprecated Use needBuild instead + * @param {TODO} fileTimestamps timestamps of files + * @param {TODO} contextTimestamps timestamps of directories + * @returns {boolean} true, if the module needs a rebuild + */ needRebuild(fileTimestamps, contextTimestamps) { return true; } @@ -475,11 +487,8 @@ class Module extends DependenciesBlock { /** * @returns {void} */ - unbuild() { - this.dependencies.length = 0; - this.blocks.length = 0; - this.buildMeta = undefined; - this.buildInfo = undefined; + invalidateBuild() { + // should be overriden to support this feature } /** @@ -505,7 +514,7 @@ class Module extends DependenciesBlock { * @param {Compilation} compilation the compilation * @param {TODO} resolver TODO * @param {TODO} fs the file system - * @param {function(Error=): void} callback callback function + * @param {function(WebpackError=): void} callback callback function * @returns {void} */ build(options, compilation, resolver, fs, callback) { diff --git a/lib/ModuleProfile.js b/lib/ModuleProfile.js index 27749af06..92d6e675f 100644 --- a/lib/ModuleProfile.js +++ b/lib/ModuleProfile.js @@ -15,9 +15,13 @@ class ModuleProfile { this.additionalIntegration = 0; } + markFactoryStart() { + this.factoryStartTime = Date.now(); + } + markFactoryEnd() { - this.factoryTime = Date.now(); - this.factory = this.factoryTime - this.startTime; + this.factoryEndTime = Date.now(); + this.factory = this.factoryEndTime - this.factoryStartTime; } markIntegrationStart() { diff --git a/lib/NormalModule.js b/lib/NormalModule.js index 7d593ee45..e06f21226 100644 --- a/lib/NormalModule.js +++ b/lib/NormalModule.js @@ -31,6 +31,7 @@ const contextify = require("./util/identifier").contextify; /** @typedef {import("./Module").SourceContext} SourceContext */ /** @typedef {import("./RequestShortener")} RequestShortener */ /** @typedef {import("./RuntimeTemplate")} RuntimeTemplate */ +/** @typedef {import("./WebpackError")} WebpackError */ /** @typedef {import("./util/createHash").Hash} Hash */ const asString = buf => { @@ -106,6 +107,7 @@ class NormalModule extends Module { // Cache this._lastSuccessfulBuildMeta = {}; + this._forceBuild = true; } /** @@ -417,17 +419,20 @@ class NormalModule extends Module { * @param {Compilation} compilation the compilation * @param {TODO} resolver TODO * @param {TODO} fs the file system - * @param {function(Error=): void} callback callback function + * @param {function(WebpackError=): void} callback callback function * @returns {void} */ build(options, compilation, resolver, fs, callback) { this.buildTimestamp = Date.now(); + this._forceBuild = false; this._source = null; this._ast = null; this._buildHash = ""; this.error = null; this.errors.length = 0; this.warnings.length = 0; + this.dependencies.length = 0; + this.blocks.length = 0; this.buildMeta = {}; this.buildInfo = { cacheable: false, @@ -549,12 +554,22 @@ class NormalModule extends Module { return this._source; } + /** + * @returns {void} + */ + invalidateBuild() { + this._forceBuild = true; + } + /** * @param {TODO} fileTimestamps timestamps of files * @param {TODO} contextTimestamps timestamps of directories * @returns {boolean} true, if the module needs a rebuild */ - needRebuild(fileTimestamps, contextTimestamps) { + needBuild(fileTimestamps, contextTimestamps) { + // build if enforced + if (this._forceBuild) return true; + // always try to rebuild in case of an error if (this.error) return true; diff --git a/lib/RawModule.js b/lib/RawModule.js index 8d0558eec..52f140dd5 100644 --- a/lib/RawModule.js +++ b/lib/RawModule.js @@ -15,6 +15,7 @@ const Module = require("./Module"); /** @typedef {import("./Module").SourceContext} SourceContext */ /** @typedef {import("./RequestShortener")} RequestShortener */ /** @typedef {import("./RuntimeTemplate")} RuntimeTemplate */ +/** @typedef {import("./WebpackError")} WebpackError */ /** @typedef {import("./util/createHash").Hash} Hash */ module.exports = class RawModule extends Module { @@ -52,8 +53,8 @@ module.exports = class RawModule extends Module { * @param {TODO} contextTimestamps timestamps of directories * @returns {boolean} true, if the module needs a rebuild */ - needRebuild(fileTimestamps, contextTimestamps) { - return false; + needBuild(fileTimestamps, contextTimestamps) { + return !this.buildMeta; } /** @@ -61,7 +62,7 @@ module.exports = class RawModule extends Module { * @param {Compilation} compilation the compilation * @param {TODO} resolver TODO * @param {TODO} fs the file system - * @param {function(Error=): void} callback callback function + * @param {function(WebpackError=): void} callback callback function * @returns {void} */ build(options, compilation, resolver, fs, callback) { diff --git a/lib/dependencies/LoaderPlugin.js b/lib/dependencies/LoaderPlugin.js index 0c060a63a..3d633753b 100644 --- a/lib/dependencies/LoaderPlugin.js +++ b/lib/dependencies/LoaderPlugin.js @@ -5,7 +5,6 @@ "use strict"; -const NormalModule = require("../NormalModule"); const LoaderDependency = require("./LoaderDependency"); /** @typedef {import("../Module")} Module */ @@ -57,61 +56,50 @@ class LoaderPlugin { ) ); } - compilation.semaphore.release(); - compilation.addModuleDependencies( - module, - [ - { - factory, - dependencies: [dep] - } - ], - true, + compilation.buildQueue.increaseParallelism(); + compilation.handleModuleCreation( + { + factory, + dependencies: [dep], + originModule: loaderContext._module, + context: loaderContext.context + }, err => { - compilation.semaphore.acquire(() => { - if (err) { - return callback(err); + compilation.buildQueue.decreaseParallelism(); + if (err) { + return callback(err); + } + const referencedModule = moduleGraph.getModule(dep); + if (!referencedModule) { + return callback(new Error("Cannot load the module")); + } + const moduleSource = referencedModule.originalSource(); + if (!moduleSource) { + throw new Error( + "The module created for a LoaderDependency must have an original source" + ); + } + let source, map; + if (moduleSource.sourceAndMap) { + const sourceAndMap = moduleSource.sourceAndMap(); + map = sourceAndMap.map; + source = sourceAndMap.source; + } else { + map = moduleSource.map(); + source = moduleSource.source(); + } + if (referencedModule.buildInfo.fileDependencies) { + for (const d of referencedModule.buildInfo.fileDependencies) { + loaderContext.addDependency(d); } - const referencedModule = moduleGraph.getModule(dep); - if (!referencedModule) { - return callback(new Error("Cannot load the module")); + } + if (referencedModule.buildInfo.contextDependencies) { + for (const d of referencedModule.buildInfo + .contextDependencies) { + loaderContext.addContextDependency(d); } - // TODO consider removing this in webpack 5 - if ( - referencedModule instanceof NormalModule && - referencedModule.error - ) { - return callback(referencedModule.error); - } - const moduleSource = referencedModule.originalSource(); - if (!moduleSource) { - throw new Error( - "The module created for a LoaderDependency must have an original source" - ); - } - let source, map; - if (moduleSource.sourceAndMap) { - const sourceAndMap = moduleSource.sourceAndMap(); - map = sourceAndMap.map; - source = sourceAndMap.source; - } else { - map = moduleSource.map(); - source = moduleSource.source(); - } - if (referencedModule.buildInfo.fileDependencies) { - for (const d of referencedModule.buildInfo - .fileDependencies) { - loaderContext.addDependency(d); - } - } - if (referencedModule.buildInfo.contextDependencies) { - for (const d of referencedModule.buildInfo - .contextDependencies) { - loaderContext.addContextDependency(d); - } - } - return callback(null, source, map, referencedModule); - }); + } + return callback(null, source, map, referencedModule); } ); }; diff --git a/lib/optimize/ConcatenatedModule.js b/lib/optimize/ConcatenatedModule.js index 2a49f8348..4bb30d34b 100644 --- a/lib/optimize/ConcatenatedModule.js +++ b/lib/optimize/ConcatenatedModule.js @@ -33,6 +33,7 @@ const createHash = require("../util/createHash"); /** @typedef {import("../ModuleGraph")} ModuleGraph */ /** @typedef {import("../RequestShortener")} RequestShortener */ /** @typedef {import("../RuntimeTemplate")} RuntimeTemplate */ +/** @typedef {import("../WebpackError")} WebpackError */ /** @typedef {import("../util/createHash").Hash} Hash */ /** @@ -441,7 +442,7 @@ class ConcatenatedModule extends Module { * @param {Compilation} compilation the compilation * @param {TODO} resolver TODO * @param {TODO} fs the file system - * @param {function(Error=): void} callback callback function + * @param {function(WebpackError=): void} callback callback function * @returns {void} */ build(options, compilation, resolver, fs, callback) { diff --git a/lib/util/AsyncQueue.js b/lib/util/AsyncQueue.js new file mode 100644 index 000000000..c8d749359 --- /dev/null +++ b/lib/util/AsyncQueue.js @@ -0,0 +1,224 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const { SyncHook, AsyncSeriesHook } = require("tapable"); + +/** @template R @typedef {(err?: Error|null, result?: R) => void} Callback */ + +/** + * @template T + * @template R + */ +class AsyncQueue { + /** + * @param {Object} options options object + * @param {string=} options.name name of the queue + * @param {number} options.parallelism how many items should be processed at once + * @param {function(T, Callback): void} options.processor async function to process items + */ + constructor({ name, parallelism, processor }) { + this._name = name; + this._parallelism = parallelism; + this._processor = processor; + /** @type {Map[]>} */ + this._callbacks = new Map(); + /** @type {Set} */ + this._queued = new Set(); + /** @type {Set} */ + this._processing = new Set(); + /** @type {Map} */ + this._results = new Map(); + this._activeTasks = 0; + this._willEnsureProcessing = false; + this._stopped = false; + + this.hooks = { + beforeAdd: new AsyncSeriesHook(["item"]), + added: new SyncHook(["item"]), + beforeStart: new AsyncSeriesHook(["item"]), + started: new SyncHook(["item"]), + result: new SyncHook(["item", "error", "result"]) + }; + + this._ensureProcessing = this._ensureProcessing.bind(this); + } + + /** + * @param {T} item a item + * @param {Callback} callback callback function + * @returns {void} + */ + add(item, callback) { + if (this._stopped) return callback(new Error("Queue was stopped")); + this.hooks.beforeAdd.callAsync(item, err => { + if (err) { + callback(err); + return; + } + const result = this._results.get(item); + if (result !== undefined) { + process.nextTick(() => callback(result[0], result[1])); + return; + } + let callbacks = this._callbacks.get(item); + if (callbacks !== undefined) { + callbacks.push(callback); + return; + } + callbacks = [callback]; + this._callbacks.set(item, callbacks); + if (this._stopped) { + this.hooks.added.call(item); + this._activeTasks++; + this._handleResult(item, new Error("Queue was stopped")); + } else { + this._queued.add(item); + if (this._willEnsureProcessing === false) { + this._willEnsureProcessing = true; + process.nextTick(this._ensureProcessing); + } + this.hooks.added.call(item); + } + }); + } + + /** + * @param {T} item a item + * @returns {void} + */ + invalidate(item) { + this._results.delete(item); + } + + /** + * @returns {void} + */ + stop() { + this._stopped = true; + for (const item of this._queued) { + this._activeTasks++; + this._queued.delete(item); + this._handleResult(item, new Error("Queue was stopped")); + } + } + + /** + * @returns {void} + */ + increaseParallelism() { + this._parallelism++; + if (this._willEnsureProcessing === false && this._queued.size > 0) { + this._willEnsureProcessing = true; + process.nextTick(this._ensureProcessing); + } + } + + /** + * @returns {void} + */ + decreaseParallelism() { + this._parallelism--; + } + + /** + * @param {T} item an item + * @returns {boolean} true, if the item is currently being processed + */ + isProcessing(item) { + return this._processing.has(item); + } + + /** + * @param {T} item an item + * @returns {boolean} true, if the item is currently queued + */ + isQueued(item) { + return this._queued.has(item); + } + + /** + * @param {T} item an item + * @returns {boolean} true, if the item is currently queued + */ + isDone(item) { + return this._results.has(item); + } + + /** + * @returns {void} + */ + _ensureProcessing() { + if (this._activeTasks >= this._parallelism) { + this._willEnsureProcessing = false; + return; + } + for (const item of this._queued) { + this._activeTasks++; + this._queued.delete(item); + this._processing.add(item); + this._startProcessing(item); + if (this._activeTasks >= this._parallelism) { + this._willEnsureProcessing = false; + return; + } + } + this._willEnsureProcessing = false; + } + + /** + * @param {T} item an item + * @returns {void} + */ + _startProcessing(item) { + this.hooks.beforeStart.callAsync(item, err => { + if (err) { + this._handleResult(item, err); + return; + } + try { + this._processor(item, (e, r) => { + process.nextTick(() => { + this._handleResult(item, e, r); + }); + }); + } catch (err) { + console.error(err); + this._handleResult(item, err, null); + } + this.hooks.started.call(item); + }); + } + + /** + * @param {T} item an item + * @param {Error=} err error, if any + * @param {R=} result result, if any + * @returns {void} + */ + _handleResult(item, err, result) { + this.hooks.result.callAsync(item, err, result, hookError => { + const error = hookError || err; + + const callbacks = this._callbacks.get(item); + this._processing.delete(item); + this._results.set(item, [error, result]); + this._callbacks.delete(item); + this._activeTasks--; + + if (this._willEnsureProcessing === false && this._queued.size > 0) { + this._willEnsureProcessing = true; + process.nextTick(this._ensureProcessing); + } + + for (const callback of callbacks) { + callback(error, result); + } + }); + } +} + +module.exports = AsyncQueue; diff --git a/test/NormalModule.unittest.js b/test/NormalModule.unittest.js index 1ed0a3ad5..e1faf0c13 100644 --- a/test/NormalModule.unittest.js +++ b/test/NormalModule.unittest.js @@ -190,7 +190,7 @@ describe("NormalModule", () => { expect(normalModule.hasDependencies()).toBe(false); }); }); - describe("#needRebuild", () => { + describe("#needBuild", () => { let fileTimestamps; let contextTimestamps; let fileDependencies; @@ -211,13 +211,14 @@ describe("NormalModule", () => { fileTimestamps = new Map([[fileA, 1], [fileB, 1]]); contextTimestamps = new Map([[fileA, 1], [fileB, 1]]); normalModule.buildTimestamp = 2; + normalModule._forceBuild = false; setDeps(fileDependencies, contextDependencies); }); describe("given all timestamps are older than the buildTimestamp", () => { it("returns false", () => { - expect( - normalModule.needRebuild(fileTimestamps, contextTimestamps) - ).toBe(false); + expect(normalModule.needBuild(fileTimestamps, contextTimestamps)).toBe( + false + ); }); }); describe("given a file timestamp is newer than the buildTimestamp", () => { @@ -225,9 +226,9 @@ describe("NormalModule", () => { fileTimestamps.set(fileA, 3); }); it("returns true", () => { - expect( - normalModule.needRebuild(fileTimestamps, contextTimestamps) - ).toBe(true); + expect(normalModule.needBuild(fileTimestamps, contextTimestamps)).toBe( + true + ); }); }); describe("given a no file timestamp exists", () => { @@ -235,9 +236,9 @@ describe("NormalModule", () => { fileTimestamps = new Map(); }); it("returns true", () => { - expect( - normalModule.needRebuild(fileTimestamps, contextTimestamps) - ).toBe(true); + expect(normalModule.needBuild(fileTimestamps, contextTimestamps)).toBe( + true + ); }); }); describe("given a context timestamp is newer than the buildTimestamp", () => { @@ -245,9 +246,9 @@ describe("NormalModule", () => { contextTimestamps.set(fileA, 3); }); it("returns true", () => { - expect( - normalModule.needRebuild(fileTimestamps, contextTimestamps) - ).toBe(true); + expect(normalModule.needBuild(fileTimestamps, contextTimestamps)).toBe( + true + ); }); }); describe("given a no context timestamp exists", () => { @@ -255,9 +256,9 @@ describe("NormalModule", () => { contextTimestamps = new Map(); }); it("returns true", () => { - expect( - normalModule.needRebuild(fileTimestamps, contextTimestamps) - ).toBe(true); + expect(normalModule.needBuild(fileTimestamps, contextTimestamps)).toBe( + true + ); }); }); }); diff --git a/test/RawModule.unittest.js b/test/RawModule.unittest.js index aa1ab2202..117a6d83c 100644 --- a/test/RawModule.unittest.js +++ b/test/RawModule.unittest.js @@ -36,12 +36,6 @@ describe("RawModule", () => { ); }); - describe("needRebuild", () => { - it("returns false", () => { - expect(myRawModule.needRebuild()).toBe(false); - }); - }); - describe("source", () => { it( "returns a new OriginalSource instance with sourceStr attribute and " +