Merge pull request #14319 from webpack/feature/affected-mem-cache

declarations/WebpackOptions.d.ts

@@ -880,6 +880,10 @@ export interface WebpackOptions {
  * Options object for in-memory caching.
  */
 export interface MemoryCacheOptions {
+	/**
+	 * Additionally cache computation of modules that are unchanged and reference only unchanged modules.
+	 */
+	cacheUnaffected?: boolean;
 	/**
 	 * Number of generations unused cache entries stay in memory cache at minimum (1 = may be removed after unused for a single compilation, ..., Infinity: kept forever).
 	 */
@@ -950,6 +954,10 @@ export interface FileCacheOptions {
 	 * Number of generations unused cache entries stay in memory cache at minimum (0 = no memory cache used, 1 = may be removed after unused for a single compilation, ..., Infinity: kept forever). Cache entries will be deserialized from disk when removed from memory cache.
 	 */
 	maxMemoryGenerations?: number;
+	/**
+	 * Additionally cache computation of modules that are unchanged and reference only unchanged modules in memory.
+	 */
+	memoryCacheUnaffected?: boolean;
 	/**
 	 * Name for the cache. Different names will lead to different coexisting caches.
 	 */
@@ -1106,6 +1114,10 @@ export interface Experiments {
 	 * Build http(s): urls using a lockfile and resource content cache.
 	 */
 	buildHttp?: boolean | HttpUriOptions;
+	/**
+	 * Enable additional in memory caching of modules that are unchanged and reference only unchanged modules.
+	 */
+	cacheUnaffected?: boolean;
 	/**
 	 * Apply defaults of next major version.
 	 */

lib/ChunkGraph.js

@@ -1376,22 +1376,29 @@ Caller might not support runtime-dependent code generation (opt-out via optimiza
 	/**
 	 * @param {Module} module the module
 	 * @param {RuntimeSpec} runtime the runtime
-	 * @param {Set<string>} items runtime requirements to be added (ownership of this Set is given to ChunkGraph)
+	 * @param {Set<string>} items runtime requirements to be added (ownership of this Set is given to ChunkGraph when transferOwnership not false)
+	 * @param {boolean} transferOwnership true: transfer ownership of the items object, false: items is immutable and shared and won't be modified
 	 * @returns {void}
 	 */
-	addModuleRuntimeRequirements(module, runtime, items) {
+	addModuleRuntimeRequirements(
+		module,
+		runtime,
+		items,
+		transferOwnership = true
+	) {
 		const cgm = this._getChunkGraphModule(module);
 		const runtimeRequirementsMap = cgm.runtimeRequirements;
 		if (runtimeRequirementsMap === undefined) {
 			const map = new RuntimeSpecMap();
-			map.set(runtime, items);
+			// TODO avoid cloning item and track ownership instead
+			map.set(runtime, transferOwnership ? items : new Set(items));
 			cgm.runtimeRequirements = map;
 			return;
 		}
 		runtimeRequirementsMap.update(runtime, runtimeRequirements => {
 			if (runtimeRequirements === undefined) {
-				return items;
-			} else if (runtimeRequirements.size >= items.size) {
+				return transferOwnership ? items : new Set(items);
+			} else if (!transferOwnership || runtimeRequirements.size >= items.size) {
 				for (const item of items) runtimeRequirements.add(item);
 				return runtimeRequirements;
 			} else {

lib/Compilation.js

@@ -38,6 +38,7 @@ const {
 	tryRunOrWebpackError
 } = require("./HookWebpackError");
 const MainTemplate = require("./MainTemplate");
+const MemCache = require("./MemCache");
 const Module = require("./Module");
 const ModuleDependencyError = require("./ModuleDependencyError");
 const ModuleDependencyWarning = require("./ModuleDependencyWarning");
@@ -76,7 +77,7 @@ const {
 	createFakeHook
 } = require("./util/deprecation");
 const processAsyncTree = require("./util/processAsyncTree");
-const { getRuntimeKey } = require("./util/runtime");
+const { getRuntimeKey, RuntimeSpecMap } = require("./util/runtime");
 const { isSourceEqual } = require("./util/source");
 
 /** @template T @typedef {import("tapable").AsArray<T>} AsArray<T> */
@@ -892,6 +893,10 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
 		};
 		defineRemovedModuleTemplates(this.moduleTemplates);
 
+		/** @type {MemCache | undefined} */
+		this.memCache = undefined;
+		/** @type {WeakMap<Module, MemCache> | undefined} */
+		this.moduleMemCaches = undefined;
 		this.moduleGraph = new ModuleGraph();
 		/** @type {ChunkGraph} */
 		this.chunkGraph = undefined;
@@ -2012,6 +2017,79 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
 		});
 	}
 
+	_computeAffectedModules(modules) {
+		const moduleMemCacheCache = this.compiler.moduleMemCaches;
+		if (!moduleMemCacheCache) return;
+		if (!this.moduleMemCaches) this.moduleMemCaches = new WeakMap();
+		if (!this.memCache) this.memCache = new MemCache();
+		const { moduleGraph, memCache, moduleMemCaches } = this;
+		const affectedModules = new Set();
+		const infectedModules = new Set();
+		let statNew = 0;
+		let statChanged = 0;
+		let statUnchanged = 0;
+		let statWithoutHash = 0;
+		for (const module of modules) {
+			const hash = module.buildInfo && module.buildInfo.hash;
+			if (typeof hash === "string") {
+				const cachedMemCache = moduleMemCacheCache.get(module);
+				if (cachedMemCache === undefined) {
+					// create a new entry
+					moduleMemCacheCache.set(module, {
+						hash: hash,
+						memCache
+					});
+					moduleMemCaches.set(module, memCache);
+					affectedModules.add(module);
+					statNew++;
+				} else if (cachedMemCache.hash === hash) {
+					// keep the old mem cache
+					moduleMemCaches.set(module, cachedMemCache.memCache);
+					statUnchanged++;
+				} else {
+					// use a new one
+					moduleMemCaches.set(module, memCache);
+					affectedModules.add(module);
+					cachedMemCache.hash = hash;
+					cachedMemCache.memCache = memCache;
+					statChanged++;
+				}
+			} else {
+				infectedModules.add(module);
+				statWithoutHash++;
+			}
+		}
+		for (const module of infectedModules) {
+			for (const referencingModule of moduleGraph
+				.getIncomingConnectionsByOriginModule(module)
+				.keys()) {
+				if (infectedModules.has(referencingModule)) continue;
+				infectedModules.add(referencingModule);
+			}
+		}
+		for (const module of affectedModules) {
+			for (const referencingModule of moduleGraph
+				.getIncomingConnectionsByOriginModule(module)
+				.keys()) {
+				if (!referencingModule) continue;
+				if (infectedModules.has(referencingModule)) continue;
+				if (affectedModules.has(referencingModule)) continue;
+				affectedModules.add(referencingModule);
+				moduleMemCaches.set(referencingModule, memCache);
+			}
+		}
+		this.logger.log(
+			`${Math.round(
+				(100 * (affectedModules.size + infectedModules.size)) /
+					this.modules.size
+			)}% (${affectedModules.size} affected + ${
+				infectedModules.size
+			} infected of ${
+				this.modules.size
+			}) modules flagged as affected (${statNew} new modules, ${statChanged} changed, ${statUnchanged} unchanged, ${statWithoutHash} without hash)`
+		);
+	}
+
 	finish(callback) {
 		this.factorizeQueue.clear();
 		if (this.profile) {
@@ -2192,17 +2270,29 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
 			);
 			this.logger.timeEnd("finish module profiles");
 		}
+		this.logger.time("compute affected modules");
+		this._computeAffectedModules(this.modules);
+		this.logger.timeEnd("compute affected modules");
 		this.logger.time("finish modules");
-		const { modules } = this;
+		const { modules, moduleMemCaches } = this;
 		this.hooks.finishModules.callAsync(modules, err => {
 			this.logger.timeEnd("finish modules");
 			if (err) return callback(err);
 
 			// extract warnings and errors from modules
-			this.logger.time("report dependency errors and warnings");
 			this.moduleGraph.freeze();
+			// TODO keep a cacheToken (= {}) for each module in the graph
+			// create a new one per compilation and flag all updated files
+			// and parents with it
+			this.logger.time("report dependency errors and warnings");
 			for (const module of modules) {
-				this.reportDependencyErrorsAndWarnings(module, [module]);
+				// TODO only run for modules with changed cacheToken
+				// global WeakMap<CacheToken, WeakSet<Module>> to keep modules without errors/warnings
+				const memCache = moduleMemCaches && moduleMemCaches.get(module);
+				if (memCache && memCache.get(module, "noWarningsOrErrors")) continue;
+				let hasProblems = this.reportDependencyErrorsAndWarnings(module, [
+					module
+				]);
 				const errors = module.getErrors();
 				if (errors !== undefined) {
 					for (const error of errors) {
@@ -2210,6 +2300,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
 							error.module = module;
 						}
 						this.errors.push(error);
+						hasProblems = true;
 					}
 				}
 				const warnings = module.getWarnings();
@@ -2219,8 +2310,11 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
 							warning.module = module;
 						}
 						this.warnings.push(warning);
+						hasProblems = true;
 					}
 				}
+				if (!hasProblems && memCache)
+					memCache.set(module, "noWarningsOrErrors", true);
 			}
 			this.moduleGraph.unfreeze();
 			this.logger.timeEnd("report dependency errors and warnings");
@@ -2577,9 +2671,10 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 	/**
 	 * @param {Module} module module to report from
 	 * @param {DependenciesBlock[]} blocks blocks to report from
-	 * @returns {void}
+	 * @returns {boolean} true, when it has warnings or errors
 	 */
 	reportDependencyErrorsAndWarnings(module, blocks) {
+		let hasProblems = false;
 		for (let indexBlock = 0; indexBlock < blocks.length; indexBlock++) {
 			const block = blocks[indexBlock];
 			const dependencies = block.dependencies;
@@ -2594,6 +2689,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 
 						const warning = new ModuleDependencyWarning(module, w, d.loc);
 						this.warnings.push(warning);
+						hasProblems = true;
 					}
 				}
 				const errors = d.getErrors(this.moduleGraph);
@@ -2603,12 +2699,15 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 
 						const error = new ModuleDependencyError(module, e, d.loc);
 						this.errors.push(error);
+						hasProblems = true;
 					}
 				}
 			}
 
-			this.reportDependencyErrorsAndWarnings(module, block.blocks);
+			if (this.reportDependencyErrorsAndWarnings(module, block.blocks))
+				hasProblems = true;
 		}
+		return hasProblems;
 	}
 
 	codeGeneration(callback) {
@@ -2796,12 +2895,41 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 		chunkGraphEntries = this._getChunkGraphEntries()
 	} = {}) {
 		const context = { chunkGraph, codeGenerationResults };
+		const { moduleMemCaches } = this;
+		this.logger.time("runtime requirements.modules");
 		const additionalModuleRuntimeRequirements =
 			this.hooks.additionalModuleRuntimeRequirements;
 		const runtimeRequirementInModule = this.hooks.runtimeRequirementInModule;
 		for (const module of modules) {
 			if (chunkGraph.getNumberOfModuleChunks(module) > 0) {
+				const memCache =
+					moduleMemCaches &&
+					// modules with async blocks depend on the chunk graph and can't be cached that way
+					module.blocks.length === 0 &&
+					moduleMemCaches.get(module);
+				/** @type {RuntimeSpecMap<Set<string>>} */
+				const moduleRuntimeRequirementsMemCache =
+					memCache &&
+					memCache.provide(
+						module,
+						"moduleRuntimeRequirements",
+						() => new RuntimeSpecMap()
+					);
 				for (const runtime of chunkGraph.getModuleRuntimes(module)) {
+					if (moduleRuntimeRequirementsMemCache) {
+						const cached = moduleRuntimeRequirementsMemCache.get(runtime);
+						if (cached !== undefined) {
+							if (cached !== null) {
+								chunkGraph.addModuleRuntimeRequirements(
+									module,
+									runtime,
+									cached,
+									false
+								);
+							}
+							continue;
+						}
+					}
 					let set;
 					const runtimeRequirements =
 						codeGenerationResults.getRuntimeRequirements(module, runtime);
@@ -2810,6 +2938,9 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 					} else if (additionalModuleRuntimeRequirements.isUsed()) {
 						set = new Set();
 					} else {
+						if (moduleRuntimeRequirementsMemCache) {
+							moduleRuntimeRequirementsMemCache.set(runtime, null);
+						}
 						continue;
 					}
 					additionalModuleRuntimeRequirements.call(module, set, context);
@@ -2818,11 +2949,29 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 						const hook = runtimeRequirementInModule.get(r);
 						if (hook !== undefined) hook.call(module, set, context);
 					}
-					chunkGraph.addModuleRuntimeRequirements(module, runtime, set);
+					if (set.size === 0) {
+						if (moduleRuntimeRequirementsMemCache) {
+							moduleRuntimeRequirementsMemCache.set(runtime, null);
+						}
+					} else {
+						if (moduleRuntimeRequirementsMemCache) {
+							moduleRuntimeRequirementsMemCache.set(runtime, set);
+							chunkGraph.addModuleRuntimeRequirements(
+								module,
+								runtime,
+								set,
+								false
+							);
+						} else {
+							chunkGraph.addModuleRuntimeRequirements(module, runtime, set);
+						}
+					}
 				}
 			}
 		}
+		this.logger.timeEnd("runtime requirements.modules");
 
+		this.logger.time("runtime requirements.chunks");
 		for (const chunk of chunks) {
 			const set = new Set();
 			for (const module of chunkGraph.getChunkModulesIterable(chunk)) {
@@ -2840,7 +2989,9 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 
 			chunkGraph.addChunkRuntimeRequirements(chunk, set);
 		}
+		this.logger.timeEnd("runtime requirements.chunks");
 
+		this.logger.time("runtime requirements.entries");
 		for (const treeEntry of chunkGraphEntries) {
 			const set = new Set();
 			for (const chunk of treeEntry.getAllReferencedChunks()) {
@@ -2863,6 +3014,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 
 			chunkGraph.addTreeRuntimeRequirements(treeEntry, set);
 		}
+		this.logger.timeEnd("runtime requirements.entries");
 	}
 
 	// TODO webpack 6 make chunkGraph argument non-optional
@@ -3207,12 +3359,35 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 
 	createModuleHashes() {
 		let statModulesHashed = 0;
-		const { chunkGraph, runtimeTemplate } = this;
+		let statModulesFromCache = 0;
+		const { chunkGraph, runtimeTemplate, moduleMemCaches } = this;
 		const { hashFunction, hashDigest, hashDigestLength } = this.outputOptions;
 		for (const module of this.modules) {
+			const memCache =
+				moduleMemCaches &&
+				// modules with async blocks depend on the chunk graph and can't be cached that way
+				module.blocks.length === 0 &&
+				moduleMemCaches.get(module);
+			/** @type {RuntimeSpecMap<string>} */
+			const moduleHashesMemCache =
+				memCache &&
+				memCache.provide(module, "moduleHashes", () => new RuntimeSpecMap());
 			for (const runtime of chunkGraph.getModuleRuntimes(module)) {
+				if (moduleHashesMemCache) {
+					const digest = moduleHashesMemCache.get(runtime);
+					if (digest !== undefined) {
+						chunkGraph.setModuleHashes(
+							module,
+							runtime,
+							digest,
+							digest.substr(0, hashDigestLength)
+						);
+						statModulesFromCache++;
+						continue;
+					}
+				}
 				statModulesHashed++;
-				this._createModuleHash(
+				const digest = this._createModuleHash(
 					module,
 					chunkGraph,
 					runtime,
@@ -3221,11 +3396,16 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
 					hashDigest,
 					hashDigestLength
 				);
+				if (moduleHashesMemCache) {
+					moduleHashesMemCache.set(runtime, digest);
+				}
 			}
 		}
 		this.logger.log(
-			`${statModulesHashed} modules hashed (${
-				Math.round((100 * statModulesHashed) / this.modules.size) / 100
+			`${statModulesHashed} modules hashed, ${statModulesFromCache} from cache (${
+				Math.round(
+					(100 * (statModulesHashed + statModulesFromCache)) / this.modules.size
+				) / 100
 			} variants per module in average)`
 		);
 	}

lib/Compiler.js

@@ -41,6 +41,7 @@ const { isSourceEqual } = require("./util/source");
 /** @typedef {import("../declarations/WebpackOptions").WebpackPluginInstance} WebpackPluginInstance */
 /** @typedef {import("./Chunk")} Chunk */
 /** @typedef {import("./FileSystemInfo").FileSystemInfoEntry} FileSystemInfoEntry */
+/** @typedef {import("./MemCache")} MemCache */
 /** @typedef {import("./Module")} Module */
 /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
 /** @typedef {import("./util/fs").IntermediateFileSystem} IntermediateFileSystem */
@@ -247,6 +248,9 @@ class Compiler {
 
 		this.cache = new Cache();
 
+		/** @type {WeakMap<Module, { hash: string, memCache: MemCache }> | undefined} */
+		this.moduleMemCaches = undefined;
+
 		this.compilerPath = "";
 
 		/** @type {boolean} */

lib/FlagDependencyExportsPlugin.js

@@ -28,12 +28,14 @@ class FlagDependencyExportsPlugin {
 			compilation => {
 				const moduleGraph = compilation.moduleGraph;
 				const cache = compilation.getCache("FlagDependencyExportsPlugin");
+				const { moduleMemCaches } = compilation;
 				compilation.hooks.finishModules.tapAsync(
 					"FlagDependencyExportsPlugin",
 					(modules, callback) => {
 						const logger = compilation.getLogger(
 							"webpack.FlagDependencyExportsPlugin"
 						);
+						let statRestoredFromMemCache = 0;
 						let statRestoredFromCache = 0;
 						let statNoExports = 0;
 						let statFlaggedUncached = 0;
@@ -58,16 +60,22 @@ class FlagDependencyExportsPlugin {
 										return callback();
 									}
 								}
-								if (
-									module.buildInfo.cacheable !== true ||
-									typeof module.buildInfo.hash !== "string"
-								) {
+								if (typeof module.buildInfo.hash !== "string") {
 									statFlaggedUncached++;
 									// Enqueue uncacheable module for determining the exports
 									queue.enqueue(module);
 									exportsInfo.setHasProvideInfo();
 									return callback();
 								}
+								const memCache = moduleMemCaches && moduleMemCaches.get(module);
+								const memCacheValue = memCache && memCache.get(this, module);
+								if (memCacheValue !== undefined) {
+									statRestoredFromMemCache++;
+									moduleGraph
+										.getExportsInfo(module)
+										.restoreProvided(memCacheValue);
+									return callback();
+								}
 								cache.get(
 									module.identifier(),
 									module.buildInfo.hash,
@@ -94,7 +102,9 @@ class FlagDependencyExportsPlugin {
 								if (err) return callback(err);
 
 								/** @type {Set<Module>} */
-								const modulesToStore = new Set();
+								const modulesToStorePersistent = new Set();
+								/** @type {Set<Module>} */
+								const modulesToStoreTransient = new Set();
 
 								/** @type {Map<Module, Set<Module>>} */
 								const dependencies = new Map();
@@ -328,7 +338,9 @@ class FlagDependencyExportsPlugin {
 									}
 
 									if (cacheable) {
-										modulesToStore.add(module);
+										modulesToStorePersistent.add(module);
+									} else {
+										modulesToStoreTransient.add(module);
 									}
 
 									if (changed) {
@@ -340,11 +352,12 @@ class FlagDependencyExportsPlugin {
 								logger.log(
 									`${Math.round(
 										(100 * (statFlaggedUncached + statNotCached)) /
-											(statRestoredFromCache +
+											(statRestoredFromMemCache +
+												statRestoredFromCache +
 												statNotCached +
 												statFlaggedUncached +
 												statNoExports)
-									)}% of exports of modules have been determined (${statNoExports} no declared exports, ${statNotCached} not cached, ${statFlaggedUncached} flagged uncacheable, ${statRestoredFromCache} from cache, ${
+									)}% of exports of modules have been determined (${statNoExports} no declared exports, ${statNotCached} not cached, ${statFlaggedUncached} flagged uncacheable, ${statRestoredFromCache} from cache, ${statRestoredFromMemCache} from mem cache, ${
 										statQueueItemsProcessed -
 										statNotCached -
 										statFlaggedUncached
@@ -353,26 +366,40 @@ class FlagDependencyExportsPlugin {
 
 								logger.time("store provided exports into cache");
 								asyncLib.each(
-									modulesToStore,
+									modulesToStorePersistent,
 									(module, callback) => {
-										if (
-											module.buildInfo.cacheable !== true ||
-											typeof module.buildInfo.hash !== "string"
-										) {
+										if (typeof module.buildInfo.hash !== "string") {
 											// not cacheable
 											return callback();
 										}
+										const cachedData = moduleGraph
+											.getExportsInfo(module)
+											.getRestoreProvidedData();
+										const memCache =
+											moduleMemCaches && moduleMemCaches.get(module);
+										if (memCache) {
+											memCache.set(this, module, cachedData);
+										}
 										cache.store(
 											module.identifier(),
 											module.buildInfo.hash,
-											moduleGraph
-												.getExportsInfo(module)
-												.getRestoreProvidedData(),
+											cachedData,
 											callback
 										);
 									},
 									err => {
 										logger.timeEnd("store provided exports into cache");
+										if (moduleMemCaches) {
+											for (const module of modulesToStoreTransient) {
+												const memCache = moduleMemCaches.get(module);
+												if (memCache) {
+													const cachedData = moduleGraph
+														.getExportsInfo(module)
+														.getRestoreProvidedData();
+													memCache.set(this, module, cachedData);
+												}
+											}
+										}
 										callback(err);
 									}
 								);

lib/MemCache.js

@@ -0,0 +1,45 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+	Author Tobias Koppers @sokra
+*/
+
+"use strict";
+
+const WeakTupleMap = require("./util/WeakTupleMap");
+
+class MemCache {
+	constructor() {
+		this._cache = new WeakTupleMap();
+	}
+
+	/**
+	 * @template {any[]} T
+	 * @template V
+	 * @param {T} args arguments
+	 * @returns {V | undefined} cached value
+	 */
+	get(...args) {
+		return this._cache.get(...args);
+	}
+
+	/**
+	 * @template {[...any[], any]} T
+	 * @param {T} args arguments
+	 * @returns {void}
+	 */
+	set(...args) {
+		this._cache.set(...args);
+	}
+
+	/**
+	 * @template {[...any[], (...args: any[]) => V]} T
+	 * @template V
+	 * @param {T} args arguments
+	 * @returns {V} computed value or cached
+	 */
+	provide(...args) {
+		return this._cache.provide(...args);
+	}
+}
+
+module.exports = MemCache;

lib/WebpackOptionsApply.js

@@ -544,6 +544,14 @@ class WebpackOptionsApply extends OptionsApply {
 						const MemoryCachePlugin = require("./cache/MemoryCachePlugin");
 						new MemoryCachePlugin().apply(compiler);
 					}
+					if (cacheOptions.cacheUnaffected) {
+						if (!options.experiments.cacheUnaffected) {
+							throw new Error(
+								"'cache.cacheUnaffected: true' is only allowed when 'experiments.cacheUnaffected' is enabled"
+							);
+						}
+						compiler.moduleMemCaches = new WeakMap();
+					}
 					break;
 				}
 				case "filesystem": {
@@ -563,6 +571,14 @@ class WebpackOptionsApply extends OptionsApply {
 							maxGenerations: cacheOptions.maxMemoryGenerations
 						}).apply(compiler);
 					}
+					if (cacheOptions.memoryCacheUnaffected) {
+						if (!options.experiments.cacheUnaffected) {
+							throw new Error(
+								"'cache.memoryCacheUnaffected: true' is only allowed when 'experiments.cacheUnaffected' is enabled"
+							);
+						}
+						compiler.moduleMemCaches = new WeakMap();
+					}
 					switch (cacheOptions.store) {
 						case "pack": {
 							const IdleFileCachePlugin = require("./cache/IdleFileCachePlugin");

lib/config/defaults.js

@@ -159,20 +159,21 @@ const applyWebpackOptionsDefaults = options => {
 	D(options, "recordsInputPath", false);
 	D(options, "recordsOutputPath", false);
 
+	applyExperimentsDefaults(options.experiments, { production, development });
+
 	F(options, "cache", () =>
 		development ? { type: /** @type {"memory"} */ ("memory") } : false
 	);
 	applyCacheDefaults(options.cache, {
 		name: name || "default",
 		mode: mode || "production",
-		development
+		development,
+		cacheUnaffected: options.experiments.cacheUnaffected
 	});
 	const cache = !!options.cache;
 
 	applySnapshotDefaults(options.snapshot, { production });
 
-	applyExperimentsDefaults(options.experiments, { production, development });
-
 	applyModuleDefaults(options.module, {
 		cache,
 		syncWebAssembly: options.experiments.syncWebAssembly,
@@ -267,6 +268,7 @@ const applyExperimentsDefaults = (experiments, { production, development }) => {
 	D(experiments, "lazyCompilation", false);
 	D(experiments, "buildHttp", false);
 	D(experiments, "futureDefaults", false);
+	D(experiments, "cacheUnaffected", experiments.futureDefaults);
 
 	if (typeof experiments.buildHttp === "object") {
 		D(experiments.buildHttp, "frozen", production);
@@ -280,9 +282,13 @@ const applyExperimentsDefaults = (experiments, { production, development }) => {
  * @param {string} options.name name
  * @param {string} options.mode mode
  * @param {boolean} options.development is development mode
+ * @param {boolean} options.cacheUnaffected the cacheUnaffected experiment is enabled
  * @returns {void}
  */
-const applyCacheDefaults = (cache, { name, mode, development }) => {
+const applyCacheDefaults = (
+	cache,
+	{ name, mode, development, cacheUnaffected }
+) => {
 	if (cache === false) return;
 	switch (cache.type) {
 		case "filesystem":
@@ -326,12 +332,14 @@ const applyCacheDefaults = (cache, { name, mode, development }) => {
 			D(cache, "maxMemoryGenerations", development ? 5 : Infinity);
 			D(cache, "maxAge", 1000 * 60 * 60 * 24 * 60); // 1 month
 			D(cache, "allowCollectingMemory", development);
+			D(cache, "memoryCacheUnaffected", development && cacheUnaffected);
 			D(cache.buildDependencies, "defaultWebpack", [
 				path.resolve(__dirname, "..") + path.sep
 			]);
 			break;
 		case "memory":
 			D(cache, "maxGenerations", Infinity);
+			D(cache, "cacheUnaffected", development && cacheUnaffected);
 			break;
 	}
 };

schemas/WebpackOptions.json

@@ -699,6 +699,10 @@
             }
           ]
         },
+        "cacheUnaffected": {
+          "description": "Enable additional in memory caching of modules that are unchanged and reference only unchanged modules.",
+          "type": "boolean"
+        },
         "futureDefaults": {
           "description": "Apply defaults of next major version.",
           "type": "boolean"
@@ -1027,6 +1031,10 @@
           "type": "number",
           "minimum": 0
         },
+        "memoryCacheUnaffected": {
+          "description": "Additionally cache computation of modules that are unchanged and reference only unchanged modules in memory.",
+          "type": "boolean"
+        },
         "name": {
           "description": "Name for the cache. Different names will lead to different coexisting caches.",
           "type": "string"
@@ -1655,6 +1663,10 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
+        "cacheUnaffected": {
+          "description": "Additionally cache computation of modules that are unchanged and reference only unchanged modules.",
+          "type": "boolean"
+        },
         "maxGenerations": {
           "description": "Number of generations unused cache entries stay in memory cache at minimum (1 = may be removed after unused for a single compilation, ..., Infinity: kept forever).",
           "type": "number",

test/Defaults.unittest.js

@@ -94,6 +94,7 @@ Object {
     "asset": false,
     "asyncWebAssembly": false,
     "buildHttp": false,
+    "cacheUnaffected": false,
     "futureDefaults": false,
     "layers": false,
     "lazyCompilation": false,
@@ -778,48 +779,49 @@ Object {
 	);
 	test("development", { mode: "development" }, e =>
 		e.toMatchInlineSnapshot(`
-		- Expected
-		+ Received
+- Expected
++ Received
 
-		@@ ... @@
-		-   "cache": false,
-		+   "cache": Object {
-		+     "maxGenerations": Infinity,
-		+     "type": "memory",
-		+   },
-		@@ ... @@
-		-   "devtool": false,
-		+   "devtool": "eval",
-		@@ ... @@
-		-   "mode": "none",
-		+   "mode": "development",
-		@@ ... @@
-		-     "unsafeCache": false,
-		+     "unsafeCache": [Function anonymous],
-		@@ ... @@
-		-     "chunkIds": "natural",
-		+     "chunkIds": "named",
-		@@ ... @@
-		-     "moduleIds": "natural",
-		-     "nodeEnv": false,
-		+     "moduleIds": "named",
-		+     "nodeEnv": "development",
-		@@ ... @@
-		-       "minRemainingSize": undefined,
-		+       "minRemainingSize": 0,
-		@@ ... @@
-		-     "pathinfo": false,
-		+     "pathinfo": true,
-		@@ ... @@
-		-     "cache": false,
-		+     "cache": true,
-		@@ ... @@
-		-       "production",
-		+       "development",
-		@@ ... @@
-		-     "cache": false,
-		+     "cache": true,
-	`)
+@@ ... @@
+-   "cache": false,
++   "cache": Object {
++     "cacheUnaffected": false,
++     "maxGenerations": Infinity,
++     "type": "memory",
++   },
+@@ ... @@
+-   "devtool": false,
++   "devtool": "eval",
+@@ ... @@
+-   "mode": "none",
++   "mode": "development",
+@@ ... @@
+-     "unsafeCache": false,
++     "unsafeCache": [Function anonymous],
+@@ ... @@
+-     "chunkIds": "natural",
++     "chunkIds": "named",
+@@ ... @@
+-     "moduleIds": "natural",
+-     "nodeEnv": false,
++     "moduleIds": "named",
++     "nodeEnv": "development",
+@@ ... @@
+-       "minRemainingSize": undefined,
++       "minRemainingSize": 0,
+@@ ... @@
+-     "pathinfo": false,
++     "pathinfo": true,
+@@ ... @@
+-     "cache": false,
++     "cache": true,
+@@ ... @@
+-       "production",
++       "development",
+@@ ... @@
+-     "cache": false,
++     "cache": true,
+`)
 	);
 	test("sync wasm", { experiments: { syncWebAssembly: true } }, e =>
 		e.toMatchInlineSnapshot(`
@@ -1480,25 +1482,26 @@ Object {
 	);
 	test("cache true", { cache: true }, e =>
 		e.toMatchInlineSnapshot(`
-		- Expected
-		+ Received
+- Expected
++ Received
 
-		@@ ... @@
-		-   "cache": false,
-		+   "cache": Object {
-		+     "maxGenerations": Infinity,
-		+     "type": "memory",
-		+   },
-		@@ ... @@
-		-     "unsafeCache": false,
-		+     "unsafeCache": [Function anonymous],
-		@@ ... @@
-		-     "cache": false,
-		+     "cache": true,
-		@@ ... @@
-		-     "cache": false,
-		+     "cache": true,
-	`)
+@@ ... @@
+-   "cache": false,
++   "cache": Object {
++     "cacheUnaffected": false,
++     "maxGenerations": Infinity,
++     "type": "memory",
++   },
+@@ ... @@
+-     "unsafeCache": false,
++     "unsafeCache": [Function anonymous],
+@@ ... @@
+-     "cache": false,
++     "cache": true,
+@@ ... @@
+-     "cache": false,
++     "cache": true,
+`)
 	);
 	test("cache filesystem", { cache: { type: "filesystem" } }, e =>
 		e.toMatchInlineSnapshot(`
@@ -1523,6 +1526,7 @@ Object {
 +     "idleTimeoutForInitialStore": 5000,
 +     "maxAge": 5184000000,
 +     "maxMemoryGenerations": Infinity,
++     "memoryCacheUnaffected": false,
 +     "name": "default-none",
 +     "profile": false,
 +     "store": "pack",
@@ -1545,66 +1549,67 @@ Object {
 		{ mode: "development", cache: { type: "filesystem" } },
 		e =>
 			e.toMatchInlineSnapshot(`
-			- Expected
-			+ Received
+- Expected
++ Received
 
-			@@ ... @@
-			-   "cache": false,
-			+   "cache": Object {
-			+     "allowCollectingMemory": true,
-			+     "buildDependencies": Object {
-			+       "defaultWebpack": Array [
-			+         "<cwd>/lib/",
-			+       ],
-			+     },
-			+     "cacheDirectory": "<cwd>/node_modules/.cache/webpack",
-			+     "cacheLocation": "<cwd>/node_modules/.cache/webpack/default-development",
-			+     "compression": false,
-			+     "hashAlgorithm": "md4",
-			+     "idleTimeout": 60000,
-			+     "idleTimeoutAfterLargeChanges": 1000,
-			+     "idleTimeoutForInitialStore": 5000,
-			+     "maxAge": 5184000000,
-			+     "maxMemoryGenerations": 5,
-			+     "name": "default-development",
-			+     "profile": false,
-			+     "store": "pack",
-			+     "type": "filesystem",
-			+     "version": "",
-			+   },
-			@@ ... @@
-			-   "devtool": false,
-			+   "devtool": "eval",
-			@@ ... @@
-			-   "mode": "none",
-			+   "mode": "development",
-			@@ ... @@
-			-     "unsafeCache": false,
-			+     "unsafeCache": [Function anonymous],
-			@@ ... @@
-			-     "chunkIds": "natural",
-			+     "chunkIds": "named",
-			@@ ... @@
-			-     "moduleIds": "natural",
-			-     "nodeEnv": false,
-			+     "moduleIds": "named",
-			+     "nodeEnv": "development",
-			@@ ... @@
-			-       "minRemainingSize": undefined,
-			+       "minRemainingSize": 0,
-			@@ ... @@
-			-     "pathinfo": false,
-			+     "pathinfo": true,
-			@@ ... @@
-			-     "cache": false,
-			+     "cache": true,
-			@@ ... @@
-			-       "production",
-			+       "development",
-			@@ ... @@
-			-     "cache": false,
-			+     "cache": true,
-		`)
+@@ ... @@
+-   "cache": false,
++   "cache": Object {
++     "allowCollectingMemory": true,
++     "buildDependencies": Object {
++       "defaultWebpack": Array [
++         "<cwd>/lib/",
++       ],
++     },
++     "cacheDirectory": "<cwd>/node_modules/.cache/webpack",
++     "cacheLocation": "<cwd>/node_modules/.cache/webpack/default-development",
++     "compression": false,
++     "hashAlgorithm": "md4",
++     "idleTimeout": 60000,
++     "idleTimeoutAfterLargeChanges": 1000,
++     "idleTimeoutForInitialStore": 5000,
++     "maxAge": 5184000000,
++     "maxMemoryGenerations": 5,
++     "memoryCacheUnaffected": false,
++     "name": "default-development",
++     "profile": false,
++     "store": "pack",
++     "type": "filesystem",
++     "version": "",
++   },
+@@ ... @@
+-   "devtool": false,
++   "devtool": "eval",
+@@ ... @@
+-   "mode": "none",
++   "mode": "development",
+@@ ... @@
+-     "unsafeCache": false,
++     "unsafeCache": [Function anonymous],
+@@ ... @@
+-     "chunkIds": "natural",
++     "chunkIds": "named",
+@@ ... @@
+-     "moduleIds": "natural",
+-     "nodeEnv": false,
++     "moduleIds": "named",
++     "nodeEnv": "development",
+@@ ... @@
+-       "minRemainingSize": undefined,
++       "minRemainingSize": 0,
+@@ ... @@
+-     "pathinfo": false,
++     "pathinfo": true,
+@@ ... @@
+-     "cache": false,
++     "cache": true,
+@@ ... @@
+-       "production",
++       "development",
+@@ ... @@
+-     "cache": false,
++     "cache": true,
+`)
 	);
 
 	test(
@@ -1814,6 +1819,7 @@ Object {
 +     "idleTimeoutForInitialStore": 5000,
 +     "maxAge": 5184000000,
 +     "maxMemoryGenerations": Infinity,
++     "memoryCacheUnaffected": false,
 +     "name": "default-none",
 +     "profile": false,
 +     "store": "pack",
@@ -1890,7 +1896,9 @@ Object {
 + Received
 
 @@ ... @@
+-     "cacheUnaffected": false,
 -     "futureDefaults": false,
++     "cacheUnaffected": true,
 +     "futureDefaults": true,
 @@ ... @@
 -     "__dirname": "mock",

test/__snapshots__/Cli.test.js.snap

@@ -95,6 +95,19 @@ Object {
     "multiple": false,
     "simpleType": "string",
   },
+  "cache-cache-unaffected": Object {
+    "configs": Array [
+      Object {
+        "description": "Additionally cache computation of modules that are unchanged and reference only unchanged modules.",
+        "multiple": false,
+        "path": "cache.cacheUnaffected",
+        "type": "boolean",
+      },
+    ],
+    "description": "Additionally cache computation of modules that are unchanged and reference only unchanged modules.",
+    "multiple": false,
+    "simpleType": "boolean",
+  },
   "cache-compression": Object {
     "configs": Array [
       Object {
@@ -256,6 +269,19 @@ Object {
     "multiple": false,
     "simpleType": "number",
   },
+  "cache-memory-cache-unaffected": Object {
+    "configs": Array [
+      Object {
+        "description": "Additionally cache computation of modules that are unchanged and reference only unchanged modules in memory.",
+        "multiple": false,
+        "path": "cache.memoryCacheUnaffected",
+        "type": "boolean",
+      },
+    ],
+    "description": "Additionally cache computation of modules that are unchanged and reference only unchanged modules in memory.",
+    "multiple": false,
+    "simpleType": "boolean",
+  },
   "cache-name": Object {
     "configs": Array [
       Object {
@@ -524,6 +550,19 @@ Object {
     "multiple": false,
     "simpleType": "boolean",
   },
+  "experiments-cache-unaffected": Object {
+    "configs": Array [
+      Object {
+        "description": "Enable additional in memory caching of modules that are unchanged and reference only unchanged modules.",
+        "multiple": false,
+        "path": "experiments.cacheUnaffected",
+        "type": "boolean",
+      },
+    ],
+    "description": "Enable additional in memory caching of modules that are unchanged and reference only unchanged modules.",
+    "multiple": false,
+    "simpleType": "boolean",
+  },
   "experiments-future-defaults": Object {
     "configs": Array [
       Object {

test/__snapshots__/StatsTestCases.basictest.js.snap

@@ -1268,15 +1268,15 @@ asset <CLR=32,BOLD>main.js</CLR> 84 bytes <CLR=32,BOLD>[emitted]</CLR> (name: ma
 <CLR=31,BOLD>DEBUG</CLR> <CLR=BOLD>LOG from ./node_modules/custom-loader/index.js Named Logger ./node_modules/custom-loader/index.js!./index.js</CLR>
     Message with named logger
 
-<CLR=BOLD>LOG from webpack.FlagDependencyExportsPlugin</CLR>
-    <CLR=BOLD>0% of exports of modules have been determined (1 no declared exports, 0 not cached, 0 flagged uncacheable, 0 from cache, 0 additional calculations due to dependencies)</CLR>
-+ 3 hidden lines
-
 <CLR=BOLD>LOG from webpack.Compilation</CLR>
-    <CLR=BOLD>1 modules hashed (1 variants per module in average)</CLR>
+    <CLR=BOLD>1 modules hashed, 0 from cache (1 variants per module in average)</CLR>
     <CLR=BOLD>100% code generated (1 generated, 0 from cache)</CLR>
     <CLR=BOLD>NaN% code generated (0 generated, 0 from cache)</CLR>
-+ 19 hidden lines
++ 23 hidden lines
+
+<CLR=BOLD>LOG from webpack.FlagDependencyExportsPlugin</CLR>
+    <CLR=BOLD>0% of exports of modules have been determined (1 no declared exports, 0 not cached, 0 flagged uncacheable, 0 from cache, 0 from mem cache, 0 additional calculations due to dependencies)</CLR>
++ 3 hidden lines
 
 <CLR=BOLD>LOG from webpack.buildChunkGraph</CLR>
     <CLR=BOLD>2 queue items processed (1 blocks)</CLR>
@@ -2057,15 +2057,15 @@ LOG from LogTestPlugin
     End
 + 6 hidden lines
 
-LOG from webpack.FlagDependencyExportsPlugin
-    0% of exports of modules have been determined (6 no declared exports, 0 not cached, 0 flagged uncacheable, 0 from cache, 0 additional calculations due to dependencies)
-+ 3 hidden lines
-
 LOG from webpack.Compilation
-    6 modules hashed (1 variants per module in average)
+    6 modules hashed, 0 from cache (1 variants per module in average)
     100% code generated (6 generated, 0 from cache)
     100% code generated (7 generated, 0 from cache)
-+ 19 hidden lines
++ 23 hidden lines
+
+LOG from webpack.FlagDependencyExportsPlugin
+    0% of exports of modules have been determined (6 no declared exports, 0 not cached, 0 flagged uncacheable, 0 from cache, 0 from mem cache, 0 additional calculations due to dependencies)
++ 3 hidden lines
 
 LOG from webpack.buildChunkGraph
     15 queue items processed (9 blocks)
@@ -2371,15 +2371,19 @@ LOG from webpack.Compiler
 
 LOG from webpack.Compilation
 <t> finish module profiles: X ms
+<t> compute affected modules: X ms
 <t> finish modules: X ms
 <t> report dependency errors and warnings: X ms
 <t> optimize dependencies: X ms
 <t> create chunks: X ms
 <t> optimize: X ms
-    6 modules hashed (1 variants per module in average)
+    6 modules hashed, 0 from cache (1 variants per module in average)
 <t> module hashing: X ms
     100% code generated (6 generated, 0 from cache)
 <t> code generation: X ms
+<t> runtime requirements.modules: X ms
+<t> runtime requirements.chunks: X ms
+<t> runtime requirements.entries: X ms
 <t> runtime requirements: X ms
 <t> hashing: initialize hash: X ms
 <t> hashing: sort chunks: X ms
@@ -2397,7 +2401,7 @@ LOG from webpack.Compilation
 LOG from webpack.FlagDependencyExportsPlugin
 <t> restore cached provided exports: X ms
 <t> figure out provided exports: X ms
-    0% of exports of modules have been determined (6 no declared exports, 0 not cached, 0 flagged uncacheable, 0 from cache, 0 additional calculations due to dependencies)
+    0% of exports of modules have been determined (6 no declared exports, 0 not cached, 0 flagged uncacheable, 0 from cache, 0 from mem cache, 0 additional calculations due to dependencies)
 <t> store provided exports into cache: X ms
 
 LOG from webpack.InnerGraphPlugin

test/watchCases/cache/add-defines/webpack.config.js

@@ -4,7 +4,8 @@ const currentWatchStep = require("../../../helpers/currentWatchStep");
 /** @type {import("../../../../").Configuration} */
 module.exports = {
 	cache: {
-		type: "memory"
+		type: "memory",
+		cacheUnaffected: false
 	},
 	plugins: [
 		compiler => {

types.d.ts

@@ -891,7 +891,8 @@ declare class ChunkGraph {
 	addModuleRuntimeRequirements(
 		module: Module,
 		runtime: RuntimeSpec,
-		items: Set<string>
+		items: Set<string>,
+		transferOwnership?: boolean
 	): void;
 	addChunkRuntimeRequirements(chunk: Chunk, items: Set<string>): void;
 	addTreeRuntimeRequirements(chunk: Chunk, items: Iterable<string>): void;
@@ -1458,6 +1459,8 @@ declare class Compilation {
 	chunkTemplate: ChunkTemplate;
 	runtimeTemplate: RuntimeTemplate;
 	moduleTemplates: { javascript: ModuleTemplate };
+	memCache?: MemCache;
+	moduleMemCaches?: WeakMap<Module, MemCache>;
 	moduleGraph: ModuleGraph;
 	chunkGraph: ChunkGraph;
 	codeGenerationResults: CodeGenerationResults;
@@ -1594,7 +1597,7 @@ declare class Compilation {
 	reportDependencyErrorsAndWarnings(
 		module: Module,
 		blocks: DependenciesBlock[]
-	): void;
+	): boolean;
 	codeGeneration(callback?: any): void;
 	processRuntimeRequirements(__0?: {
 		/**
@@ -1894,6 +1897,7 @@ declare class Compiler {
 	context: string;
 	requestShortener: RequestShortener;
 	cache: Cache;
+	moduleMemCaches?: WeakMap<Module, { hash: string; memCache: MemCache }>;
 	compilerPath: string;
 	running: boolean;
 	idle: boolean;
@@ -3282,6 +3286,11 @@ declare interface Experiments {
 	 */
 	buildHttp?: boolean | HttpUriOptions;
 
+	/**
+	 * Enable additional in memory caching of modules that are unchanged and reference only unchanged modules.
+	 */
+	cacheUnaffected?: boolean;
+
 	/**
 	 * Apply defaults of next major version.
 	 */
@@ -3909,6 +3918,11 @@ declare interface FileCacheOptions {
 	 */
 	maxMemoryGenerations?: number;
 
+	/**
+	 * Additionally cache computation of modules that are unchanged and reference only unchanged modules in memory.
+	 */
+	memoryCacheUnaffected?: boolean;
+
 	/**
 	 * Name for the cache. Different names will lead to different coexisting caches.
 	 */
@@ -6299,11 +6313,21 @@ declare interface MapOptions {
 	columns?: boolean;
 	module?: boolean;
 }
+declare abstract class MemCache {
+	get<T extends any[], V>(...args: T): undefined | V;
+	set<T extends [any, ...any[]]>(...args: T): void;
+	provide<T extends [any, ...((...args: any[]) => V)[]], V>(...args: T): V;
+}
 
 /**
  * Options object for in-memory caching.
  */
 declare interface MemoryCacheOptions {
+	/**
+	 * Additionally cache computation of modules that are unchanged and reference only unchanged modules.
+	 */
+	cacheUnaffected?: boolean;
+
 	/**
 	 * Number of generations unused cache entries stay in memory cache at minimum (1 = may be removed after unused for a single compilation, ..., Infinity: kept forever).
 	 */