avoid leaking unused memory in buffer backing stores in development mode

declarations/WebpackOptions.d.ts

@@ -916,6 +916,10 @@ export interface MemoryCacheOptions {
  * Options object for persistent file-based caching.
  */
 export interface FileCacheOptions {
+	/**
+	 * Allows to collect unused memory allocated during deserialization. This requires copying data into smaller buffers and has a performance cost.
+	 */
+	allowCollectingMemory?: boolean;
 	/**
 	 * Dependencies the build depends on (in multiple categories, default categories: 'defaultWebpack').
 	 */

lib/WebpackOptionsApply.js

@@ -568,7 +568,8 @@ class WebpackOptionsApply extends OptionsApply {
 										"webpack.cache.PackFileCacheStrategy"
 									),
 									snapshot: options.snapshot,
-									maxAge: cacheOptions.maxAge
+									maxAge: cacheOptions.maxAge,
+									allowCollectingMemory: cacheOptions.allowCollectingMemory
 								}),
 								cacheOptions.idleTimeout,
 								cacheOptions.idleTimeoutForInitialStore

lib/cache/PackFileCacheStrategy.js

@@ -768,6 +768,14 @@ class PackContent {
 	}
 }
 
+const allowCollectingMemory = buf => {
+	const wasted = buf.buffer.byteLength - buf.byteLength;
+	if ((wasted > 8192 && wasted > 1048576) || wasted > buf.byteLength) {
+		return Buffer.from(buf);
+	}
+	return buf;
+};
+
 class PackFileCacheStrategy {
 	/**
 	 * @param {Object} options options
@@ -779,6 +787,7 @@ class PackFileCacheStrategy {
 	 * @param {Logger} options.logger a logger
 	 * @param {SnapshotOptions} options.snapshot options regarding snapshotting
 	 * @param {number} options.maxAge max age of cache items
+	 * @param {boolean} options.allowCollectingMemory allow to collect unused memory created during deserialization
 	 */
 	constructor({
 		compiler,
@@ -788,7 +797,8 @@ class PackFileCacheStrategy {
 		version,
 		logger,
 		snapshot,
-		maxAge
+		maxAge,
+		allowCollectingMemory
 	}) {
 		this.fileSerializer = createFileSerializer(fs);
 		this.fileSystemInfo = new FileSystemInfo(fs, {
@@ -802,6 +812,7 @@ class PackFileCacheStrategy {
 		this.version = version;
 		this.logger = logger;
 		this.maxAge = maxAge;
+		this.allowCollectingMemory = allowCollectingMemory;
 		this.snapshot = snapshot;
 		/** @type {Set<string>} */
 		this.buildDependencies = new Set();
@@ -845,7 +856,10 @@ class PackFileCacheStrategy {
 			.deserialize(null, {
 				filename: `${cacheLocation}/index.pack`,
 				extension: ".pack",
-				logger
+				logger,
+				retainedBuffer: this.allowCollectingMemory
+					? allowCollectingMemory
+					: undefined
 			})
 			.catch(err => {
 				if (err.code !== "ENOENT") {

lib/config/defaults.js

@@ -297,6 +297,7 @@ const applyCacheDefaults = (cache, { name, mode, development }) => {
 			D(cache, "idleTimeoutForInitialStore", 0);
 			D(cache, "maxMemoryGenerations", development ? 10 : Infinity);
 			D(cache, "maxAge", 1000 * 60 * 60 * 24 * 60); // 1 month
+			D(cache, "allowCollectingMemory", development);
 			D(cache.buildDependencies, "defaultWebpack", [
 				path.resolve(__dirname, "..") + path.sep
 			]);

lib/serialization/BinaryMiddleware.js

@@ -521,6 +521,8 @@ class BinaryMiddleware extends SerializerMiddleware {
 		let currentIsBuffer = Buffer.isBuffer(currentBuffer);
 		let currentPosition = 0;
 
+		const retainedBuffer = context.retainedBuffer || (x => x);
+
 		const checkOverflow = () => {
 			if (currentPosition >= currentBuffer.length) {
 				currentPosition = 0;
@@ -634,7 +636,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 								do {
 									const buf = readUpTo(l);
 									l -= buf.length;
-									content.push(buf);
+									content.push(retainedBuffer(buf));
 								} while (l > 0);
 							}
 						}
@@ -643,7 +645,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 				case BUFFER_HEADER:
 					return () => {
 						const len = readU32();
-						result.push(read(len));
+						result.push(retainedBuffer(read(len)));
 					};
 				case TRUE_HEADER:
 					return () => result.push(true);

schemas/WebpackOptions.json

@@ -924,6 +924,10 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
+        "allowCollectingMemory": {
+          "description": "Allows to collect unused memory allocated during deserialization. This requires copying data into smaller buffers and has a performance cost.",
+          "type": "boolean"
+        },
         "buildDependencies": {
           "description": "Dependencies the build depends on (in multiple categories, default categories: 'defaultWebpack').",
           "type": "object",

types.d.ts

@@ -3759,6 +3759,11 @@ declare class FetchCompileWasmPlugin {
  * Options object for persistent file-based caching.
  */
 declare interface FileCacheOptions {
+	/**
+	 * Allows to collect unused memory allocated during deserialization. This requires copying data into smaller buffers and has a performance cost.
+	 */
+	allowCollectingMemory?: boolean;
+
 	/**
 	 * Dependencies the build depends on (in multiple categories, default categories: 'defaultWebpack').
 	 */