diff --git a/lib/cache/PackFileCacheStrategy.js b/lib/cache/PackFileCacheStrategy.js index 6cd91cba8..cd6fd1da0 100644 --- a/lib/cache/PackFileCacheStrategy.js +++ b/lib/cache/PackFileCacheStrategy.js @@ -76,7 +76,9 @@ makeSerializable( const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB const CONTENT_COUNT_TO_MERGE = 10; +const MIN_ITEMS_IN_FRESH_PACK = 100; const MAX_ITEMS_IN_FRESH_PACK = 50000; +const MAX_TIME_IN_FRESH_PACK = 1 * 60 * 1000; // 1 min class PackItemInfo { /** @@ -99,6 +101,7 @@ class Pack { this.itemInfo = new Map(); /** @type {string[]} */ this.requests = []; + this.requestsTimeout = undefined; /** @type {Map} */ this.freshContent = new Map(); /** @type {(undefined | PackContent)[]} */ @@ -108,6 +111,16 @@ class Pack { this.maxAge = maxAge; } + _addRequest(identifier) { + this.requests.push(identifier); + if (this.requestsTimeout === undefined) { + this.requestsTimeout = setTimeout(() => { + this.requests.push(undefined); + this.requestsTimeout = undefined; + }, MAX_TIME_IN_FRESH_PACK); + } + } + /** * @param {string} identifier unique name for the resource * @param {string | null} etag etag of the resource @@ -115,7 +128,7 @@ class Pack { */ get(identifier, etag) { const info = this.itemInfo.get(identifier); - this.requests.push(identifier); + this._addRequest(identifier); if (info === undefined) { return undefined; } @@ -147,12 +160,12 @@ class Pack { if (info === undefined) { const newInfo = new PackItemInfo(identifier, etag, data); this.itemInfo.set(identifier, newInfo); - this.requests.push(identifier); + this._addRequest(identifier); this.freshContent.set(identifier, newInfo); } else { const loc = info.location; if (loc >= 0) { - this.requests.push(identifier); + this._addRequest(identifier); this.freshContent.set(identifier, info); const content = this.content[loc]; content.delete(identifier); @@ -220,27 +233,39 @@ class Pack { } _persistFreshContent() { - if (this.freshContent.size > 0) { - const packCount = Math.ceil( - this.freshContent.size / MAX_ITEMS_IN_FRESH_PACK - ); - const itemsPerPack = Math.ceil(this.freshContent.size / packCount); - this.logger.log(`${this.freshContent.size} fresh items in cache`); - const packs = Array.from({ length: packCount }, () => { + const itemsCount = this.freshContent.size; + if (itemsCount > 0) { + const packCount = Math.ceil(itemsCount / MAX_ITEMS_IN_FRESH_PACK); + const itemsPerPack = Math.ceil(itemsCount / packCount); + const packs = []; + let i = 0; + let ignoreNextTimeTick = false; + const createNextPack = () => { const loc = this._findLocation(); this.content[loc] = null; // reserve - return { + const pack = { /** @type {Set} */ items: new Set(), /** @type {Map} */ map: new Map(), loc }; - }); - let i = 0; - let pack = packs[0]; - let packIndex = 0; + packs.push(pack); + return pack; + }; + let pack = createNextPack(); + if (this.requestsTimeout !== undefined) + clearTimeout(this.requestsTimeout); for (const identifier of this.requests) { + if (identifier === undefined) { + if (ignoreNextTimeTick) { + ignoreNextTimeTick = false; + } else if (pack.items.size >= MIN_ITEMS_IN_FRESH_PACK) { + i = 0; + pack = createNextPack(); + } + continue; + } const info = this.freshContent.get(identifier); if (info === undefined) continue; pack.items.add(identifier); @@ -250,9 +275,11 @@ class Pack { this.freshContent.delete(identifier); if (++i > itemsPerPack) { i = 0; - pack = packs[++packIndex]; + pack = createNextPack(); + ignoreNextTimeTick = true; } } + this.requests.length = 0; for (const pack of packs) { this.content[pack.loc] = new PackContent( pack.items, @@ -260,6 +287,15 @@ class Pack { new PackContentItems(pack.map) ); } + this.logger.log( + `${itemsCount} fresh items in cache put into pack ${ + packs.length > 1 + ? packs + .map(pack => `${pack.loc} (${pack.items.size} items)`) + .join(", ") + : packs[0].loc + }` + ); } } @@ -332,7 +368,9 @@ class Pack { addToMergedMap.push(async map => { // unpack existing content // after that values are accessible in .content - await content.unpack(); + await content.unpack( + "it should be merged with other small pack contents" + ); for (const [identifier, value] of content.content) { map.set(identifier, value); } @@ -392,7 +430,9 @@ class Pack { usedItems, new Set(usedItems), async () => { - await content.unpack(); + await content.unpack( + "it should be splitted into used and unused items" + ); const map = new Map(); for (const identifier of usedItems) { map.set(identifier, content.content.get(identifier)); @@ -417,7 +457,9 @@ class Pack { unusedItems, usedOfUnusedItems, async () => { - await content.unpack(); + await content.unpack( + "it should be splitted into used and unused items" + ); const map = new Map(); for (const identifier of unusedItems) { map.set(identifier, content.content.get(identifier)); @@ -466,7 +508,9 @@ class Pack { this.content[loc] = items.size > 0 ? new PackContent(items, usedItems, async () => { - await content.unpack(); + await content.unpack( + "it contains old items that should be garbage collected" + ); const map = new Map(); for (const identifier of items) { map.set(identifier, content.content.get(identifier)); @@ -496,7 +540,7 @@ class Pack { const content = this.content[i]; if (content !== undefined) { write(content.items); - writeSeparate(content.getLazyContentItems(), { name: `${i}` }); + content.writeLazy(lazy => writeSeparate(lazy, { name: `${i}` })); } else { write(undefined); // undefined marks an empty content slot } @@ -669,6 +713,25 @@ makeSerializable( ); class PackContent { + /* + This class can be in these states: + | this.lazy | this.content | this.outdated | state + A1 | undefined | Map | false | fresh content + A2 | undefined | Map | true | (will not happen) + B1 | lazy () => {} | undefined | false | not deserialized + B2 | lazy () => {} | undefined | true | not deserialized, but some items has been removed + C1 | lazy* () => {} | Map | false | deserialized + C2 | lazy* () => {} | Map | true | deserialized, and some items has been removed + + this.used is a subset of this.items. + this.items is a subset of this.content.keys() resp. this.lazy().map.keys() + When this.outdated === false, this.items === this.content.keys() resp. this.lazy().map.keys() + When this.outdated === true, this.items should be used to recreated this.lazy/this.content. + When this.lazy and this.content is set, they contain the same data. + this.get must only be called with a valid item from this.items. + In state C this.lazy is unMemoized + */ + /** * @param {Set} items keys * @param {Set} usedItems used keys @@ -678,7 +741,7 @@ class PackContent { */ constructor(items, usedItems, dataOrFn, logger, lazyName) { this.items = items; - /** @type {function(): Promise | PackContentItems } */ + /** @type {function(): Promise | PackContentItems} */ this.lazy = typeof dataOrFn === "function" ? dataOrFn : undefined; /** @type {Map} */ this.content = typeof dataOrFn === "function" ? undefined : dataOrFn.map; @@ -693,6 +756,8 @@ class PackContent { if (this.content) { return this.content.get(identifier); } + + // We are in state B const { lazyName } = this; let timeMessage; if (lazyName) { @@ -715,6 +780,7 @@ class PackContent { if (timeMessage) { this.logger.timeEnd(timeMessage); } + // Move to state C this.content = map; this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy); return map.get(identifier); @@ -724,6 +790,7 @@ class PackContent { if (timeMessage) { this.logger.timeEnd(timeMessage); } + // Move to state C this.content = map; this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy); return map.get(identifier); @@ -731,10 +798,13 @@ class PackContent { } /** + * @param {string} reason explanation why unpack is necessary * @returns {void | Promise} maybe a promise if lazy */ - unpack() { + unpack(reason) { if (this.content) return; + + // Move from state B to C if (this.lazy) { const { lazyName } = this; let timeMessage; @@ -744,6 +814,11 @@ class PackContent { timeMessage = `unpack cache content ${lazyName} (${formatSize( this.getSize() )})`; + this.logger.log( + `starting to unpack cache content ${lazyName} (${formatSize( + this.getSize() + )}) because ${reason}` + ); this.logger.time(timeMessage); } const value = this.lazy(); @@ -782,48 +857,93 @@ class PackContent { } /** - * @returns {function(): PackContentItems | Promise} lazy content items + * @template T + * @param {function(any): function(): Promise | PackContentItems} write write function + * @returns {void} */ - getLazyContentItems() { - if (!this.outdated && this.lazy) return this.lazy; + writeLazy(write) { + if (!this.outdated && this.lazy) { + // State B1 or C1 + // this.lazy is still the valid deserialized version + write(this.lazy); + return; + } if (!this.outdated && this.content) { + // State A1 const map = new Map(this.content); - return (this.lazy = memoize(() => new PackContentItems(map))); + // Move to state C1 + this.lazy = SerializerMiddleware.unMemoizeLazy( + write(() => new PackContentItems(map)) + ); + return; } - this.outdated = false; if (this.content) { - return (this.lazy = memoize(() => { - /** @type {Map} */ - const map = new Map(); - for (const item of this.items) { - map.set(item, this.content.get(item)); - } - return new PackContentItems(map); - })); + // State A2 or C2 + /** @type {Map} */ + const map = new Map(); + for (const item of this.items) { + map.set(item, this.content.get(item)); + } + // Move to state C1 + this.outdated = false; + this.content = map; + this.lazy = SerializerMiddleware.unMemoizeLazy( + write(() => new PackContentItems(map)) + ); + return; } - const lazy = this.lazy; - return (this.lazy = () => { - const value = lazy(); - if (value instanceof Promise) { - return value.then(data => { + // State B2 + const { lazyName } = this; + let timeMessage; + if (lazyName) { + // only log once + this.lazyName = undefined; + timeMessage = `unpack cache content ${lazyName} (${formatSize( + this.getSize() + )})`; + this.logger.log( + `starting to unpack cache content ${lazyName} (${formatSize( + this.getSize() + )}) because it's outdated and need to be serialized` + ); + this.logger.time(timeMessage); + } + const value = this.lazy(); + this.outdated = false; + if (value instanceof Promise) { + // Move to state B1 + this.lazy = write(() => + value.then(data => { + if (timeMessage) { + this.logger.timeEnd(timeMessage); + } const oldMap = data.map; /** @type {Map} */ const map = new Map(); for (const item of this.items) { map.set(item, oldMap.get(item)); } + // Move to state C1 (or maybe C2) + this.content = map; + this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy); + return new PackContentItems(map); - }); - } else { - const oldMap = value.map; - /** @type {Map} */ - const map = new Map(); - for (const item of this.items) { - map.set(item, oldMap.get(item)); - } - return new PackContentItems(map); + }) + ); + } else { + // Move to state C1 + if (timeMessage) { + this.logger.timeEnd(timeMessage); } - }); + const oldMap = value.map; + /** @type {Map} */ + const map = new Map(); + for (const item of this.items) { + map.set(item, oldMap.get(item)); + } + this.content = map; + this.lazy = write(() => new PackContentItems(map)); + } } } @@ -1110,10 +1230,10 @@ class PackFileCacheStrategy { const packPromise = this.packPromise; if (packPromise === undefined) return Promise.resolve(); const reportProgress = ProgressPlugin.getReporter(this.compiler); - this.packPromise = undefined; return (this.storePromise = packPromise .then(pack => { if (!pack.invalid) return; + this.packPromise = undefined; this.logger.log(`Storing pack...`); let promise; const newBuildDependencies = new Set(); diff --git a/lib/serialization/ObjectMiddleware.js b/lib/serialization/ObjectMiddleware.js index 7c8f9a915..ad7e3f083 100644 --- a/lib/serialization/ObjectMiddleware.js +++ b/lib/serialization/ObjectMiddleware.js @@ -510,11 +510,11 @@ class ObjectMiddleware extends SerializerMiddleware { } else if (SerializerMiddleware.isLazy(item, this)) { throw new Error("Not implemented"); } else { - result.push( - SerializerMiddleware.serializeLazy(item, data => - this.serialize([data], context) - ) + const data = SerializerMiddleware.serializeLazy(item, data => + this.serialize([data], context) ); + SerializerMiddleware.setLazySerializedValue(item, data); + result.push(data); } } else if (item === undefined) { result.push(ESCAPE, ESCAPE_UNDEFINED); diff --git a/lib/util/serialization.js b/lib/util/serialization.js index 46d97d851..cc3bf61cd 100644 --- a/lib/util/serialization.js +++ b/lib/util/serialization.js @@ -111,9 +111,13 @@ module.exports = { ); }; context.writeSeparate = (value, options) => { - context.write( - SerializerMiddleware.createLazy(value, fileMiddleware, options) + const lazy = SerializerMiddleware.createLazy( + value, + fileMiddleware, + options ); + context.write(lazy); + return lazy; }; } }),