root
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Actions 1 Packages Projects Releases Wiki Activity

add "snapshot" configuration to allow to configure how snapshots are created 

expose snapshot options for module, resolve, buildDependencies and resolveBuildDependencies
move managedPaths and immutablePaths from config.cache to config.snapshot
add timestamp + hash snapshotting mode
use timestamp + hash mode by default in production and for buildDependencies
This commit is contained in:
Tobias Koppers 2020-08-26 00:36:16 +02:00
parent 86ca074290
commit d6e637f33c
22 changed files with 1117 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
declarations/WebpackOptions.d.ts vendored
View File

@ -679,6 +679,10 @@ export interface WebpackOptions {
* Options for the resolver when resolving loaders.
*/
resolveLoader?: ResolveLoader;
/**
* Options affecting how file system snapshots are created and validated.
*/
snapshot?: SnapshotOptions;
/**
* Stats options object or preset name.
*/
@ -700,14 +704,6 @@ export interface WebpackOptions {
* Options object for in-memory caching.
*/
export interface MemoryCacheOptions {
/**
* List of paths that are managed by a package manager and contain a version or hash in its path so all files are immutable.
*/
immutablePaths?: string[];
/**
* List of paths that are managed by a package manager and can be trusted to not be modified otherwise.
*/
managedPaths?: string[];
/**
* In memory caching.
*/
@ -1751,6 +1747,71 @@ export interface PerformanceOptions {
*/
maxEntrypointSize?: number;
}
/**
* Options affecting how file system snapshots are created and validated.
*/
export interface SnapshotOptions {
/**
* Options for snapshotting build dependencies to determine if the whole cache need to be invalidated.
*/
buildDependencies?: {
/**
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
};
/**
* List of paths that are managed by a package manager and contain a version or hash in its path so all files are immutable.
*/
immutablePaths?: string[];
/**
* List of paths that are managed by a package manager and can be trusted to not be modified otherwise.
*/
managedPaths?: string[];
/**
* Options for snapshotting dependencies of modules to determine if they need to be built again.
*/
module?: {
/**
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
};
/**
* Options for snapshotting dependencies of request resolving to determine if requests need to be re-resolved.
*/
resolve?: {
/**
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
};
/**
* Options for snapshotting the resolving of build dependencies to determine if the build dependencies need to be re-resolved.
*/
resolveBuildDependencies?: {
/**
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
};
}
/**
* Stats options object.
*/
@ -2303,6 +2364,10 @@ export interface WebpackOptionsNormalized {
* Options for the resolver when resolving loaders.
*/
resolveLoader: ResolveLoader;
/**
* Options affecting how file system snapshots are created and validated.
*/
snapshot: SnapshotOptions;
/**
* Stats options object or preset name.
*/

4
lib/ContextModule.js
View File

@ -81,6 +81,8 @@ const makeSerializable = require("./util/makeSerializable");
* @param {ResolveDependenciesCallback} callback
*/
const SNAPSHOT_OPTIONS = { timestamp: true };
const TYPES = new Set(["javascript"]);
class ContextModule extends Module {
@ -428,7 +430,7 @@ class ContextModule extends Module {
null,
[this.context],
null,
{},
SNAPSHOT_OPTIONS,
(err, snapshot) => {
if (err) return callback(err);
this.buildInfo.snapshot = snapshot;

536
lib/FileSystemInfo.js
View File

@ -46,6 +46,14 @@ const INVALID = Symbol("invalid");
* @property {string=} timestampHash
*/
/**
* @typedef {Object} TimestampAndHash
* @property {number} safeTime
* @property {number=} timestamp
* @property {string=} timestampHash
* @property {string} hash
*/
/**
* @typedef {Object} SnapshotOptimizationEntry
* @property {Snapshot} snapshot
@ -68,6 +76,10 @@ const INVALID = Symbol("invalid");
const DONE_ITERATOR_RESULT = new Set().keys().next();
// cspell:word tshs
// Tsh = Timestamp + Hash
// Tshs = Timestamp + Hash combinations
class Snapshot {
constructor() {
this._flags = 0;
@ -77,10 +89,14 @@ class Snapshot {
this.fileTimestamps = undefined;
/** @type {Map<string, string> | undefined} */
this.fileHashes = undefined;
/** @type {Map<string, TimestampAndHash | string> | undefined} */
this.fileTshs = undefined;
/** @type {Map<string, FileSystemInfoEntry> | undefined} */
this.contextTimestamps = undefined;
/** @type {Map<string, string> | undefined} */
this.contextHashes = undefined;
/** @type {Map<string, TimestampAndHash | string> | undefined} */
this.contextTshs = undefined;
/** @type {Map<string, boolean> | undefined} */
this.missingExistence = undefined;
/** @type {Map<string, string> | undefined} */
@ -134,75 +150,93 @@ class Snapshot {
this.fileHashes = value;
}
hasContextTimestamps() {
hasFileTshs() {
return (this._flags & 8) !== 0;
}
setContextTimestamps(value) {
setFileTshs(value) {
this._flags = this._flags | 8;
this.fileTshs = value;
}
hasContextTimestamps() {
return (this._flags & 0x10) !== 0;
}
setContextTimestamps(value) {
this._flags = this._flags | 0x10;
this.contextTimestamps = value;
}
hasContextHashes() {
return (this._flags & 0x10) !== 0;
}
setContextHashes(value) {
this._flags = this._flags | 0x10;
this.contextHashes = value;
}
hasMissingExistence() {
return (this._flags & 0x20) !== 0;
}
setMissingExistence(value) {
setContextHashes(value) {
this._flags = this._flags | 0x20;
this.contextHashes = value;
}
hasContextTshs() {
return (this._flags & 0x40) !== 0;
}
setContextTshs(value) {
this._flags = this._flags | 0x40;
this.contextTshs = value;
}
hasMissingExistence() {
return (this._flags & 0x80) !== 0;
}
setMissingExistence(value) {
this._flags = this._flags | 0x80;
this.missingExistence = value;
}
hasManagedItemInfo() {
return (this._flags & 0x40) !== 0;
return (this._flags & 0x100) !== 0;
}
setManagedItemInfo(value) {
this._flags = this._flags | 0x40;
this._flags = this._flags | 0x100;
this.managedItemInfo = value;
}
hasManagedFiles() {
return (this._flags & 0x80) !== 0;
return (this._flags & 0x200) !== 0;
}
setManagedFiles(value) {
this._flags = this._flags | 0x80;
this._flags = this._flags | 0x200;
this.managedFiles = value;
}
hasManagedContexts() {
return (this._flags & 0x100) !== 0;
return (this._flags & 0x400) !== 0;
}
setManagedContexts(value) {
this._flags = this._flags | 0x100;
this._flags = this._flags | 0x400;
this.managedContexts = value;
}
hasManagedMissing() {
return (this._flags & 0x200) !== 0;
return (this._flags & 0x800) !== 0;
}
setManagedMissing(value) {
this._flags = this._flags | 0x200;
this._flags = this._flags | 0x800;
this.managedMissing = value;
}
hasChildren() {
return (this._flags & 0x400) !== 0;
return (this._flags & 0x1000) !== 0;
}
setChildren(value) {
this._flags = this._flags | 0x400;
this._flags = this._flags | 0x1000;
this.children = value;
}
@ -218,8 +252,10 @@ class Snapshot {
if (this.hasStartTime()) write(this.startTime);
if (this.hasFileTimestamps()) write(this.fileTimestamps);
if (this.hasFileHashes()) write(this.fileHashes);
if (this.hasFileTshs()) write(this.fileTshs);
if (this.hasContextTimestamps()) write(this.contextTimestamps);
if (this.hasContextHashes()) write(this.contextHashes);
if (this.hasContextTshs()) write(this.contextTshs);
if (this.hasMissingExistence()) write(this.missingExistence);
if (this.hasManagedItemInfo()) write(this.managedItemInfo);
if (this.hasManagedFiles()) write(this.managedFiles);
@ -233,8 +269,10 @@ class Snapshot {
if (this.hasStartTime()) this.startTime = read();
if (this.hasFileTimestamps()) this.fileTimestamps = read();
if (this.hasFileHashes()) this.fileHashes = read();
if (this.hasFileTshs()) this.fileTshs = read();
if (this.hasContextTimestamps()) this.contextTimestamps = read();
if (this.hasContextHashes()) this.contextHashes = read();
if (this.hasContextTshs()) this.contextTshs = read();
if (this.hasMissingExistence()) this.missingExistence = read();
if (this.hasManagedItemInfo()) this.managedItemInfo = read();
if (this.hasManagedFiles()) this.managedFiles = read();
@ -244,58 +282,43 @@ class Snapshot {
}
/**
* @param {function(Snapshot): Map<string, any> | undefined} getMap first
* @param {function(Snapshot): Set<string> | undefined} getSet second
* @param {function(Snapshot): Iterable<string> | undefined} getIterable 3rd
* @param {function(Snapshot): (Map<string, any> | Set<string>)[]} getMaps first
* @returns {Iterable<string>} iterable
*/
_createIterable(getMap, getSet, getIterable) {
_createIterable(getMaps) {
let snapshot = this;
return {
[Symbol.iterator]() {
let state = 0;
/** @type {IterableIterator<string>} */
let it;
let maps = getMaps(snapshot);
const queue = [];
return {
next() {
for (;;) {
switch (state) {
case 0: {
const map = getMap(snapshot);
case 0:
if (maps.length > 0) {
const map = maps.pop();
if (map !== undefined) {
it = map.keys();
state = 1;
} else {
state = 2;
break;
}
} else {
state = 2;
break;
}
/* falls through */
case 1: {
const result = it.next();
if (!result.done) return result;
state = 2;
state = 0;
}
/* falls through */
case 2: {
const set = getSet(snapshot);
if (set !== undefined) {
it = set.keys();
state = 3;
} else {
state = 4;
break;
}
}
/* falls through */
case 3: {
const result = it.next();
if (!result.done) return result;
state = 4;
}
/* falls through */
case 4: {
const children = snapshot.children;
if (children !== undefined) {
for (const child of children) {
@ -304,14 +327,15 @@ class Snapshot {
}
if (queue.length > 0) {
snapshot = queue.pop();
maps = getMaps(snapshot);
state = 0;
break;
} else {
state = 5;
state = 3;
}
}
/* falls through */
case 5:
case 3:
return DONE_ITERATOR_RESULT;
}
}
@ -325,33 +349,31 @@ class Snapshot {
* @returns {Iterable<string>} iterable
*/
getFileIterable() {
return this._createIterable(
s => s.fileTimestamps,
s => s.managedFiles,
s => s.getFileIterable()
);
return this._createIterable(s => [
s.fileTimestamps,
s.fileHashes,
s.fileTshs,
s.managedFiles
]);
}
/**
* @returns {Iterable<string>} iterable
*/
getContextIterable() {
return this._createIterable(
s => s.contextTimestamps,
s => s.managedContexts,
s => s.getContextIterable()
);
return this._createIterable(s => [
s.contextTimestamps,
s.contextHashes,
s.contextTshs,
s.managedContexts
]);
}
/**
* @returns {Iterable<string>} iterable
*/
getMissingIterable() {
return this._createIterable(
s => s.missingExistence,
s => s.managedMissing,
s => s.getMissingIterable()
);
return this._createIterable(s => [s.missingExistence, s.managedMissing]);
}
}
@ -611,6 +633,23 @@ const mergeMaps = (a, b) => {
return map;
};
/**
* @template T
* @template K
* @param {Set<T, K>} a source map
* @param {Set<T, K>} b joining map
* @returns {Set<T, K>} joined map
*/
const mergeSets = (a, b) => {
if (!b || b.size === 0) return a;
if (!a || a.size === 0) return b;
const map = new Set(a);
for (const item of b) {
map.add(item);
}
return map;
};
/**
* Finding file or directory to manage
* @param {string} managedPath path that is managing by {@link FileSystemInfo}
@ -731,6 +770,11 @@ class FileSystemInfo {
s => s.fileHashes,
(s, v) => s.setFileHashes(v)
);
this._fileTshsOptimization = new SnapshotOptimization(
s => s.hasFileTshs(),
s => s.fileTshs,
(s, v) => s.setFileTshs(v)
);
this._contextTimestampsOptimization = new SnapshotOptimization(
s => s.hasContextTimestamps(),
s => s.contextTimestamps,
@ -741,6 +785,11 @@ class FileSystemInfo {
s => s.contextHashes,
(s, v) => s.setContextHashes(v)
);
this._contextTshsOptimization = new SnapshotOptimization(
s => s.hasContextTshs(),
s => s.contextTshs,
(s, v) => s.setContextTshs(v)
);
this._missingExistenceOptimization = new SnapshotOptimization(
s => s.hasMissingExistence(),
s => s.missingExistence,
@ -773,10 +822,14 @@ class FileSystemInfo {
this._fileTimestamps = new Map();
/** @type {Map<string, string>} */
this._fileHashes = new Map();
/** @type {Map<string, TimestampAndHash | string>} */
this._fileTshs = new Map();
/** @type {Map<string, FileSystemInfoEntry | "ignore" | null>} */
this._contextTimestamps = new Map();
/** @type {Map<string, string>} */
this._contextHashes = new Map();
/** @type {Map<string, TimestampAndHash | string>} */
this._contextTshs = new Map();
/** @type {Map<string, string>} */
this._managedItems = new Map();
/** @type {AsyncQueue<string, string, FileSystemInfoEntry | null>} */
@ -864,7 +917,7 @@ class FileSystemInfo {
);
this.logger.log(`${this._statTestedEntries} entries tested`);
this.logger.log(
`File info in cache: ${this._fileTimestamps.size} timestamps ${this._fileHashes.size} hashes`
`File info in cache: ${this._fileTimestamps.size} timestamps ${this._fileHashes.size} hashes ${this._fileTshs.size} timestamp hash combinations`
);
logWhenMessage(
`File timestamp snapshot optimization`,
@ -874,8 +927,12 @@ class FileSystemInfo {
`File hash snapshot optimization`,
this._fileHashesOptimization.getStatisticMessage()
);
logWhenMessage(
`File timestamp hash combination snapshot optimization`,
this._fileTshsOptimization.getStatisticMessage()
);
this.logger.log(
`Directory info in cache: ${this._contextTimestamps.size} timestamps ${this._contextHashes.size} hashes`
`Directory info in cache: ${this._contextTimestamps.size} timestamps ${this._contextHashes.size} hashes ${this._contextTshs.size} timestamp hash combinations`
);
logWhenMessage(
`Directory timestamp snapshot optimization`,
@ -885,6 +942,10 @@ class FileSystemInfo {
`Directory hash snapshot optimization`,
this._contextHashesOptimization.getStatisticMessage()
);
logWhenMessage(
`Directory timestamp hash combination snapshot optimization`,
this._contextTshsOptimization.getStatisticMessage()
);
logWhenMessage(
`Missing items snapshot optimization`,
this._missingExistenceOptimization.getStatisticMessage()
@ -1336,10 +1397,14 @@ class FileSystemInfo {
const fileTimestamps = new Map();
/** @type {Map<string, string>} */
const fileHashes = new Map();
/** @type {Map<string, TimestampAndHash | string>} */
const fileTshs = new Map();
/** @type {Map<string, FileSystemInfoEntry>} */
const contextTimestamps = new Map();
/** @type {Map<string, string>} */
const contextHashes = new Map();
/** @type {Map<string, TimestampAndHash | string>} */
const contextTshs = new Map();
/** @type {Map<string, boolean>} */
const missingExistence = new Map();
/** @type {Map<string, string>} */
@ -1358,10 +1423,14 @@ class FileSystemInfo {
/** @type {Set<string>} */
let unsharedFileHashes;
/** @type {Set<string>} */
let unsharedFileTshs;
/** @type {Set<string>} */
let unsharedContextTimestamps;
/** @type {Set<string>} */
let unsharedContextHashes;
/** @type {Set<string>} */
let unsharedContextTshs;
/** @type {Set<string>} */
let unsharedMissingExistence;
/** @type {Set<string>} */
let unsharedManagedItemInfo;
@ -1369,6 +1438,9 @@ class FileSystemInfo {
/** @type {Set<string>} */
const managedItems = new Set();
/** 1 = timestamp, 2 = hash, 3 = timestamp + hash */
const mode = options && options.hash ? (options.timestamp ? 3 : 2) : 1;
let jobs = 1;
const jobDone = () => {
if (--jobs === 0) {
@ -1388,6 +1460,13 @@ class FileSystemInfo {
unsharedFileHashes
);
}
if (fileTshs.size !== 0) {
snapshot.setFileTshs(fileTshs);
this._fileTshsOptimization.storeUnsharedSnapshot(
snapshot,
unsharedFileTshs
);
}
if (contextTimestamps.size !== 0) {
snapshot.setContextTimestamps(contextTimestamps);
this._contextTimestampsOptimization.storeUnsharedSnapshot(
@ -1402,6 +1481,13 @@ class FileSystemInfo {
unsharedContextHashes
);
}
if (contextTshs.size !== 0) {
snapshot.setContextTshs(contextTshs);
this._contextTshsOptimization.storeUnsharedSnapshot(
snapshot,
unsharedContextTshs
);
}
if (missingExistence.size !== 0) {
snapshot.setMissingExistence(missingExistence);
this._missingExistenceOptimization.storeUnsharedSnapshot(
@ -1496,7 +1582,36 @@ class FileSystemInfo {
};
if (files) {
const capturedFiles = captureNonManaged(files, managedFiles);
if (options && options.hash) {
switch (mode) {
case 3:
unsharedFileTshs = this._fileTshsOptimization.optimize(
capturedFiles,
undefined,
children
);
for (const path of capturedFiles) {
const cache = this._fileTshs.get(path);
if (cache !== undefined) {
fileTshs.set(path, cache);
} else {
jobs++;
this._getFileTimestampAndHash(path, (err, entry) => {
if (err) {
if (this.logger) {
this.logger.debug(
`Error snapshotting file timestamp hash combination of ${path}: ${err}`
);
}
jobError();
} else {
fileTshs.set(path, entry);
jobDone();
}
});
}
}
break;
case 2:
unsharedFileHashes = this._fileHashesOptimization.optimize(
capturedFiles,
undefined,
@ -1523,7 +1638,8 @@ class FileSystemInfo {
});
}
}
} else {
break;
case 1:
unsharedFileTimestamps = this._fileTimestampsOptimization.optimize(
capturedFiles,
startTime,
@ -1552,6 +1668,7 @@ class FileSystemInfo {
});
}
}
break;
}
}
if (directories) {
@ -1559,7 +1676,36 @@ class FileSystemInfo {
directories,
managedContexts
);
if (options && options.hash) {
switch (mode) {
case 3:
unsharedContextTshs = this._contextTshsOptimization.optimize(
capturedDirectories,
undefined,
children
);
for (const path of capturedDirectories) {
const cache = this._contextTshs.get(path);
if (cache !== undefined) {
contextTshs.set(path, cache);
} else {
jobs++;
this._getContextTimestampAndHash(path, (err, entry) => {
if (err) {
if (this.logger) {
this.logger.debug(
`Error snapshotting context timestamp hash combination of ${path}: ${err}`
);
}
jobError();
} else {
contextTshs.set(path, entry);
jobDone();
}
});
}
}
break;
case 2:
unsharedContextHashes = this._contextHashesOptimization.optimize(
capturedDirectories,
undefined,
@ -1586,7 +1732,8 @@ class FileSystemInfo {
});
}
}
} else {
break;
case 1:
unsharedContextTimestamps = this._contextTimestampsOptimization.optimize(
capturedDirectories,
startTime,
@ -1615,6 +1762,7 @@ class FileSystemInfo {
});
}
}
break;
}
}
if (missing) {
@ -1683,48 +1831,67 @@ class FileSystemInfo {
* @returns {Snapshot} merged snapshot
*/
mergeSnapshots(snapshot1, snapshot2) {
/** @type {Snapshot} */
const snapshot = {};
if (snapshot1.startTime && snapshot2.startTime)
snapshot.startTime = Math.min(snapshot1.startTime, snapshot2.startTime);
else if (snapshot2.startTime) snapshot.startTime = snapshot2.startTime;
else if (snapshot1.startTime) snapshot.startTime = snapshot1.startTime;
if (snapshot1.fileTimestamps || snapshot2.fileTimestamps) {
snapshot.fileTimestamps = mergeMaps(
snapshot1.fileTimestamps,
snapshot2.fileTimestamps
const snapshot = new Snapshot();
if (snapshot1.hasStartTime() && snapshot2.hasStartTime())
snapshot.setStartTime(Math.min(snapshot1.startTime, snapshot2.startTime));
else if (snapshot2.hasStartTime()) snapshot.startTime = snapshot2.startTime;
else if (snapshot1.hasStartTime()) snapshot.startTime = snapshot1.startTime;
if (snapshot1.hasFileTimestamps() || snapshot2.hasFileTimestamps()) {
snapshot.setFileTimestamps(
mergeMaps(snapshot1.fileTimestamps, snapshot2.fileTimestamps)
);
}
if (snapshot1.fileHashes || snapshot2.fileHashes) {
snapshot.fileHashes = mergeMaps(
snapshot1.fileHashes,
snapshot2.fileHashes
if (snapshot1.hasFileHashes() || snapshot2.hasFileHashes()) {
snapshot.setFileHashes(
mergeMaps(snapshot1.fileHashes, snapshot2.fileHashes)
);
}
if (snapshot1.contextTimestamps || snapshot2.contextTimestamps) {
snapshot.contextTimestamps = mergeMaps(
snapshot1.contextTimestamps,
snapshot2.contextTimestamps
if (snapshot1.hasFileTshs() || snapshot2.hasFileTshs()) {
snapshot.setFileTshs(mergeMaps(snapshot1.fileTshs, snapshot2.fileTshs));
}
if (snapshot1.hasContextTimestamps() || snapshot2.hasContextTimestamps()) {
snapshot.setContextTimestamps(
mergeMaps(snapshot1.contextTimestamps, snapshot2.contextTimestamps)
);
}
if (snapshot1.contextHashes || snapshot2.contextHashes) {
snapshot.contextHashes = mergeMaps(
snapshot1.contextHashes,
snapshot2.contextHashes
if (snapshot1.hasContextHashes() || snapshot2.hasContextHashes()) {
snapshot.setContextHashes(
mergeMaps(snapshot1.contextHashes, snapshot2.contextHashes)
);
}
if (snapshot1.missingExistence || snapshot2.missingExistence) {
snapshot.missingExistence = mergeMaps(
snapshot1.missingExistence,
snapshot2.missingExistence
if (snapshot1.hasContextTshs() || snapshot2.hasContextTshs()) {
snapshot.setContextTshs(
mergeMaps(snapshot1.contextTshs, snapshot2.contextTshs)
);
}
if (snapshot1.managedItemInfo || snapshot2.managedItemInfo) {
snapshot.managedItemInfo = mergeMaps(
snapshot1.managedItemInfo,
snapshot2.managedItemInfo
if (snapshot1.hasMissingExistence() || snapshot2.hasMissingExistence()) {
snapshot.setMissingExistence(
mergeMaps(snapshot1.missingExistence, snapshot2.missingExistence)
);
}
if (snapshot1.hasManagedItemInfo() || snapshot2.hasManagedItemInfo()) {
snapshot.setManagedItemInfo(
mergeMaps(snapshot1.managedItemInfo, snapshot2.managedItemInfo)
);
}
if (snapshot1.hasManagedFiles() || snapshot2.hasManagedFiles()) {
snapshot.setManagedFiles(
mergeSets(snapshot1.managedFiles, snapshot2.managedFiles)
);
}
if (snapshot1.hasManagedContexts() || snapshot2.hasManagedContexts()) {
snapshot.setManagedContexts(
mergeSets(snapshot1.managedContexts, snapshot2.managedContexts)
);
}
if (snapshot1.hasManagedMissing() || snapshot2.hasManagedMissing()) {
snapshot.setManagedMissing(
mergeSets(snapshot1.managedMissing, snapshot2.managedMissing)
);
}
if (snapshot1.hasChildren() || snapshot2.hasChildren()) {
snapshot.setChildren(mergeSets(snapshot1.children, snapshot2.children));
}
if (
this._snapshotCache.get(snapshot1) === true &&
this._snapshotCache.get(snapshot2) === true
@ -1826,14 +1993,15 @@ class FileSystemInfo {
* @param {string} path file path
* @param {FileSystemInfoEntry} current current entry
* @param {FileSystemInfoEntry} snap entry from snapshot
* @param {boolean} log log reason
* @returns {boolean} true, if ok
*/
const checkFile = (path, current, snap) => {
const checkFile = (path, current, snap, log = true) => {
if (current === snap) return true;
if (!current !== !snap) {
// If existence of item differs
// it's invalid
if (this._remainingLogs > 0) {
if (log && this._remainingLogs > 0) {
this._log(
path,
current ? "it didn't exist before" : "it does no longer exist"
@ -1846,7 +2014,7 @@ class FileSystemInfo {
if (typeof startTime === "number" && current.safeTime > startTime) {
// If a change happened after starting reading the item
// this may no longer be valid
if (this._remainingLogs > 0) {
if (log && this._remainingLogs > 0) {
this._log(
path,
`it may have changed (%d) after the start time of the snapshot (%d)`,
@ -1862,7 +2030,7 @@ class FileSystemInfo {
) {
// If we have a timestamp (it was a file or symlink) and it differs from current timestamp
// it's invalid
if (this._remainingLogs > 0) {
if (log && this._remainingLogs > 0) {
this._log(
path,
`timestamps differ (%d != %d)`,
@ -1878,7 +2046,7 @@ class FileSystemInfo {
) {
// If we have a timestampHash (it was a directory) and it differs from current timestampHash
// it's invalid
if (this._remainingLogs > 0) {
if (log && this._remainingLogs > 0) {
this._log(
path,
`timestamps hashes differ (%s != %s)`,
@ -1940,10 +2108,7 @@ class FileSystemInfo {
}
}
}
if (snapshot.hasFileHashes()) {
const { fileHashes } = snapshot;
this._statTestedEntries += fileHashes.size;
for (const [path, hash] of fileHashes) {
const processFileHashSnapshot = (path, hash) => {
const cache = this._fileHashes.get(path);
if (cache !== undefined) {
if (cache !== "ignore" && !checkHash(path, cache, hash)) {
@ -1961,6 +2126,37 @@ class FileSystemInfo {
}
});
}
};
if (snapshot.hasFileHashes()) {
const { fileHashes } = snapshot;
this._statTestedEntries += fileHashes.size;
for (const [path, hash] of fileHashes) {
processFileHashSnapshot(path, hash);
}
}
if (snapshot.hasFileTshs()) {
const { fileTshs } = snapshot;
this._statTestedEntries += fileTshs.size;
for (const [path, tsh] of fileTshs) {
if (typeof tsh === "string") {
processFileHashSnapshot(path, tsh);
} else {
const cache = this._fileTimestamps.get(path);
if (cache !== undefined) {
if (cache === "ignore" || !checkFile(path, cache, tsh, false)) {
processFileHashSnapshot(path, tsh.hash);
}
} else {
jobs++;
this.fileTimestampQueue.add(path, (err, entry) => {
if (err) return invalidWithError(path, err);
if (!checkFile(path, entry, tsh, false)) {
processFileHashSnapshot(path, tsh.hash);
}
jobDone();
});
}
}
}
}
if (snapshot.hasContextTimestamps()) {
@ -1986,10 +2182,7 @@ class FileSystemInfo {
}
}
}
if (snapshot.hasContextHashes()) {
const { contextHashes } = snapshot;
this._statTestedEntries += contextHashes.size;
for (const [path, hash] of contextHashes) {
const processContextHashSnapshot = (path, hash) => {
const cache = this._contextHashes.get(path);
if (cache !== undefined) {
if (cache !== "ignore" && !checkHash(path, cache, hash)) {
@ -2007,6 +2200,37 @@ class FileSystemInfo {
}
});
}
};
if (snapshot.hasContextHashes()) {
const { contextHashes } = snapshot;
this._statTestedEntries += contextHashes.size;
for (const [path, hash] of contextHashes) {
processContextHashSnapshot(path, hash);
}
}
if (snapshot.hasContextTshs()) {
const { contextTshs } = snapshot;
this._statTestedEntries += contextTshs.size;
for (const [path, tsh] of contextTshs) {
if (typeof tsh === "string") {
processContextHashSnapshot(path, tsh);
} else {
const cache = this._contextTimestamps.get(path);
if (cache !== undefined) {
if (cache === "ignore" || !checkFile(path, cache, tsh, false)) {
processContextHashSnapshot(path, tsh.hash);
}
} else {
jobs++;
this.contextTimestampQueue.add(path, (err, entry) => {
if (err) return invalidWithError(path, err);
if (!checkFile(path, entry, tsh, false)) {
processContextHashSnapshot(path, tsh.hash);
}
jobDone();
});
}
}
}
}
if (snapshot.hasMissingExistence()) {
@ -2130,6 +2354,49 @@ class FileSystemInfo {
});
}
_getFileTimestampAndHash(path, callback) {
const continueWithHash = hash => {
const cache = this._fileTimestamps.get(path);
if (cache !== undefined) {
if (cache !== "ignore") {
const result = {
...cache,
hash
};
this._fileTshs.set(path, result);
return callback(null, result);
} else {
this._fileTshs.set(path, hash);
return callback(null, hash);
}
} else {
this.fileTimestampQueue.add(path, (err, entry) => {
if (err) {
return callback(err);
}
const result = {
...entry,
hash
};
this._fileTshs.set(path, result);
return callback(null, result);
});
}
};
const cache = this._fileHashes.get(path);
if (cache !== undefined) {
continueWithHash(cache);
} else {
this.fileHashQueue.add(path, (err, entry) => {
if (err) {
return callback(err);
}
continueWithHash(entry);
});
}
}
_readContextTimestamp(path, callback) {
this.fs.readdir(path, (err, files) => {
if (err) {
@ -2296,6 +2563,49 @@ class FileSystemInfo {
});
}
_getContextTimestampAndHash(path, callback) {
const continueWithHash = hash => {
const cache = this._contextTimestamps.get(path);
if (cache !== undefined) {
if (cache !== "ignore") {
const result = {
...cache,
hash
};
this._contextTshs.set(path, result);
return callback(null, result);
} else {
this._contextTshs.set(path, hash);
return callback(null, hash);
}
} else {
this.contextTimestampQueue.add(path, (err, entry) => {
if (err) {
return callback(err);
}
const result = {
...entry,
hash
};
this._contextTshs.set(path, result);
return callback(null, result);
});
}
};
const cache = this._contextHashes.get(path);
if (cache !== undefined) {
continueWithHash(cache);
} else {
this.contextHashQueue.add(path, (err, entry) => {
if (err) {
return callback(err);
}
continueWithHash(entry);
});
}
}
_getManagedItemDirectoryInfo(path, callback) {
this.fs.readdir(path, (err, elements) => {
if (err) {

5
lib/NormalModule.js
View File

@ -804,7 +804,8 @@ class NormalModule extends Module {
};
const handleBuildDone = () => {
if (!this.buildInfo.cacheable) {
const snapshotOptions = compilation.options.snapshot.module;
if (!this.buildInfo.cacheable || !snapshotOptions) {
return callback();
}
// convert file/context/missingDependencies into filesystem snapshot
@ -813,7 +814,7 @@ class NormalModule extends Module {
this.buildInfo.fileDependencies,
this.buildInfo.contextDependencies,
this.buildInfo.missingDependencies,
null,
snapshotOptions,
(err, snapshot) => {
if (err) {
this.markModuleAsErrored(err);

12
lib/WebpackOptionsApply.js
View File

@ -490,13 +490,14 @@ class WebpackOptionsApply extends OptionsApply {
new WarnCaseSensitiveModulesPlugin().apply(compiler);
if (options.cache && typeof options.cache === "object") {
const cacheOptions = options.cache;
const AddManagedPathsPlugin = require("./cache/AddManagedPathsPlugin");
new AddManagedPathsPlugin(
cacheOptions.managedPaths,
cacheOptions.immutablePaths
options.snapshot.managedPaths,
options.snapshot.immutablePaths
).apply(compiler);
if (options.cache && typeof options.cache === "object") {
const cacheOptions = options.cache;
switch (cacheOptions.type) {
case "memory": {
const MemoryCachePlugin = require("./cache/MemoryCachePlugin");
@ -525,8 +526,7 @@ class WebpackOptionsApply extends OptionsApply {
logger: compiler.getInfrastructureLogger(
"webpack.cache.PackFileCacheStrategy"
),
managedPaths: cacheOptions.managedPaths,
immutablePaths: cacheOptions.immutablePaths
snapshot: options.snapshot
}),
cacheOptions.idleTimeout,
cacheOptions.idleTimeoutForInitialStore

28
lib/cache/PackFileCacheStrategy.js vendored
View File

@ -13,6 +13,7 @@ const makeSerializable = require("../util/makeSerializable");
const memorize = require("../util/memorize");
const { createFileSerializer } = require("../util/serialization");
/** @typedef {import("../../declarations/WebpackOptions").SnapshotOptions} SnapshotOptions */
/** @typedef {import("../Cache").Etag} Etag */
/** @typedef {import("../Compiler")} Compiler */
/** @typedef {import("../FileSystemInfo").Snapshot} Snapshot */
@ -721,8 +722,7 @@ class PackFileCacheStrategy {
* @param {string} options.cacheLocation the location of the cache data
* @param {string} options.version version identifier
* @param {Logger} options.logger a logger
* @param {Iterable<string>} options.managedPaths paths managed only by package manager
* @param {Iterable<string>} options.immutablePaths immutable paths
* @param {SnapshotOptions} options.snapshot options regarding snapshotting
*/
constructor({
compiler,
@ -731,13 +731,12 @@ class PackFileCacheStrategy {
cacheLocation,
version,
logger,
managedPaths,
immutablePaths
snapshot
}) {
this.fileSerializer = createFileSerializer(fs);
this.fileSystemInfo = new FileSystemInfo(fs, {
managedPaths,
immutablePaths,
managedPaths: snapshot.managedPaths,
immutablePaths: snapshot.immutablePaths,
logger: logger.getChildLogger("webpack.FileSystemInfo")
});
this.compiler = compiler;
@ -745,6 +744,7 @@ class PackFileCacheStrategy {
this.cacheLocation = cacheLocation;
this.version = version;
this.logger = logger;
this.snapshot = snapshot;
/** @type {Set<string>} */
this.buildDependencies = new Set();
/** @type {LazySet<string>} */
@ -996,17 +996,19 @@ class PackFileCacheStrategy {
} else {
this.resolveResults = resolveResults;
}
if (reportProgress)
if (reportProgress) {
reportProgress(
0.6,
"snapshot build dependencies (timestamps)"
"snapshot build dependencies",
"resolving"
);
}
this.fileSystemInfo.createSnapshot(
undefined,
resolveDependencies.files,
resolveDependencies.directories,
resolveDependencies.missing,
{},
this.snapshot.resolveBuildDependencies,
(err, snapshot) => {
if (err) {
this.logger.timeEnd("snapshot build dependencies");
@ -1026,17 +1028,19 @@ class PackFileCacheStrategy {
} else {
this.resolveBuildDependenciesSnapshot = snapshot;
}
if (reportProgress)
if (reportProgress) {
reportProgress(
0.7,
"snapshot build dependencies (hashes)"
"snapshot build dependencies",
"modules"
);
}
this.fileSystemInfo.createSnapshot(
undefined,
files,
directories,
missing,
{ hash: true },
this.snapshot.buildDependencies,
(err, snapshot) => {
this.logger.timeEnd("snapshot build dependencies");
if (err) return reject(err);

4
lib/cache/ResolverCachePlugin.js vendored
View File

@ -79,11 +79,13 @@ class ResolverCachePlugin {
const cache = compiler.getCache("ResolverCachePlugin");
/** @type {FileSystemInfo} */
let fileSystemInfo;
let snapshotOptions;
let realResolves = 0;
let cachedResolves = 0;
let cacheInvalidResolves = 0;
let concurrentResolves = 0;
compiler.hooks.thisCompilation.tap("ResolverCachePlugin", compilation => {
snapshotOptions = compilation.options.snapshot.resolve;
fileSystemInfo = compilation.fileSystemInfo;
compilation.hooks.finishModules.tap("ResolverCachePlugin", () => {
if (realResolves + cachedResolves > 0) {
@ -151,7 +153,7 @@ class ResolverCachePlugin {
fileDependencies,
contextDependencies,
missingDependencies,
{ timestamp: true },
snapshotOptions,
(err, snapshot) => {
if (err) return callback(err);
if (!snapshot) {

27
lib/config/defaults.js
View File

@ -23,6 +23,7 @@ const { cleverMerge } = require("../util/cleverMerge");
/** @typedef {import("../../declarations/WebpackOptions").Performance} Performance */
/** @typedef {import("../../declarations/WebpackOptions").ResolveOptions} ResolveOptions */
/** @typedef {import("../../declarations/WebpackOptions").RuleSetRules} RuleSetRules */
/** @typedef {import("../../declarations/WebpackOptions").SnapshotOptions} SnapshotOptions */
/** @typedef {import("../../declarations/WebpackOptions").Target} Target */
/** @typedef {import("../../declarations/WebpackOptions").WebpackOptionsNormalized} WebpackOptions */
@ -148,6 +149,8 @@ const applyWebpackOptionsDefaults = options => {
});
const cache = !!options.cache;
applySnapshotDefaults(options.snapshot, { production });
applyExperimentsDefaults(options.experiments);
applyModuleDefaults(options.module, {
@ -267,7 +270,16 @@ const applyCacheDefaults = (cache, { name, mode }) => {
]);
break;
}
A(cache, "managedPaths", () => {
};
/**
* @param {SnapshotOptions} snapshot options
* @param {Object} options options
* @param {boolean} options.production is production
* @returns {void}
*/
const applySnapshotDefaults = (snapshot, { production }) => {
A(snapshot, "managedPaths", () => {
if (process.versions.pnp === "3") {
const match = /^(.+?)[\\/]cache[\\/]watchpack-npm-[^\\/]+\.zip[\\/]node_modules[\\/]/.exec(
require.resolve("watchpack")
@ -286,7 +298,7 @@ const applyCacheDefaults = (cache, { name, mode }) => {
}
return [];
});
A(cache, "immutablePaths", () => {
A(snapshot, "immutablePaths", () => {
if (process.versions.pnp === "1") {
const match = /^(.+?[\\/]v4)[\\/]npm-watchpack-[^\\/]+-[\da-f]{40}[\\/]node_modules[\\/]/.exec(
require.resolve("watchpack")
@ -304,6 +316,17 @@ const applyCacheDefaults = (cache, { name, mode }) => {
}
return [];
});
F(snapshot, "resolveBuildDependencies", () => ({
timestamp: true,
hash: true
}));
F(snapshot, "buildDependencies", () => ({ timestamp: true, hash: true }));
F(snapshot, "module", () =>
production ? { timestamp: true, hash: true } : { timestamp: true }
);
F(snapshot, "resolve", () =>
production ? { timestamp: true, hash: true } : { timestamp: true }
);
};
/**

36
lib/config/normalization.js
View File

@ -115,10 +115,6 @@ const getNormalizedWebpackOptions = config => {
hashAlgorithm: cache.hashAlgorithm,
idleTimeout: cache.idleTimeout,
idleTimeoutForInitialStore: cache.idleTimeoutForInitialStore,
immutablePaths: optionalNestedArray(cache.immutablePaths, p => [
...p
]),
managedPaths: optionalNestedArray(cache.managedPaths, p => [...p]),
name: cache.name,
store: cache.store,
version: cache.version
@ -126,11 +122,7 @@ const getNormalizedWebpackOptions = config => {
case undefined:
case "memory":
return {
type: "memory",
immutablePaths: optionalNestedArray(cache.immutablePaths, p => [
...p
]),
managedPaths: optionalNestedArray(cache.managedPaths, p => [...p])
type: "memory"
};
default:
// @ts-expect-error Property 'type' does not exist on type 'never'. ts(2339)
@ -307,6 +299,32 @@ const getNormalizedWebpackOptions = config => {
resolveLoader: nestedConfig(config.resolveLoader, resolve => ({
...resolve
})),
snapshot: nestedConfig(config.snapshot, snapshot => ({
resolveBuildDependencies: optionalNestedConfig(
snapshot.resolveBuildDependencies,
resolveBuildDependencies => ({
timestamp: resolveBuildDependencies.timestamp,
hash: resolveBuildDependencies.hash
})
),
buildDependencies: optionalNestedConfig(
snapshot.buildDependencies,
buildDependencies => ({
timestamp: buildDependencies.timestamp,
hash: buildDependencies.hash
})
),
resolve: optionalNestedConfig(snapshot.resolve, resolve => ({
timestamp: resolve.timestamp,
hash: resolve.hash
})),
module: optionalNestedConfig(snapshot.module, module => ({
timestamp: module.timestamp,
hash: module.hash
})),
immutablePaths: optionalNestedArray(snapshot.immutablePaths, p => [...p]),
managedPaths: optionalNestedArray(snapshot.managedPaths, p => [...p])
})),
stats: nestedConfig(config.stats, stats => {
if (stats === false) {
return {

2
lib/validateSchema.js
View File

@ -23,6 +23,8 @@ const DID_YOU_MEAN = {
pathinfo: "output.pathinfo",
pathInfo: "output.pathinfo",
splitChunks: "optimization.splitChunks",
immutablePaths: "snapshot.immutablePaths",
managedPaths: "snapshot.managedPaths",
hashedModuleIds:
'optimization.moduleIds: "hashed" (BREAKING CHANGE since webpack 5)',
namedChunks:

114
schemas/WebpackOptions.json
View File

@ -854,26 +854,6 @@
"type": "object",
"additionalProperties": false,
"properties": {
"immutablePaths": {
"description": "List of paths that are managed by a package manager and contain a version or hash in its path so all files are immutable.",
"type": "array",
"items": {
"description": "A path to a immutable directory (usually a package manager cache directory).",
"type": "string",
"absolutePath": true,
"minLength": 1
}
},
"managedPaths": {
"description": "List of paths that are managed by a package manager and can be trusted to not be modified otherwise.",
"type": "array",
"items": {
"description": "A path to a managed directory (usually a node_modules directory).",
"type": "string",
"absolutePath": true,
"minLength": 1
}
},
"type": {
"description": "In memory caching.",
"enum": ["memory"]
@ -2802,6 +2782,93 @@
"description": "This option enables loading async chunks via a custom script type, such as script type=\"module\".",
"enum": [false, "text/javascript", "module"]
},
"SnapshotOptions": {
"description": "Options affecting how file system snapshots are created and validated.",
"type": "object",
"additionalProperties": false,
"properties": {
"buildDependencies": {
"description": "Options for snapshotting build dependencies to determine if the whole cache need to be invalidated.",
"type": "object",
"additionalProperties": false,
"properties": {
"hash": {
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"type": "boolean"
},
"timestamp": {
"description": "Use timestamps of the files/directories to determine invalidation.",
"type": "boolean"
}
}
},
"immutablePaths": {
"description": "List of paths that are managed by a package manager and contain a version or hash in its path so all files are immutable.",
"type": "array",
"items": {
"description": "A path to a immutable directory (usually a package manager cache directory).",
"type": "string",
"absolutePath": true,
"minLength": 1
}
},
"managedPaths": {
"description": "List of paths that are managed by a package manager and can be trusted to not be modified otherwise.",
"type": "array",
"items": {
"description": "A path to a managed directory (usually a node_modules directory).",
"type": "string",
"absolutePath": true,
"minLength": 1
}
},
"module": {
"description": "Options for snapshotting dependencies of modules to determine if they need to be built again.",
"type": "object",
"additionalProperties": false,
"properties": {
"hash": {
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"type": "boolean"
},
"timestamp": {
"description": "Use timestamps of the files/directories to determine invalidation.",
"type": "boolean"
}
}
},
"resolve": {
"description": "Options for snapshotting dependencies of request resolving to determine if requests need to be re-resolved.",
"type": "object",
"additionalProperties": false,
"properties": {
"hash": {
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"type": "boolean"
},
"timestamp": {
"description": "Use timestamps of the files/directories to determine invalidation.",
"type": "boolean"
}
}
},
"resolveBuildDependencies": {
"description": "Options for snapshotting the resolving of build dependencies to determine if the build dependencies need to be re-resolved.",
"type": "object",
"additionalProperties": false,
"properties": {
"hash": {
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"type": "boolean"
},
"timestamp": {
"description": "Use timestamps of the files/directories to determine invalidation.",
"type": "boolean"
}
}
}
}
},
"SourceMapFilename": {
"description": "The filename of the SourceMaps for the JavaScript files. They are inside the `output.path` directory.",
"type": "string",
@ -3308,6 +3375,9 @@
"resolveLoader": {
"$ref": "#/definitions/ResolveLoader"
},
"snapshot": {
"$ref": "#/definitions/SnapshotOptions"
},
"stats": {
"$ref": "#/definitions/StatsValue"
},
@ -3323,6 +3393,7 @@
},
"required": [
"cache",
"snapshot",
"entry",
"experiments",
"externals",
@ -3445,6 +3516,9 @@
"resolveLoader": {
"$ref": "#/definitions/ResolveLoader"
},
"snapshot": {
"$ref": "#/definitions/SnapshotOptions"
},
"stats": {
"$ref": "#/definitions/StatsValue"
},

4
test/ConfigCacheTestCases.test.js
View File

@ -4,7 +4,9 @@ const { describeCases } = require("./ConfigTestCases.template");
describeCases({
name: "ConfigCacheTestCases",
cache: {
type: "filesystem",
type: "filesystem"
},
snapshot: {
managedPaths: [path.resolve(__dirname, "../node_modules")]
}
});

5
test/ConfigTestCases.template.js
View File

@ -72,6 +72,11 @@ const describeCases = config => {
...config.cache
};
}
if (config.snapshot) {
options.snapshot = {
...config.snapshot
};
}
});
testConfig = {
findBundle: function (i, options) {

40
test/Defaults.unittest.js
View File

@ -444,6 +444,26 @@ describe("Defaults", () => {
"index",
],
},
"snapshot": Object {
"buildDependencies": Object {
"hash": true,
"timestamp": true,
},
"immutablePaths": Array [],
"managedPaths": Array [
"<cwd>/node_modules",
],
"module": Object {
"timestamp": true,
},
"resolve": Object {
"timestamp": true,
},
"resolveBuildDependencies": Object {
"hash": true,
"timestamp": true,
},
},
"stats": Object {},
"target": "web",
"watch": false,
@ -527,6 +547,10 @@ describe("Defaults", () => {
+ "maxAssetSize": 250000,
+ "maxEntrypointSize": 250000,
+ },
@@ ... @@
+ "hash": true,
@@ ... @@
+ "hash": true,
`)
);
test("production", { mode: "production" }, e =>
@ -585,6 +609,10 @@ describe("Defaults", () => {
+ "maxAssetSize": 250000,
+ "maxEntrypointSize": 250000,
+ },
@@ ... @@
+ "hash": true,
@@ ... @@
+ "hash": true,
`)
);
test("development", { mode: "development" }, e =>
@ -595,10 +623,6 @@ describe("Defaults", () => {
@@ ... @@
- "cache": false,
+ "cache": Object {
+ "immutablePaths": Array [],
+ "managedPaths": Array [
+ "<cwd>/node_modules",
+ ],
+ "type": "memory",
+ },
@@ ... @@
@ -1271,10 +1295,6 @@ describe("Defaults", () => {
@@ ... @@
- "cache": false,
+ "cache": Object {
+ "immutablePaths": Array [],
+ "managedPaths": Array [
+ "<cwd>/node_modules",
+ ],
+ "type": "memory",
+ },
@@ ... @@
@ -1306,10 +1326,6 @@ describe("Defaults", () => {
+ "hashAlgorithm": "md4",
+ "idleTimeout": 60000,
+ "idleTimeoutForInitialStore": 0,
+ "immutablePaths": Array [],
+ "managedPaths": Array [
+ "<cwd>/node_modules",
+ ],
+ "name": "default-none",
+ "store": "pack",
+ "type": "filesystem",

4
test/TestCasesCachePack.test.js
View File

@ -5,7 +5,9 @@ describe("TestCases", () => {
describeCases({
name: "cache pack",
cache: {
type: "filesystem",
type: "filesystem"
},
snapshot: {
managedPaths: [path.resolve(__dirname, "../node_modules")]
},
optimization: {

16
test/Validation.test.js
View File

@ -24,7 +24,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialized using a configuration object that does not match the API schema.
- configuration should be an object:
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, snapshot?, stats?, target?, watch?, watchOptions? }
-> Options object as provided by the user."
`)
);
@ -33,7 +33,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialized using a configuration object that does not match the API schema.
- configuration should be an object:
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, snapshot?, stats?, target?, watch?, watchOptions? }
-> Options object as provided by the user."
`)
);
@ -196,7 +196,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialized using a configuration object that does not match the API schema.
- configuration has an unknown property 'postcss'. These properties are valid:
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, snapshot?, stats?, target?, watch?, watchOptions? }
-> Options object as provided by the user.
For typos: please correct them.
For loader options: webpack >= v2.0.0 no longer allows custom properties in configuration.
@ -426,7 +426,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialized using a configuration object that does not match the API schema.
- configuration has an unknown property 'debug'. These properties are valid:
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, snapshot?, stats?, target?, watch?, watchOptions? }
-> Options object as provided by the user.
The 'debug' property was removed in webpack 2.0.0.
Loaders should be updated to allow passing this option via loader options in module.rules.
@ -482,7 +482,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialized using a configuration object that does not match the API schema.
- configuration[1] should be an object:
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, snapshot?, stats?, target?, watch?, watchOptions? }
-> Options object as provided by the user."
`)
);
@ -621,7 +621,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialized using a configuration object that does not match the API schema.
- configuration has an unknown property 'rules'. These properties are valid:
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, snapshot?, stats?, target?, watch?, watchOptions? }
-> Options object as provided by the user.
Did you mean module.rules?"
`)
@ -635,7 +635,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialized using a configuration object that does not match the API schema.
- configuration has an unknown property 'splitChunks'. These properties are valid:
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, snapshot?, stats?, target?, watch?, watchOptions? }
-> Options object as provided by the user.
Did you mean optimization.splitChunks?"
`)
@ -649,7 +649,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialized using a configuration object that does not match the API schema.
- configuration has an unknown property 'noParse'. These properties are valid:
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, stats?, target?, watch?, watchOptions? }
object { amd?, bail?, cache?, context?, dependencies?, devServer?, devtool?, entry?, experiments?, externals?, externalsType?, infrastructureLogging?, loader?, mode?, module?, name?, node?, optimization?, output?, parallelism?, performance?, plugins?, profile?, recordsInputPath?, recordsOutputPath?, recordsPath?, resolve?, resolveLoader?, snapshot?, stats?, target?, watch?, watchOptions? }
-> Options object as provided by the user.
Did you mean module.noParse?"
`)

156
test/__snapshots__/Cli.test.js.snap
View File

@ -3552,6 +3552,162 @@ Object {
"multiple": false,
"simpleType": "boolean",
},
"snapshot-build-dependencies-hash": Object {
"configs": Array [
Object {
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"multiple": false,
"path": "snapshot.buildDependencies.hash",
"type": "boolean",
},
],
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-build-dependencies-timestamp": Object {
"configs": Array [
Object {
"description": "Use timestamps of the files/directories to determine invalidation.",
"multiple": false,
"path": "snapshot.buildDependencies.timestamp",
"type": "boolean",
},
],
"description": "Use timestamps of the files/directories to determine invalidation.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-immutable-paths": Object {
"configs": Array [
Object {
"description": "A path to a immutable directory (usually a package manager cache directory).",
"multiple": true,
"path": "snapshot.immutablePaths[]",
"type": "path",
},
],
"description": "A path to a immutable directory (usually a package manager cache directory).",
"multiple": true,
"simpleType": "string",
},
"snapshot-immutable-paths-reset": Object {
"configs": Array [
Object {
"description": "Clear all items provided in configuration. List of paths that are managed by a package manager and contain a version or hash in its path so all files are immutable.",
"multiple": false,
"path": "snapshot.immutablePaths",
"type": "reset",
},
],
"description": "Clear all items provided in configuration. List of paths that are managed by a package manager and contain a version or hash in its path so all files are immutable.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-managed-paths": Object {
"configs": Array [
Object {
"description": "A path to a managed directory (usually a node_modules directory).",
"multiple": true,
"path": "snapshot.managedPaths[]",
"type": "path",
},
],
"description": "A path to a managed directory (usually a node_modules directory).",
"multiple": true,
"simpleType": "string",
},
"snapshot-managed-paths-reset": Object {
"configs": Array [
Object {
"description": "Clear all items provided in configuration. List of paths that are managed by a package manager and can be trusted to not be modified otherwise.",
"multiple": false,
"path": "snapshot.managedPaths",
"type": "reset",
},
],
"description": "Clear all items provided in configuration. List of paths that are managed by a package manager and can be trusted to not be modified otherwise.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-module-hash": Object {
"configs": Array [
Object {
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"multiple": false,
"path": "snapshot.module.hash",
"type": "boolean",
},
],
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-module-timestamp": Object {
"configs": Array [
Object {
"description": "Use timestamps of the files/directories to determine invalidation.",
"multiple": false,
"path": "snapshot.module.timestamp",
"type": "boolean",
},
],
"description": "Use timestamps of the files/directories to determine invalidation.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-resolve-build-dependencies-hash": Object {
"configs": Array [
Object {
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"multiple": false,
"path": "snapshot.resolveBuildDependencies.hash",
"type": "boolean",
},
],
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-resolve-build-dependencies-timestamp": Object {
"configs": Array [
Object {
"description": "Use timestamps of the files/directories to determine invalidation.",
"multiple": false,
"path": "snapshot.resolveBuildDependencies.timestamp",
"type": "boolean",
},
],
"description": "Use timestamps of the files/directories to determine invalidation.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-resolve-hash": Object {
"configs": Array [
Object {
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"multiple": false,
"path": "snapshot.resolve.hash",
"type": "boolean",
},
],
"description": "Use hashes of the content of the files/directories to determine invalidation.",
"multiple": false,
"simpleType": "boolean",
},
"snapshot-resolve-timestamp": Object {
"configs": Array [
Object {
"description": "Use timestamps of the files/directories to determine invalidation.",
"multiple": false,
"path": "snapshot.resolve.timestamp",
"type": "boolean",
},
],
"description": "Use timestamps of the files/directories to determine invalidation.",
"multiple": false,
"simpleType": "boolean",
},
"stats": Object {
"configs": Array [
Object {

18
test/__snapshots__/StatsTestCases.test.js.snap
View File

@ -1164,9 +1164,9 @@ Entrypoint <CLR=BOLD>main</CLR> = <CLR=32,BOLD>main.js</CLR>
<CLR=BOLD>0% root snapshot uncached (0 / 0)</CLR>
<CLR=BOLD>0% children snapshot uncached (0 / 0)</CLR>
<CLR=BOLD>0 entries tested</CLR>
<CLR=BOLD>File info in cache: 1 timestamps 0 hashes</CLR>
<CLR=BOLD>File timestamp snapshot optimization: 0% (0/1) entries shared via 0 shared snapshots (0 times referenced)</CLR>
<CLR=BOLD>Directory info in cache: 0 timestamps 0 hashes</CLR>
<CLR=BOLD>File info in cache: 1 timestamps 1 hashes 1 timestamp hash combinations</CLR>
<CLR=BOLD>File timestamp hash combination snapshot optimization: 0% (0/1) entries shared via 0 shared snapshots (0 times referenced)</CLR>
<CLR=BOLD>Directory info in cache: 0 timestamps 0 hashes 0 timestamp hash combinations</CLR>
<CLR=BOLD>Managed items info in cache: 0 items</CLR>
"
`;
@ -1973,9 +1973,9 @@ LOG from webpack.FileSystemInfo
0% root snapshot uncached (0 / 0)
0% children snapshot uncached (0 / 0)
0 entries tested
File info in cache: 6 timestamps 0 hashes
File timestamp snapshot optimization: 0% (0/6) entries shared via 0 shared snapshots (0 times referenced)
Directory info in cache: 0 timestamps 0 hashes
File info in cache: 6 timestamps 6 hashes 6 timestamp hash combinations
File timestamp hash combination snapshot optimization: 0% (0/6) entries shared via 0 shared snapshots (0 times referenced)
Directory info in cache: 0 timestamps 0 hashes 0 timestamp hash combinations
Managed items info in cache: 0 items
"
`;
@ -2310,9 +2310,9 @@ LOG from webpack.FileSystemInfo
0% root snapshot uncached (0 / 0)
0% children snapshot uncached (0 / 0)
0 entries tested
File info in cache: 6 timestamps 0 hashes
File timestamp snapshot optimization: 0% (0/6) entries shared via 0 shared snapshots (0 times referenced)
Directory info in cache: 0 timestamps 0 hashes
File info in cache: 6 timestamps 6 hashes 6 timestamp hash combinations
File timestamp hash combination snapshot optimization: 0% (0/6) entries shared via 0 shared snapshots (0 times referenced)
Directory info in cache: 0 timestamps 0 hashes 0 timestamp hash combinations
Managed items info in cache: 0 items
"
`;

2
test/fixtures/buildDependencies/run.js vendored
View File

@ -31,7 +31,9 @@ webpack(
__filename,
path.resolve(__dirname, "../../../node_modules/.yarn-integrity")
]
}
},
snapshot: {
managedPaths: [path.resolve(__dirname, "../../../node_modules")]
}
},

4
test/watchCases/cache/managedPath/webpack.config.js vendored
View File

@ -4,7 +4,9 @@ const path = require("path");
module.exports = {
mode: "development",
cache: {
type: "memory",
type: "memory"
},
snapshot: {
managedPaths: [
path.resolve(
__dirname,

4
test/watchCases/snapshot/unable-to-snapshot/webpack.config.js
View File

@ -2,7 +2,9 @@ const path = require("path");
/** @type {function(any, any): import("../../../../").Configuration} */
module.exports = (env, { srcPath }) => ({
cache: {
type: "memory",
type: "memory"
},
snapshot: {
managedPaths: [path.resolve(srcPath, "node_modules")]
}
});

107
types.d.ts vendored
View File

@ -1741,6 +1741,11 @@ declare interface Configuration {
*/
resolveLoader?: ResolveOptionsWebpackOptions;
/**
* Options affecting how file system snapshots are created and validated.
*/
snapshot?: SnapshotOptions;
/**
* Stats options object or preset name.
*/
@ -3090,11 +3095,11 @@ declare abstract class FileSystemInfo {
missing: Iterable<string>,
options: {
/**
* should use hash to snapshot
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* should use timestamp to snapshot
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
},
@ -4236,16 +4241,6 @@ declare interface MapOptions {
* Options object for in-memory caching.
*/
declare interface MemoryCacheOptions {
/**
* List of paths that are managed by a package manager and contain a version or hash in its path so all files are immutable.
*/
immutablePaths?: string[];
/**
* List of paths that are managed by a package manager and can be trusted to not be modified otherwise.
*/
managedPaths?: string[];
/**
* In memory caching.
*/
@ -7859,8 +7854,10 @@ declare abstract class Snapshot {
startTime: number;
fileTimestamps: Map<string, FileSystemInfoEntry>;
fileHashes: Map<string, string>;
fileTshs: Map<string, string | TimestampAndHash>;
contextTimestamps: Map<string, FileSystemInfoEntry>;
contextHashes: Map<string, string>;
contextTshs: Map<string, string | TimestampAndHash>;
missingExistence: Map<string, boolean>;
managedItemInfo: Map<string, string>;
managedFiles: Set<string>;
@ -7874,10 +7871,14 @@ declare abstract class Snapshot {
setFileTimestamps(value?: any): void;
hasFileHashes(): boolean;
setFileHashes(value?: any): void;
hasFileTshs(): boolean;
setFileTshs(value?: any): void;
hasContextTimestamps(): boolean;
setContextTimestamps(value?: any): void;
hasContextHashes(): boolean;
setContextHashes(value?: any): void;
hasContextTshs(): boolean;
setContextTshs(value?: any): void;
hasMissingExistence(): boolean;
setMissingExistence(value?: any): void;
hasManagedItemInfo(): boolean;
@ -7897,6 +7898,77 @@ declare abstract class Snapshot {
getContextIterable(): Iterable<string>;
getMissingIterable(): Iterable<string>;
}
/**
* Options affecting how file system snapshots are created and validated.
*/
declare interface SnapshotOptions {
/**
* Options for snapshotting build dependencies to determine if the whole cache need to be invalidated.
*/
buildDependencies?: {
/**
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
};
/**
* List of paths that are managed by a package manager and contain a version or hash in its path so all files are immutable.
*/
immutablePaths?: string[];
/**
* List of paths that are managed by a package manager and can be trusted to not be modified otherwise.
*/
managedPaths?: string[];
/**
* Options for snapshotting dependencies of modules to determine if they need to be built again.
*/
module?: {
/**
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
};
/**
* Options for snapshotting dependencies of request resolving to determine if requests need to be re-resolved.
*/
resolve?: {
/**
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
};
/**
* Options for snapshotting the resolving of build dependencies to determine if the build dependencies need to be re-resolved.
*/
resolveBuildDependencies?: {
/**
* Use hashes of the content of the files/directories to determine invalidation.
*/
hash?: boolean;
/**
* Use timestamps of the files/directories to determine invalidation.
*/
timestamp?: boolean;
};
}
declare abstract class SortableSet<T> extends Set<T> {
/**
* Sort with a comparer function
@ -8507,6 +8579,12 @@ declare class Template {
static NUMBER_OF_IDENTIFIER_START_CHARS: number;
static NUMBER_OF_IDENTIFIER_CONTINUATION_CHARS: number;
}
declare interface TimestampAndHash {
safeTime: number;
timestamp?: number;
timestampHash?: string;
hash: string;
}
declare const UNDEFINED_MARKER: unique symbol;
declare interface UpdateHashContextDependency {
chunkGraph: ChunkGraph;
@ -8970,6 +9048,11 @@ declare interface WebpackOptionsNormalized {
*/
resolveLoader: ResolveOptionsWebpackOptions;
/**
* Options affecting how file system snapshots are created and validated.
*/
snapshot: SnapshotOptions;
/**
* Stats options object or preset name.
*/