root
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Actions 1 Packages Projects Releases Wiki Activity

improve hashing

This commit is contained in:
Tobias Koppers 2022-02-22 12:23:24 +01:00
parent 5a446d81e3
commit 224964601d
6 changed files with 124 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
declarations/WebpackOptions.d.ts vendored
View File

@ -681,7 +681,7 @@ export type AssetGeneratorDataUrl =
| AssetGeneratorDataUrlOptions
| AssetGeneratorDataUrlFunction;
/**
* Function that executes for module and should return an DataUrl string.
* Function that executes for module and should return an DataUrl string. It can have a string as 'ident' property which contributes to the module hash.
*/
export type AssetGeneratorDataUrlFunction = (
source: string | Buffer,

60
lib/Compilation.js
View File

@ -43,6 +43,7 @@ const Module = require("./Module");
const ModuleDependencyError = require("./ModuleDependencyError");
const ModuleDependencyWarning = require("./ModuleDependencyWarning");
const ModuleGraph = require("./ModuleGraph");
const ModuleHashingError = require("./ModuleHashingError");
const ModuleNotFoundError = require("./ModuleNotFoundError");
const ModuleProfile = require("./ModuleProfile");
const ModuleRestoreError = require("./ModuleRestoreError");
@ -3883,6 +3884,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
let statModulesFromCache = 0;
const { chunkGraph, runtimeTemplate, moduleMemCaches2 } = this;
const { hashFunction, hashDigest, hashDigestLength } = this.outputOptions;
const errors = [];
for (const module of this.modules) {
const memCache = moduleMemCaches2 && moduleMemCaches2.get(module);
for (const runtime of chunkGraph.getModuleRuntimes(module)) {
@ -3907,13 +3909,20 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
hashFunction,
runtimeTemplate,
hashDigest,
hashDigestLength
hashDigestLength,
errors
);
if (memCache) {
memCache.set(`moduleHash-${getRuntimeKey(runtime)}`, digest);
}
}
}
if (errors.length > 0) {
errors.sort(compareSelect(err => err.module, compareModulesByIdentifier));
for (const error of errors) {
this.errors.push(error);
}
}
this.logger.log(
`${statModulesHashed} modules hashed, ${statModulesFromCache} from cache (${
Math.round(
@ -3930,17 +3939,22 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
hashFunction,
runtimeTemplate,
hashDigest,
hashDigestLength
hashDigestLength,
errors
) {
const moduleHash = createHash(hashFunction);
module.updateHash(moduleHash, {
chunkGraph,
runtime,
runtimeTemplate
});
const moduleHashDigest = /** @type {string} */ (
moduleHash.digest(hashDigest)
);
let moduleHashDigest;
try {
const moduleHash = createHash(hashFunction);
module.updateHash(moduleHash, {
chunkGraph,
runtime,
runtimeTemplate
});
moduleHashDigest = /** @type {string} */ (moduleHash.digest(hashDigest));
} catch (err) {
errors.push(new ModuleHashingError(module, err));
moduleHashDigest = "XXXXXX";
}
chunkGraph.setModuleHashes(
module,
runtime,
@ -4091,6 +4105,7 @@ This prevents using hashes of each other and should be avoided.`);
const codeGenerationJobs = [];
/** @type {Map<string, Map<Module, {module: Module, hash: string, runtime: RuntimeSpec, runtimes: RuntimeSpec[]}>>} */
const codeGenerationJobsMap = new Map();
const errors = [];
const processChunk = chunk => {
// Last minute module hash generation for modules that depend on chunk hashes
@ -4105,7 +4120,8 @@ This prevents using hashes of each other and should be avoided.`);
hashFunction,
runtimeTemplate,
hashDigest,
hashDigestLength
hashDigestLength,
errors
);
let hashMap = codeGenerationJobsMap.get(hash);
if (hashMap) {
@ -4128,10 +4144,10 @@ This prevents using hashes of each other and should be avoided.`);
codeGenerationJobs.push(job);
}
}
this.logger.timeAggregate("hashing: hash runtime modules");
this.logger.time("hashing: hash chunks");
const chunkHash = createHash(hashFunction);
try {
this.logger.timeAggregate("hashing: hash runtime modules");
this.logger.time("hashing: hash chunks");
const chunkHash = createHash(hashFunction);
if (outputOptions.hashSalt) {
chunkHash.update(outputOptions.hashSalt);
}
@ -4162,6 +4178,12 @@ This prevents using hashes of each other and should be avoided.`);
};
otherChunks.forEach(processChunk);
for (const chunk of runtimeChunks) processChunk(chunk);
if (errors.length > 0) {
errors.sort(compareSelect(err => err.module, compareModulesByIdentifier));
for (const error of errors) {
this.errors.push(error);
}
}
this.logger.timeAggregateEnd("hashing: hash runtime modules");
this.logger.timeAggregateEnd("hashing: hash chunks");
@ -4801,6 +4823,9 @@ This prevents using hashes of each other and should be avoided.`);
chunkGraph.connectChunkAndModule(chunk, module);
}
/** @type {WebpackError[]} */
const errors = [];
// Hash modules
for (const module of modules) {
this._createModuleHash(
@ -4810,15 +4835,14 @@ This prevents using hashes of each other and should be avoided.`);
hashFunction,
runtimeTemplate,
hashDigest,
hashDigestLength
hashDigestLength,
errors
);
}
const codeGenerationResults = new CodeGenerationResults(
this.outputOptions.hashFunction
);
/** @type {WebpackError[]} */
const errors = [];
/**
* @param {Module} module the module
* @param {Callback} callback callback

29
lib/ModuleHashingError.js Normal file
View File

@ -0,0 +1,29 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const WebpackError = require("./WebpackError");
/** @typedef {import("./Module")} Module */
class ModuleHashingError extends WebpackError {
/**
* Create a new ModuleHashingError
* @param {Module} module related module
* @param {Error} error Original error
*/
constructor(module, error) {
super();
this.name = "ModuleHashingError";
this.error = error;
this.message = error.message;
this.details = error.stack;
this.module = module;
}
}
module.exports = ModuleHashingError;

62
lib/asset/AssetGenerator.js
View File

@ -147,14 +147,18 @@ class AssetGenerator extends Generator {
/**
* @param {NormalModule} module module
* @returns {string|undefined} mime type
* @returns {string} mime type
*/
getMimeType(module) {
if (typeof this.dataUrlOptions === "function") return;
if (typeof this.dataUrlOptions === "function") {
throw new Error(
"This method must not be called when dataUrlOptions is a function"
);
}
let mimeType = this.dataUrlOptions.mimetype;
if (mimeType === undefined) {
let ext = path.extname(module.nameForCondition());
const ext = path.extname(module.nameForCondition());
if (
module.resourceResolveData &&
module.resourceResolveData.mimetype !== undefined
@ -164,9 +168,26 @@ class AssetGenerator extends Generator {
module.resourceResolveData.parameters;
} else if (ext) {
mimeType = mimeTypes.lookup(ext);
if (typeof mimeType !== "string") {
throw new Error(
"DataUrl can't be generated automatically, " +
`because there is no mimetype for "${ext}" in mimetype database. ` +
'Either pass a mimetype via "generator.mimetype" or ' +
'use type: "asset/resource" to create a resource file instead of a DataUrl'
);
}
}
}
if (typeof mimeType !== "string") {
throw new Error(
"DataUrl can't be generated automatically. " +
'Either pass a mimetype via "generator.mimetype" or ' +
'use type: "asset/resource" to create a resource file instead of a DataUrl'
);
}
return mimeType;
}
@ -212,16 +233,6 @@ class AssetGenerator extends Generator {
encoding = DEFAULT_ENCODING;
}
const mimeType = this.getMimeType(module);
if (typeof mimeType !== "string") {
throw new Error(
"DataUrl can't be generated automatically, " +
`because there is no mimetype for "${path.extname(
module.nameForCondition()
)}" in mimetype database. ` +
'Either pass a mimetype via "generator.mimetype" or ' +
'use type: "asset/resource" to create a resource file instead of a DataUrl'
);
}
let encodedContent;
@ -397,12 +408,18 @@ class AssetGenerator extends Generator {
updateHash(hash, { module, runtime, runtimeTemplate, chunkGraph }) {
if (module.buildInfo.dataUrl) {
hash.update("data-url");
// this.dataUrlOptions as function should be pure and only depend on input source and filename
// therefore it doesn't need to be hashed
if (typeof this.dataUrlOptions === "function") {
hash.update("unknown-encoding");
hash.update("unknown-mimetype");
const ident = /** @type {{ ident?: string }} */ (this.dataUrlOptions)
.ident;
if (ident) hash.update(ident);
} else {
hash.update(this.dataUrlOptions.encoding || DEFAULT_ENCODING);
hash.update(this.getMimeType(module) || "unknown-mimetype");
if (this.dataUrlOptions.encoding)
hash.update(this.dataUrlOptions.encoding);
if (this.dataUrlOptions.mimetype)
hash.update(this.dataUrlOptions.mimetype);
// computed mimetype depends only on module filename which is already part of the hash
}
} else {
hash.update("resource");
@ -416,21 +433,26 @@ class AssetGenerator extends Generator {
};
if (typeof this.publicPath === "function") {
hash.update(`path:${this.publicPath(pathData, {})}`);
hash.update("path");
const assetInfo = {};
hash.update(this.publicPath(pathData, assetInfo));
hash.update(JSON.stringify(assetInfo));
} else if (this.publicPath) {
hash.update(`path:${this.publicPath}`);
hash.update("path");
hash.update(this.publicPath);
} else {
hash.update("no-path");
}
const assetModuleFilename =
this.filename || runtimeTemplate.outputOptions.assetModuleFilename;
const { path: filename } =
const { path: filename, info } =
runtimeTemplate.compilation.getAssetPathWithInfo(
assetModuleFilename,
pathData
);
hash.update(filename);
hash.update(JSON.stringify(info));
}
}
}

2
schemas/WebpackOptions.json
View File

@ -70,7 +70,7 @@
]
},
"AssetGeneratorDataUrlFunction": {
"description": "Function that executes for module and should return an DataUrl string.",
"description": "Function that executes for module and should return an DataUrl string. It can have a string as 'ident' property which contributes to the module hash.",
"instanceof": "Function",
"tsType": "((source: string | Buffer, context: { filename: string, module: import('../lib/Module') }) => string)"
},

18
test/__snapshots__/StatsTestCases.basictest.js.snap
View File

@ -152,7 +152,7 @@ webpack/runtime/make namespace object 274 bytes {main} [code generated]
[no exports]
[used exports unknown]
1970-04-20 12:42:42: webpack x.x.x compiled successfully in X ms (5406e3ea7ff9b8dee68e)"
1970-04-20 12:42:42: webpack x.x.x compiled successfully in X ms (d0d97703a88fdf5418be)"
`;
exports[`StatsTestCases should print correct stats for asset 1`] = `
@ -2536,7 +2536,7 @@ LOG from webpack.FileSystemInfo
exports[`StatsTestCases should print correct stats for real-content-hash 1`] = `
"a-normal:
assets by path *.js 3.23 KiB
asset 8b31b7e863da4eb900ec-8b31b7.js 2.75 KiB [emitted] [immutable] [minimized] (name: runtime)
asset 9690d063d027560c1900-9690d0.js 2.75 KiB [emitted] [immutable] [minimized] (name: runtime)
asset a6d438a0676f93383d79-a6d438.js 262 bytes [emitted] [immutable] [minimized] (name: lazy)
asset cbb9c74e42f00ada40f7-cbb9c7.js 212 bytes [emitted] [immutable] [minimized] (name: index)
asset 666f2b8847021ccc7608-666f2b.js 21 bytes [emitted] [immutable] [minimized] (name: a, b)
@ -2544,7 +2544,7 @@ exports[`StatsTestCases should print correct stats for real-content-hash 1`] = `
asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
asset 7382fad5b015914e0811.jpg?query 5.89 KiB [cached] [immutable] [from: file.jpg?query] (auxiliary name: lazy)
asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: index)
Entrypoint index 2.96 KiB (5.89 KiB) = 8b31b7e863da4eb900ec-8b31b7.js 2.75 KiB cbb9c74e42f00ada40f7-cbb9c7.js 212 bytes 1 auxiliary asset
Entrypoint index 2.96 KiB (5.89 KiB) = 9690d063d027560c1900-9690d0.js 2.75 KiB cbb9c74e42f00ada40f7-cbb9c7.js 212 bytes 1 auxiliary asset
Entrypoint a 21 bytes = 666f2b8847021ccc7608-666f2b.js
Entrypoint b 21 bytes = 666f2b8847021ccc7608-666f2b.js
runtime modules 7.28 KiB 9 modules
@ -2590,8 +2590,8 @@ b-normal:
a-source-map:
assets by path *.js 3.45 KiB
asset db6c639d86e9740bd622-db6c63.js 2.8 KiB [emitted] [immutable] [minimized] (name: runtime)
sourceMap db6c639d86e9740bd622-db6c63.js.map 14.5 KiB [emitted] [dev] (auxiliary name: runtime)
asset 3c4c8b6907eb902d46e3-3c4c8b.js 2.8 KiB [emitted] [immutable] [minimized] (name: runtime)
sourceMap 3c4c8b6907eb902d46e3-3c4c8b.js.map 14.5 KiB [emitted] [dev] (auxiliary name: runtime)
asset da6ceedb86c86e79a49a-da6cee.js 318 bytes [emitted] [immutable] [minimized] (name: lazy)
sourceMap da6ceedb86c86e79a49a-da6cee.js.map 401 bytes [emitted] [dev] (auxiliary name: lazy)
asset 9e0ae6ff74fb2c3c821b-9e0ae6.js 268 bytes [emitted] [immutable] [minimized] (name: index)
@ -2602,7 +2602,7 @@ a-source-map:
asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
asset 7382fad5b015914e0811.jpg?query 5.89 KiB [cached] [immutable] [from: file.jpg?query] (auxiliary name: lazy)
asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: index)
Entrypoint index 3.06 KiB (20.7 KiB) = db6c639d86e9740bd622-db6c63.js 2.8 KiB 9e0ae6ff74fb2c3c821b-9e0ae6.js 268 bytes 3 auxiliary assets
Entrypoint index 3.06 KiB (20.7 KiB) = 3c4c8b6907eb902d46e3-3c4c8b.js 2.8 KiB 9e0ae6ff74fb2c3c821b-9e0ae6.js 268 bytes 3 auxiliary assets
Entrypoint a 77 bytes (254 bytes) = 222c2acc68675174e6b2-222c2a.js 1 auxiliary asset
Entrypoint b 77 bytes (254 bytes) = 222c2acc68675174e6b2-222c2a.js 1 auxiliary asset
runtime modules 7.28 KiB 9 modules
@ -2621,8 +2621,8 @@ a-source-map:
b-source-map:
assets by path *.js 3.45 KiB
asset 796d1329e3aa20b86016-796d13.js 2.8 KiB [emitted] [immutable] [minimized] (name: runtime)
sourceMap 796d1329e3aa20b86016-796d13.js.map 14.5 KiB [emitted] [dev] (auxiliary name: runtime)
asset 3c4c8b6907eb902d46e3-3c4c8b.js 2.8 KiB [emitted] [immutable] [minimized] (name: runtime)
sourceMap 3c4c8b6907eb902d46e3-3c4c8b.js.map 14.5 KiB [emitted] [dev] (auxiliary name: runtime)
asset da6ceedb86c86e79a49a-da6cee.js 318 bytes [emitted] [immutable] [minimized] (name: lazy)
sourceMap da6ceedb86c86e79a49a-da6cee.js.map 397 bytes [emitted] [dev] (auxiliary name: lazy)
asset 9e0ae6ff74fb2c3c821b-9e0ae6.js 268 bytes [emitted] [immutable] [minimized] (name: index)
@ -2633,7 +2633,7 @@ b-source-map:
asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
asset 7382fad5b015914e0811.jpg?query 5.89 KiB [cached] [immutable] [from: file.jpg?query] (auxiliary name: lazy)
asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: index)
Entrypoint index 3.06 KiB (20.7 KiB) = 796d1329e3aa20b86016-796d13.js 2.8 KiB 9e0ae6ff74fb2c3c821b-9e0ae6.js 268 bytes 3 auxiliary assets
Entrypoint index 3.06 KiB (20.7 KiB) = 3c4c8b6907eb902d46e3-3c4c8b.js 2.8 KiB 9e0ae6ff74fb2c3c821b-9e0ae6.js 268 bytes 3 auxiliary assets
Entrypoint a 77 bytes (254 bytes) = 222c2acc68675174e6b2-222c2a.js 1 auxiliary asset
Entrypoint b 77 bytes (254 bytes) = 222c2acc68675174e6b2-222c2a.js 1 auxiliary asset
runtime modules 7.28 KiB 9 modules