root
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

move filename processing from asset generator to parser 

remove memory leak from asset generator
This commit is contained in:
Tobias Koppers 2021-03-11 10:40:03 +01:00
parent 20b6d3c991
commit f93aacd55f
7 changed files with 165 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
declarations/WebpackOptions.d.ts vendored
View File

@ -705,6 +705,11 @@ export type AssetParserDataUrlFunction = (
source: string | Buffer,
context: {filename: string; module: import("../lib/Module")}
) => boolean;
/**
* Parser options for asset modules.
*/
export type AssetParserOptions = AssetResourceParserOptions &
AssetParserOptionsExtra;
/**
* A Function returning a Promise resolving to a normalized entry.
*/
@ -2571,19 +2576,19 @@ export interface AssetParserDataUrlOptions {
*/
maxSize?: number;
}
/**
* Parser options for asset modules.
*/
export interface AssetParserOptions {
/**
* The condition for inlining the asset as DataUrl.
*/
dataUrlCondition?: AssetParserDataUrlOptions | AssetParserDataUrlFunction;
}
/**
* Generator options for asset/resource modules.
*/
export interface AssetResourceGeneratorOptions {
/**
* This is deprecated and has moved to 'parser.filename'.
*/
filename?: FilenameTemplate;
}
/**
* Parser options for asset/resource modules.
*/
export interface AssetResourceParserOptions {
/**
* Specifies the filename template of output files on disk. You must **not** specify an absolute path here, but the path may contain folders separated by '/'! The specified path is joined with the value of the 'output.path' option to determine the location on disk.
*/
@ -3137,6 +3142,15 @@ export interface WebpackOptionsNormalized {
*/
watchOptions: WatchOptions;
}
/**
* Parser options for asset modules.
*/
export interface AssetParserOptionsExtra {
/**
* The condition for inlining the asset as DataUrl.
*/
dataUrlCondition?: AssetParserDataUrlOptions | AssetParserDataUrlFunction;
}
/**
* If an dependency matches exactly a property of the object, the property value is used as dependency.
*/
@ -3213,9 +3227,9 @@ export interface ParserOptionsByModuleTypeKnown {
*/
"asset/inline"?: EmptyParserOptions;
/**
* No parser options are supported for this module type.
* Parser options for asset/resource modules.
*/
"asset/resource"?: EmptyParserOptions;
"asset/resource"?: AssetResourceParserOptions;
/**
* No parser options are supported for this module type.
*/

55
lib/asset/AssetGenerator.js
View File

@ -10,8 +10,6 @@ const path = require("path");
const { RawSource } = require("webpack-sources");
const Generator = require("../Generator");
const RuntimeGlobals = require("../RuntimeGlobals");
const createHash = require("../util/createHash");
const { makePathsRelative } = require("../util/identifier");
/** @typedef {import("webpack-sources").Source} Source */
/** @typedef {import("../../declarations/WebpackOptions").AssetGeneratorOptions} AssetGeneratorOptions */
@ -29,13 +27,11 @@ const JS_AND_ASSET_TYPES = new Set(["javascript", "asset"]);
class AssetGenerator extends Generator {
/**
* @param {Compilation} compilation the compilation
* @param {AssetGeneratorOptions["dataUrl"]=} dataUrlOptions the options for the data url
* @param {string=} filename override for output.assetModuleFilename
*/
constructor(compilation, dataUrlOptions, filename) {
constructor(dataUrlOptions, filename) {
super();
this.compilation = compilation;
this.dataUrlOptions = dataUrlOptions;
this.filename = filename;
}
@ -111,49 +107,14 @@ class AssetGenerator extends Generator {
)};`
);
} else {
const assetModuleFilename =
this.filename || runtimeTemplate.outputOptions.assetModuleFilename;
const hash = createHash(runtimeTemplate.outputOptions.hashFunction);
if (runtimeTemplate.outputOptions.hashSalt) {
hash.update(runtimeTemplate.outputOptions.hashSalt);
}
hash.update(originalSource.buffer());
const fullHash = /** @type {string} */ (hash.digest(
runtimeTemplate.outputOptions.hashDigest
));
const contentHash = fullHash.slice(
0,
runtimeTemplate.outputOptions.hashDigestLength
);
module.buildInfo.fullContentHash = fullHash;
const sourceFilename = makePathsRelative(
this.compilation.compiler.context,
module.matchResource || module.resource,
this.compilation.compiler.root
).replace(/^\.\//, "");
const {
path: filename,
info
} = this.compilation.getAssetPathWithInfo(assetModuleFilename, {
module,
runtime,
filename: sourceFilename,
chunkGraph,
contentHash
});
module.buildInfo.filename = filename;
module.buildInfo.assetInfo = {
sourceFilename,
...info
};
if (getData) {
// Due to code generation caching module.buildInfo.XXX can't used to store such information
// It need to be stored in the code generation results instead, where it's cached too
// TODO webpack 6 For back-compat reasons we also store in on module.buildInfo
// We did a mistake in some minor version of 5.x
// Now we have to keep it for backward-compat reasons
// TODO webpack 6 remove
const data = getData();
data.set("fullContentHash", fullHash);
data.set("filename", filename);
data.set("assetInfo", info);
data.set("fullContentHash", module.buildInfo.fullContentHash);
data.set("filename", module.buildInfo.filename);
data.set("assetInfo", module.buildInfo.assetInfo);
}
runtimeRequirements.add(RuntimeGlobals.publicPath); // add __webpack_require__.p
@ -161,7 +122,7 @@ class AssetGenerator extends Generator {
return new RawSource(
`${RuntimeGlobals.module}.exports = ${
RuntimeGlobals.publicPath
} + ${JSON.stringify(filename)};`
} + ${JSON.stringify(module.buildInfo.filename)};`
);
}
}

23
lib/asset/AssetModulesPlugin.js
View File

@ -31,6 +31,7 @@ const getGeneratorSchemaMap = {
const getParserSchema = memoize(() => getSchema("AssetParserOptions"));
const getAssetGenerator = memoize(() => require("./AssetGenerator"));
const getAssetParser = memoize(() => require("./AssetParser"));
const getAssetSourceParser = memoize(() => require("./AssetSourceParser"));
const getAssetSourceGenerator = memoize(() =>
require("./AssetSourceGenerator")
);
@ -70,7 +71,7 @@ class AssetModulesPlugin {
const AssetParser = getAssetParser();
return new AssetParser(dataUrlCondition);
return new AssetParser(dataUrlCondition, parserOptions.filename);
});
normalModuleFactory.hooks.createParser
.for("asset/inline")
@ -84,14 +85,14 @@ class AssetModulesPlugin {
.tap(plugin, parserOptions => {
const AssetParser = getAssetParser();
return new AssetParser(false);
return new AssetParser(false, parserOptions.filename);
});
normalModuleFactory.hooks.createParser
.for("asset/source")
.tap(plugin, parserOptions => {
const AssetParser = getAssetParser();
const AssetSourceParser = getAssetSourceParser();
return new AssetParser(false);
return new AssetSourceParser();
});
for (const type of ["asset", "asset/inline", "asset/resource"]) {
@ -123,7 +124,7 @@ class AssetModulesPlugin {
const AssetGenerator = getAssetGenerator();
return new AssetGenerator(compilation, dataUrl, filename);
return new AssetGenerator(dataUrl, filename);
});
}
normalModuleFactory.hooks.createGenerator
@ -151,17 +152,11 @@ class AssetModulesPlugin {
);
result.push({
render: () => codeGenResult.sources.get(type),
filename:
module.buildInfo.filename ||
codeGenResult.data.get("filename"),
info:
module.buildInfo.assetInfo ||
codeGenResult.data.get("assetInfo"),
filename: module.buildInfo.filename,
info: module.buildInfo.assetInfo,
auxiliary: true,
identifier: `assetModule${chunkGraph.getModuleId(module)}`,
hash:
module.buildInfo.fullContentHash ||
codeGenResult.data.get("fullContentHash")
hash: module.buildInfo.fullContentHash
});
}
}

60
lib/asset/AssetParser.js
View File

@ -6,6 +6,9 @@
"use strict";
const Parser = require("../Parser");
const createHash = require("../util/createHash");
const { makePathsRelative } = require("../util/identifier");
const AssetGenerator = require("./AssetGenerator");
/** @typedef {import("../../declarations/WebpackOptions").AssetParserOptions} AssetParserOptions */
/** @typedef {import("../Parser").ParserState} ParserState */
@ -14,10 +17,12 @@ const Parser = require("../Parser");
class AssetParser extends Parser {
/**
* @param {AssetParserOptions["dataUrlCondition"] | boolean} dataUrlCondition condition for inlining as DataUrl
* @param {string=} filename override for output.assetModuleFilename
*/
constructor(dataUrlCondition) {
constructor(dataUrlCondition, filename) {
super();
this.dataUrlCondition = dataUrlCondition;
this.filename = filename;
}
/**
@ -29,26 +34,65 @@ class AssetParser extends Parser {
if (typeof source === "object" && !Buffer.isBuffer(source)) {
throw new Error("AssetParser doesn't accept preparsed AST");
}
state.module.buildInfo.strict = true;
state.module.buildMeta.exportsType = "default";
const { module, compilation } = state;
module.buildInfo.strict = true;
module.buildMeta.exportsType = "default";
if (typeof this.dataUrlCondition === "function") {
state.module.buildInfo.dataUrl = this.dataUrlCondition(source, {
filename: state.module.matchResource || state.module.resource,
module: state.module
module.buildInfo.dataUrl = this.dataUrlCondition(source, {
filename: module.matchResource || module.resource,
module: module
});
} else if (typeof this.dataUrlCondition === "boolean") {
state.module.buildInfo.dataUrl = this.dataUrlCondition;
module.buildInfo.dataUrl = this.dataUrlCondition;
} else if (
this.dataUrlCondition &&
typeof this.dataUrlCondition === "object"
) {
state.module.buildInfo.dataUrl =
module.buildInfo.dataUrl =
Buffer.byteLength(source) <= this.dataUrlCondition.maxSize;
} else {
throw new Error("Unexpected dataUrlCondition type");
}
if (!module.buildInfo.dataUrl) {
const outputOptions = compilation.outputOptions;
const assetModuleFilename =
this.filename ||
// TODO webpack 6 remove
(module.generator instanceof AssetGenerator &&
module.generator.filename) ||
outputOptions.assetModuleFilename;
const hash = createHash(outputOptions.hashFunction);
if (outputOptions.hashSalt) {
hash.update(outputOptions.hashSalt);
}
hash.update(source);
const fullHash = /** @type {string} */ (hash.digest(
outputOptions.hashDigest
));
const contentHash = fullHash.slice(0, outputOptions.hashDigestLength);
module.buildInfo.fullContentHash = fullHash;
const sourceFilename = makePathsRelative(
compilation.compiler.context,
module.matchResource || module.resource,
compilation.compiler.root
).replace(/^\.\//, "");
const { path: filename, info } = compilation.getAssetPathWithInfo(
assetModuleFilename,
{
module,
filename: sourceFilename,
contentHash
}
);
module.buildInfo.filename = filename;
module.buildInfo.assetInfo = {
sourceFilename,
...info
};
}
return state;
}
}

31
lib/asset/AssetSourceParser.js Normal file
View File

@ -0,0 +1,31 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Yuta Hiroto @hiroppy
*/
"use strict";
const Parser = require("../Parser");
/** @typedef {import("../Parser").ParserState} ParserState */
/** @typedef {import("../Parser").PreparsedAst} PreparsedAst */
class AssetSourceParser extends Parser {
/**
* @param {string | Buffer | PreparsedAst} source the source to parse
* @param {ParserState} state the parser state
* @returns {ParserState} the parser state
*/
parse(source, state) {
if (typeof source === "object" && !Buffer.isBuffer(source)) {
throw new Error("AssetSourceParser doesn't accept preparsed AST");
}
const { module } = state;
module.buildInfo.strict = true;
module.buildMeta.exportsType = "default";
return state;
}
}
module.exports = AssetSourceParser;

28
schemas/WebpackOptions.json
View File

@ -102,7 +102,12 @@
"$ref": "#/definitions/AssetGeneratorDataUrl"
},
"filename": {
"$ref": "#/definitions/FilenameTemplate"
"description": "This is deprecated and has moved to 'parser.filename'.",
"oneOf": [
{
"$ref": "#/definitions/FilenameTemplate"
}
]
}
}
},
@ -148,6 +153,7 @@
"AssetParserOptions": {
"description": "Parser options for asset modules.",
"type": "object",
"implements": ["#/definitions/AssetResourceParserOptions"],
"additionalProperties": false,
"properties": {
"dataUrlCondition": {
@ -160,6 +166,9 @@
"$ref": "#/definitions/AssetParserDataUrlFunction"
}
]
},
"filename": {
"$ref": "#/definitions/FilenameTemplate"
}
}
},
@ -167,6 +176,21 @@
"description": "Generator options for asset/resource modules.",
"type": "object",
"additionalProperties": false,
"properties": {
"filename": {
"description": "This is deprecated and has moved to 'parser.filename'.",
"oneOf": [
{
"$ref": "#/definitions/FilenameTemplate"
}
]
}
}
},
"AssetResourceParserOptions": {
"description": "Parser options for asset/resource modules.",
"type": "object",
"additionalProperties": false,
"properties": {
"filename": {
"$ref": "#/definitions/FilenameTemplate"
@ -2814,7 +2838,7 @@
"$ref": "#/definitions/EmptyParserOptions"
},
"asset/resource": {
"$ref": "#/definitions/EmptyParserOptions"
"$ref": "#/definitions/AssetResourceParserOptions"
},
"asset/source": {
"$ref": "#/definitions/EmptyParserOptions"

17
types.d.ts vendored
View File

@ -279,11 +279,12 @@ declare interface AssetParserDataUrlOptions {
*/
maxSize?: number;
}
type AssetParserOptions = AssetResourceParserOptions & AssetParserOptionsExtra;
/**
* Parser options for asset modules.
*/
declare interface AssetParserOptions {
declare interface AssetParserOptionsExtra {
/**
* The condition for inlining the asset as DataUrl.
*/
@ -299,6 +300,16 @@ declare interface AssetParserOptions {
* Generator options for asset/resource modules.
*/
declare interface AssetResourceGeneratorOptions {
/**
* This is deprecated and has moved to 'parser.filename'.
*/
filename?: string | ((pathData: PathData, assetInfo?: AssetInfo) => string);
}
/**
* Parser options for asset/resource modules.
*/
declare interface AssetResourceParserOptions {
/**
* Specifies the filename template of output files on disk. You must **not** specify an absolute path here, but the path may contain folders separated by '/'! The specified path is joined with the value of the 'output.path' option to determine the location on disk.
*/
@ -7683,9 +7694,9 @@ declare interface ParserOptionsByModuleTypeKnown {
"asset/inline"?: EmptyParserOptions;
/**
* No parser options are supported for this module type.
* Parser options for asset/resource modules.
*/
"asset/resource"?: EmptyParserOptions;
"asset/resource"?: AssetResourceParserOptions;
/**
* No parser options are supported for this module type.