From 09862aacf87f53168c0b490b68db4cc3b5abc9c8 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 7 Jan 2021 13:06:44 +0100 Subject: [PATCH 1/2] add warning when invalid dependencies are reported by loaders/plugins add automatic workaround for invalid dependencies #12340 #12283 --- lib/InvalidDependenciesModuleWarning.js | 38 ++++++++++++++ lib/NormalModule.js | 46 +++++++++++++++++ lib/index.js | 3 ++ .../invalid-dependencies/index.js | 1 + .../invalid-dependencies/loader.js | 11 ++++ .../invalid-dependencies/warnings.js | 12 +++++ .../invalid-dependencies/webpack.config.js | 50 +++++++++++++++++++ .../define-plugin/webpack.config.js | 4 +- types.d.ts | 5 +- 9 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 lib/InvalidDependenciesModuleWarning.js create mode 100644 test/configCases/deprecations/invalid-dependencies/index.js create mode 100644 test/configCases/deprecations/invalid-dependencies/loader.js create mode 100644 test/configCases/deprecations/invalid-dependencies/warnings.js create mode 100644 test/configCases/deprecations/invalid-dependencies/webpack.config.js diff --git a/lib/InvalidDependenciesModuleWarning.js b/lib/InvalidDependenciesModuleWarning.js new file mode 100644 index 000000000..58f04bb26 --- /dev/null +++ b/lib/InvalidDependenciesModuleWarning.js @@ -0,0 +1,38 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const WebpackError = require("./WebpackError"); + +/** @typedef {import("./Dependency").DependencyLocation} DependencyLocation */ +/** @typedef {import("./Module")} Module */ + +module.exports = class InvalidDependenciesModuleWarning extends WebpackError { + /** + * @param {Module} module module tied to dependency + * @param {Iterable} deps invalid dependencies + */ + constructor(module, deps) { + const orderedDeps = Array.from(deps).sort(); + const depsList = orderedDeps.map(dep => ` * ${JSON.stringify(dep)}`); + super(`Invalid dependencies have been reported by plugins or loaders for this module. All reported dependencies need to be absolute paths. +Invalid dependencies may lead to broken watching and caching. +As best effort we try to convert all invalid values to absolute paths and converting globs into context dependencies, but this is deprecated behavior. +Loaders: Pass absolute paths to this.addDependency (existing files), this.addMissingDependency (not existing files), and this.addContextDependency (directories). +Plugins: Pass absolute paths to fileDependencies (existing files), missingDependencies (not existing files), and contextDependencies (directories). +Globs: They are not supported. Pass absolute path to the directory as context dependencies. +The following invalid values have been reported: +${depsList.slice(0, 3).join("\n")}${ + depsList.length > 3 ? "\n * and more ..." : "" + }`); + + this.name = "InvalidDependenciesModuleWarning"; + this.details = depsList.slice(3).join("\n"); + this.module = module; + + Error.captureStackTrace(this, this.constructor); + } +}; diff --git a/lib/NormalModule.js b/lib/NormalModule.js index 93918e338..b62905619 100644 --- a/lib/NormalModule.js +++ b/lib/NormalModule.js @@ -36,8 +36,10 @@ const { keepOriginalOrder } = require("./util/comparators"); const createHash = require("./util/createHash"); +const { join } = require("./util/fs"); const { contextify } = require("./util/identifier"); const makeSerializable = require("./util/makeSerializable"); +const memoize = require("./util/memoize"); /** @typedef {import("source-map").RawSourceMap} SourceMap */ /** @typedef {import("webpack-sources").Source} Source */ @@ -61,6 +63,12 @@ const makeSerializable = require("./util/makeSerializable"); /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */ /** @typedef {import("./util/runtime").RuntimeSpec} RuntimeSpec */ +const getInvalidDependenciesModuleWarning = memoize(() => + require("./InvalidDependenciesModuleWarning") +); + +const ABSOLUTE_PATH_REGEX = /^([a-zA-Z]:\\|\\\\|\/)/; + /** * @typedef {Object} LoaderItem * @property {string} loader @@ -814,6 +822,44 @@ class NormalModule extends Module { if (!this.buildInfo.cacheable || !snapshotOptions) { return callback(); } + // add warning for all non-absolute paths in fileDependencies, etc + // This makes it easier to find problems with watching and/or caching + let nonAbsoluteDependencies = undefined; + const checkDependencies = deps => { + for (const dep of deps) { + if (!ABSOLUTE_PATH_REGEX.test(dep)) { + if (nonAbsoluteDependencies === undefined) + nonAbsoluteDependencies = new Set(); + nonAbsoluteDependencies.add(dep); + deps.delete(dep); + try { + const depWithoutGlob = dep.replace(/[\\/]?\*.*$/, ""); + const absolute = join( + compilation.fileSystemInfo.fs, + this.context, + depWithoutGlob + ); + if (absolute !== dep && ABSOLUTE_PATH_REGEX.test(absolute)) { + (depWithoutGlob !== dep + ? this.buildInfo.contextDependencies + : deps + ).add(absolute); + } + } catch (e) { + // ignore + } + } + } + }; + checkDependencies(this.buildInfo.fileDependencies); + checkDependencies(this.buildInfo.missingDependencies); + checkDependencies(this.buildInfo.contextDependencies); + if (nonAbsoluteDependencies !== undefined) { + const InvalidDependenciesModuleWarning = getInvalidDependenciesModuleWarning(); + this.addWarning( + new InvalidDependenciesModuleWarning(this, nonAbsoluteDependencies) + ); + } // convert file/context/missingDependencies into filesystem snapshot compilation.fileSystemInfo.createSnapshot( startTime, diff --git a/lib/index.js b/lib/index.js index 334c0d5dd..a7b0783a8 100644 --- a/lib/index.js +++ b/lib/index.js @@ -497,6 +497,9 @@ module.exports = mergeExports(fn, { }, get cleverMerge() { return require("./util/cleverMerge").cachedCleverMerge; + }, + get LazySet() { + return require("./util/LazySet"); } }, diff --git a/test/configCases/deprecations/invalid-dependencies/index.js b/test/configCases/deprecations/invalid-dependencies/index.js new file mode 100644 index 000000000..ba421732e --- /dev/null +++ b/test/configCases/deprecations/invalid-dependencies/index.js @@ -0,0 +1 @@ +it("should compile", () => {}); diff --git a/test/configCases/deprecations/invalid-dependencies/loader.js b/test/configCases/deprecations/invalid-dependencies/loader.js new file mode 100644 index 000000000..47a39616b --- /dev/null +++ b/test/configCases/deprecations/invalid-dependencies/loader.js @@ -0,0 +1,11 @@ +module.exports = function (source) { + this.addDependency("loader.js"); + this.addDependency("../**/dir/*.js"); + this.addDependency("../**/file.js"); + this.addContextDependency("."); + this.addMissingDependency("./missing1.js"); + this.addMissingDependency("missing2.js"); + this.addMissingDependency("missing3.js"); + this.addMissingDependency("missing4.js"); + return source; +}; diff --git a/test/configCases/deprecations/invalid-dependencies/warnings.js b/test/configCases/deprecations/invalid-dependencies/warnings.js new file mode 100644 index 000000000..6debc7757 --- /dev/null +++ b/test/configCases/deprecations/invalid-dependencies/warnings.js @@ -0,0 +1,12 @@ +module.exports = [ + [ + { moduleName: /\.\/index\.js/ }, + /Invalid dependencies have been reported/, + /"\."/, + /"\.\.\/\*\*\/dir\/\*\.js"/, + { details: /"\.\/missing1\.js"/ }, + { details: /"loader\.js"/ }, + /and more/, + { details: /"missing3\.js"/ } + ] +]; diff --git a/test/configCases/deprecations/invalid-dependencies/webpack.config.js b/test/configCases/deprecations/invalid-dependencies/webpack.config.js new file mode 100644 index 000000000..5a8344b42 --- /dev/null +++ b/test/configCases/deprecations/invalid-dependencies/webpack.config.js @@ -0,0 +1,50 @@ +const webpack = require("../../../../"); +const path = require("path"); + +/** @type {import("../../../../").Configuration} */ +module.exports = { + module: { + rules: [ + { + test: /index\.js$/, + loader: "./loader.js" + } + ] + }, + plugins: [ + compiler => { + compiler.hooks.compilation.tap("Test", compilation => { + compilation.hooks.succeedModule.tap("Test", module => { + const fileDeps = new webpack.util.LazySet(); + const contextDeps = new webpack.util.LazySet(); + const missingDeps = new webpack.util.LazySet(); + const buildDeps = new webpack.util.LazySet(); + module.addCacheDependencies( + fileDeps, + contextDeps, + missingDeps, + buildDeps + ); + expect(Array.from(fileDeps).sort()).toEqual([ + path.join(__dirname, "index.js"), + path.join(__dirname, "loader.js") + ]); + expect(Array.from(contextDeps).sort()).toEqual([ + path.join(__dirname, ".."), + __dirname + ]); + expect(Array.from(missingDeps).sort()).toEqual([ + path.join(__dirname, "missing1.js"), + path.join(__dirname, "missing2.js"), + path.join(__dirname, "missing3.js"), + path.join(__dirname, "missing4.js") + ]); + expect(Array.from(fileDeps).sort()).toEqual([ + path.join(__dirname, "index.js"), + path.join(__dirname, "loader.js") + ]); + }); + }); + } + ] +}; diff --git a/test/statsCases/define-plugin/webpack.config.js b/test/statsCases/define-plugin/webpack.config.js index 1a3a41f33..e2c5fdd50 100644 --- a/test/statsCases/define-plugin/webpack.config.js +++ b/test/statsCases/define-plugin/webpack.config.js @@ -45,12 +45,12 @@ module.exports = [ plugins: [ new webpack.DefinePlugin({ VALUE: webpack.DefinePlugin.runtimeValue(() => read("123.txt"), [ - "./123.txt" + join(__dirname, "./123.txt") ]) }), new webpack.DefinePlugin({ VALUE: webpack.DefinePlugin.runtimeValue(() => read("321.txt"), [ - "./321.txt" + join(__dirname, "./321.txt") ]) }) ] diff --git a/types.d.ts b/types.d.ts index 68ef3f6dc..cf22952a9 100644 --- a/types.d.ts +++ b/types.d.ts @@ -4757,7 +4757,8 @@ declare interface KnownStatsPrinterContext { formatTime?: (time: number, boldQuantity?: boolean) => string; chunkGroupKind?: string; } -declare abstract class LazySet { +declare class LazySet { + constructor(iterable?: Iterable); readonly size: number; add(item: T): LazySet; addAll(iterable: LazySet | Iterable): LazySet; @@ -4774,6 +4775,7 @@ declare abstract class LazySet { [Symbol.iterator](): IterableIterator; readonly [Symbol.toStringTag]: string; serialize(__0: { write: any }): void; + static deserialize(__0: { read: any }): LazySet; } declare interface LibIdentOptions { /** @@ -10784,6 +10786,7 @@ declare namespace exports { export { MEASURE_START_OPERATION, MEASURE_END_OPERATION }; } export const cleverMerge: (first: T, second: O) => T | O | (T & O); + export { LazySet }; } export namespace sources { export { From aaadeda40b7817b89cb5bb4edf99f629d110cb51 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 7 Jan 2021 15:37:19 +0100 Subject: [PATCH 2/2] non-fs resources should not end up in fileDependencies fixes #12283 --- lib/NormalModule.js | 3 ++- package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/NormalModule.js b/lib/NormalModule.js index b62905619..f721ccf41 100644 --- a/lib/NormalModule.js +++ b/lib/NormalModule.js @@ -647,7 +647,7 @@ class NormalModule extends Module { resource: this.resource, loaders: this.loaders, context: loaderContext, - readResource: (resource, callback) => { + processResource: (loaderContext, resource, callback) => { const scheme = getScheme(resource); if (scheme) { hooks.readResourceForScheme @@ -660,6 +660,7 @@ class NormalModule extends Module { return callback(null, result); }); } else { + loaderContext.addDependency(resource); fs.readFile(resource, callback); } } diff --git a/package.json b/package.json index ff610b31f..0e2c9fc81 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.4", "json-parse-better-errors": "^1.0.2", - "loader-runner": "^4.1.0", + "loader-runner": "^4.2.0", "mime-types": "^2.1.27", "neo-async": "^2.6.2", "pkg-dir": "^5.0.0", diff --git a/yarn.lock b/yarn.lock index 07cf230ca..398301073 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4348,10 +4348,10 @@ listr2@^3.2.2: rxjs "^6.6.3" through "^2.3.8" -loader-runner@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.1.0.tgz#f70bc0c29edbabdf2043e7ee73ccc3fe1c96b42d" - integrity sha512-oR4lB4WvwFoC70ocraKhn5nkKSs23t57h9udUgw8o0iH8hMXeEoRuUgfcvgUwAJ1ZpRqBvcou4N2SMvM1DwMrA== +loader-runner@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.2.0.tgz#d7022380d66d14c5fb1d496b89864ebcfd478384" + integrity sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw== loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0: version "1.4.0"