feat: extractSourceMap supports HTTP URLs (#19882)

lib/NormalModule.js

@@ -264,6 +264,10 @@ makeSerializable(
  * @property {boolean=} extractSourceMap enable/disable extracting source map
  */
 
+/**
+ * @typedef {(resourcePath: string, getLoaderContext: (resourcePath: string) => LoaderContext<EXPECTED_ANY>) => Promise<string | Buffer<ArrayBufferLike>>} ReadResource
+ */
+
 /** @type {WeakMap<Compilation, NormalModuleCompilationHooks>} */
 const compilationHooksMap = new WeakMap();
 
@@ -1023,43 +1027,76 @@ class NormalModule extends Module {
 				 * @param {LoaderContext<EXPECTED_ANY>} loaderContext the loader context
 				 * @param {string} resourcePath the resource Path
 				 * @param {(err: Error | null, result?: string | Buffer, sourceMap?: Result[1]) => void} callback callback
+				 * @returns {Promise<void>}
 				 */
-				processResource: (loaderContext, resourcePath, callback) => {
+				processResource: async (loaderContext, resourcePath, callback) => {
-					const resource = loaderContext.resource;
+					/** @type {ReadResource} */
-					const scheme = getScheme(resource);
+					const readResource = (resourcePath, getLoaderContext) => {
+						const scheme = getScheme(resourcePath);
+						return new Promise((resolve, reject) => {
 							hooks.readResource
 								.for(scheme)
-						.callAsync(loaderContext, async (err, result) => {
+								.callAsync(getLoaderContext(resourcePath), (err, result) => {
-							if (err) return callback(err);
+									if (err) {
+										reject(err);
+									} else {
 										if (typeof result !== "string" && !result) {
-								return callback(
+											return reject(
 												new UnhandledSchemeError(
 													/** @type {string} */
 													(scheme),
-										resource
+													resourcePath
 												)
 											);
 										}
+										resolve(result);
+									}
+								});
+						});
+					};
+					try {
+						const result = await readResource(
+							resourcePath,
+							() => loaderContext
+						);
 						if (
 							this.extractSourceMap &&
 							(this.useSourceMap || this.useSimpleSourceMap)
 						) {
 							try {
-									const { source, sourceMap, fileDependencies } =
+								const { source, sourceMap } = await getExtractSourceMap()(
-										await getExtractSourceMap()(result, resourcePath, fs);
+									result,
-									if (this.buildInfo && this.buildInfo.fileDependencies) {
+									resourcePath,
-										this.buildInfo.fileDependencies.addAll(fileDependencies);
+									/** @type {ReadResource} */
-									}
+									(resourcePath) =>
+										readResource(
+											resourcePath,
+											(resourcePath) =>
+												/** @type {LoaderContext<EXPECTED_ANY>} */ ({
+													addDependency(dependency) {
+														loaderContext.addDependency(dependency);
+													},
+													fs: loaderContext.fs,
+													_module: undefined,
+													resourcePath,
+													resource: resourcePath
+												})
+										).catch((err) => {
+											throw new Error(
+												`Failed to parse source map. ${/** @type {Error} */ (err).message}`
+											);
+										})
+								);
 								return callback(null, source, sourceMap);
 							} catch (err) {
-									this.addWarning(
+								this.addWarning(new ModuleWarning(/** @type {Error} */ (err)));
-										new ModuleWarning(/** @type {Error} */ (err))
-									);
 								return callback(null, result);
 							}
 						}
 						return callback(null, result);
-						});
+					} catch (error) {
+						return callback(/** @type {Error} */ (error));
+					}
 				}
 			},
 			(err, result) => {

lib/schemes/DataUriPlugin.js

@@ -35,6 +35,7 @@ class DataUriPlugin {
 							resourceData.data.encodedContent = match[4] || "";
 						}
 					});
+
 				NormalModule.getCompilationHooks(compilation)
 					.readResourceForScheme.for("data")
 					.tap(PLUGIN_NAME, (resource) => decodeDataURI(resource));

lib/schemes/FileUriPlugin.js

@@ -40,8 +40,11 @@ class FileUriPlugin {
 					.for(undefined)
 					.tapAsync(PLUGIN_NAME, (loaderContext, callback) => {
 						const { resourcePath } = loaderContext;
+						loaderContext.fs.readFile(resourcePath, (err, result) => {
+							if (err) return callback(err);
 							loaderContext.addDependency(resourcePath);
-						loaderContext.fs.readFile(resourcePath, callback);
+							callback(null, result);
+						});
 					});
 			}
 		);

lib/schemes/HttpUriPlugin.js

@@ -1199,8 +1199,10 @@ Run build with un-frozen lockfile to automatically fix lockfile.`
 							getInfo(resource, (err, _result) => {
 								if (err) return callback(err);
 								const result = /** @type {Info} */ (_result);
+								if (module) {
 									/** @type {BuildInfo} */
 									(module.buildInfo).resourceIntegrity = result.entry.integrity;
+								}
 								callback(null, result.content);
 							})
 						);

lib/util/extractSourceMap.js

@@ -7,7 +7,6 @@
 
 const path = require("path");
 const urlUtils = require("url");
-const { decodeDataURI } = require("./dataURL");
 const { isAbsolute, join } = require("./fs");
 
 /** @typedef {import("../../declarations/WebpackOptions").RuleSetRule["extractSourceMap"]} ExtractSourceMapOptions */
@@ -19,6 +18,10 @@ const { isAbsolute, join } = require("./fs");
 
 /** @typedef {import("webpack-sources").RawSourceMap} RawSourceMap */
 
+/**
+ * @typedef {(resourcePath: string) => Promise<string | Buffer<ArrayBufferLike>>} ReadResource
+ */
+
 /**
  * @typedef {object} SourceMappingURL
  * @property {string} sourceMappingURL
@@ -71,22 +74,21 @@ function getSourceMappingURL(code) {
 
 /**
  * Get absolute path for source file
- * @param {InputFileSystem} fs file system
  * @param {string} context context directory
  * @param {string} request file request
  * @param {string} sourceRoot source root directory
  * @returns {string} absolute path
  */
-function getAbsolutePath(fs, context, request, sourceRoot) {
+function getAbsolutePath(context, request, sourceRoot) {
 	if (sourceRoot) {
 		if (isAbsolute(sourceRoot)) {
-			return join(fs, sourceRoot, request);
+			return join(undefined, sourceRoot, request);
 		}
 
-		return join(fs, join(fs, context, sourceRoot), request);
+		return join(undefined, join(undefined, context, sourceRoot), request);
 	}
 
-	return join(fs, context, request);
+	return join(undefined, context, request);
 }
 
 /**
@@ -98,76 +100,22 @@ function isURL(value) {
 	return validProtocolPattern.test(value) && !path.win32.isAbsolute(value);
 }
 
-/**
- * Fetch source content from data URL
- * @param {InputFileSystem} fs file system
- * @param {string} sourceURL data URL
- * @returns {string} source content promise
- */
-function fetchFromDataURL(fs, sourceURL) {
-	const content = decodeDataURI(sourceURL);
-
-	if (content) {
-		return content.toString("utf8");
-	}
-
-	throw new Error(`Failed to parse source map from "data" URL: ${sourceURL}`);
-}
-
-/**
- * Fetch source content from file system
- * @param {InputFileSystem} fs file system
- * @param {string} sourceURL file URL
- * @returns {Promise<{path: string, data?: string}>} source content promise
- */
-async function fetchFromFilesystem(fs, sourceURL) {
-	let buffer;
-
-	if (isURL(sourceURL)) {
-		return { path: sourceURL };
-	}
-
-	try {
-		buffer = await new Promise((resolve, reject) => {
-			fs.readFile(
-				sourceURL,
-				(
-					/** @type {Error | null} */ error,
-					/** @type {Buffer<ArrayBufferLike> | undefined} */ data
-				) => {
-					if (error) {
-						return reject(error);
-					}
-
-					return resolve(data);
-				}
-			);
-		});
-	} catch (error) {
-		throw new Error(
-			`Failed to parse source map from '${sourceURL}' file: ${error}`
-		);
-	}
-
-	return { path: sourceURL, data: buffer.toString() };
-}
-
 /**
  * Fetch from multiple possible file paths
- * @param {InputFileSystem} fs file system
+ * @param {ReadResource} readResource read resource function
  * @param {string[]} possibleRequests array of possible file paths
  * @param {string} errorsAccumulator accumulated error messages
  * @returns {Promise<{path: string, data?: string}>} source content promise
  */
-async function fetchPathsFromFilesystem(
+async function fetchPathsFromURL(
-	fs,
+	readResource,
 	possibleRequests,
 	errorsAccumulator = ""
 ) {
 	let result;
 
 	try {
-		result = await fetchFromFilesystem(fs, possibleRequests[0]);
+		result = await readResource(possibleRequests[0]);
 	} catch (error) {
 		errorsAccumulator += `${/** @type {Error} */ (error).message}\n\n`;
 
@@ -179,63 +127,55 @@ async function fetchPathsFromFilesystem(
 			throw error;
 		}
 
-		return fetchPathsFromFilesystem(
+		return fetchPathsFromURL(
-			fs,
+			readResource,
 			tailPossibleRequests,
 			errorsAccumulator
 		);
 	}
 
-	return result;
+	return {
+		path: possibleRequests[0],
+		data: result.toString("utf8")
+	};
 }
 
 /**
  * Fetch source content from URL
- * @param {InputFileSystem} fs file system
+ * @param {ReadResource} readResource The read resource function
  * @param {string} context context directory
  * @param {string} url source URL
  * @param {string=} sourceRoot source root directory
  * @param {boolean=} skipReading whether to skip reading file content
- * @returns {Promise<{sourceURL: string, sourceContent?: string}>} source content promise
+ * @returns {Promise<{sourceURL: string, sourceContent?: string | Buffer<ArrayBufferLike>}>} source content promise
  */
-async function fetchFromURL(fs, context, url, sourceRoot, skipReading = false) {
+async function fetchFromURL(
+	readResource,
+	context,
+	url,
+	sourceRoot,
+	skipReading = false
+) {
 	// 1. It's an absolute url and it is not `windows` path like `C:\dir\file`
 	if (isURL(url)) {
 		// eslint-disable-next-line n/no-deprecated-api
 		const { protocol } = urlUtils.parse(url);
-
 		if (protocol === "data:") {
-			if (skipReading) {
+			const sourceContent = skipReading ? "" : await readResource(url);
-				return { sourceURL: "" };
-			}
-
-			const sourceContent = fetchFromDataURL(fs, url);
 
 			return { sourceURL: "", sourceContent };
 		}
 
-		if (skipReading) {
-			return { sourceURL: url };
-		}
-
 		if (protocol === "file:") {
 			const pathFromURL = urlUtils.fileURLToPath(url);
 			const sourceURL = path.normalize(pathFromURL);
-			const { data: sourceContent } = await fetchFromFilesystem(fs, sourceURL);
+			const sourceContent = skipReading ? "" : await readResource(sourceURL);
 
 			return { sourceURL, sourceContent };
 		}
 
-		throw new Error(
+		const sourceContent = skipReading ? "" : await readResource(url);
-			`Failed to parse source map: '${url}' URL is not supported`
+		return { sourceURL: url, sourceContent };
-		);
-	}
-
-	// 2. It's a scheme-relative
-	if (/^\/\//.test(url)) {
-		throw new Error(
-			`Failed to parse source map: '${url}' URL is not supported`
-		);
 	}
 
 	// 3. Absolute path
@@ -249,11 +189,11 @@ async function fetchFromURL(fs, context, url, sourceRoot, skipReading = false) {
 
 			if (url.startsWith("/")) {
 				possibleRequests.push(
-					getAbsolutePath(fs, context, sourceURL.slice(1), sourceRoot || "")
+					getAbsolutePath(context, sourceURL.slice(1), sourceRoot || "")
 				);
 			}
 
-			const result = await fetchPathsFromFilesystem(fs, possibleRequests);
+			const result = await fetchPathsFromURL(readResource, possibleRequests);
 
 			sourceURL = result.path;
 			sourceContent = result.data;
@@ -263,14 +203,11 @@ async function fetchFromURL(fs, context, url, sourceRoot, skipReading = false) {
 	}
 
 	// 4. Relative path
-	const sourceURL = getAbsolutePath(fs, context, url, sourceRoot || "");
+	const sourceURL = getAbsolutePath(context, url, sourceRoot || "");
-
 	let sourceContent;
 
 	if (!skipReading) {
-		const { data } = await fetchFromFilesystem(fs, sourceURL);
+		sourceContent = await readResource(sourceURL);
-
-		sourceContent = data;
 	}
 
 	return { sourceURL, sourceContent };
@@ -280,10 +217,10 @@ async function fetchFromURL(fs, context, url, sourceRoot, skipReading = false) {
  * Extract source map from code content
  * @param {string | Buffer<ArrayBufferLike>} stringOrBuffer The input code content as string or buffer
  * @param {string} resourcePath The path to the resource file
- * @param {InputFileSystem} fs The file system interface for reading files
+ * @param {ReadResource} readResource The read resource function
- * @returns {Promise<{source: string | Buffer<ArrayBufferLike>, sourceMap: string | RawSourceMap | undefined, fileDependencies: string[]}>} Promise resolving to extracted source map information
+ * @returns {Promise<{source: string | Buffer<ArrayBufferLike>, sourceMap: string | RawSourceMap | undefined}>} Promise resolving to extracted source map information
  */
-async function extractSourceMap(stringOrBuffer, resourcePath, fs) {
+async function extractSourceMap(stringOrBuffer, resourcePath, readResource) {
 	const input =
 		typeof stringOrBuffer === "string"
 			? stringOrBuffer
@@ -291,8 +228,7 @@ async function extractSourceMap(stringOrBuffer, resourcePath, fs) {
 	const inputSourceMap = undefined;
 	const output = {
 		source: stringOrBuffer,
-		sourceMap: inputSourceMap,
+		sourceMap: inputSourceMap
-		fileDependencies: /** @type {string[]} */ ([])
 	};
 	const { sourceMappingURL, replacementString } = getSourceMappingURL(input);
 
@@ -303,21 +239,19 @@ async function extractSourceMap(stringOrBuffer, resourcePath, fs) {
 	const baseContext = path.dirname(resourcePath);
 
 	const { sourceURL, sourceContent } = await fetchFromURL(
-		fs,
+		readResource,
 		baseContext,
 		sourceMappingURL
 	);
 
-	if (sourceURL) {
-		output.fileDependencies.push(/** @type {string} */ (sourceURL));
-	}
-
 	if (!sourceContent) {
 		return output;
 	}
 
 	/** @type {RawSourceMap} */
-	const map = JSON.parse(sourceContent.replace(/^\)\]\}'/, ""));
+	const map = JSON.parse(
+		sourceContent.toString("utf8").replace(/^\)\]\}'/, "")
+	);
 
 	const context = sourceURL ? path.dirname(sourceURL) : baseContext;
 
@@ -335,7 +269,7 @@ async function extractSourceMap(stringOrBuffer, resourcePath, fs) {
 				// This is necessary so that for sourceMaps with the same file structure in sources, name collisions do not occur.
 				// https://github.com/webpack-contrib/source-map-loader/issues/51
 				let { sourceURL, sourceContent } = await fetchFromURL(
-					fs,
+					readResource,
 					context,
 					source,
 					map.sourceRoot,
@@ -344,8 +278,6 @@ async function extractSourceMap(stringOrBuffer, resourcePath, fs) {
 
 				if (skipReading) {
 					sourceContent = originalSourceContent;
-				} else if (sourceURL && !isURL(sourceURL)) {
-					output.fileDependencies.push(/** @type {string} */ (sourceURL));
 				}
 
 				// Return original value of `source` when error happens
@@ -366,7 +298,9 @@ async function extractSourceMap(stringOrBuffer, resourcePath, fs) {
 		const { sourceURL, sourceContent } = source;
 
 		newMap.sources.push(sourceURL || "");
-		newMap.sourcesContent.push(sourceContent || "");
+		newMap.sourcesContent.push(
+			sourceContent ? sourceContent.toString("utf8") : ""
+		);
 	}
 
 	const sourcesContentIsEmpty =
@@ -378,8 +312,7 @@ async function extractSourceMap(stringOrBuffer, resourcePath, fs) {
 
 	return {
 		source: input.replace(replacementString, ""),
-		sourceMap: /** @type {RawSourceMap} */ (newMap),
+		sourceMap: /** @type {RawSourceMap} */ (newMap)
-		fileDependencies: output.fileDependencies
 	};
 }
 

test/configCases/source-map/extract-source-map/extract4.js

@@ -0,0 +1,12 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+require("./test4");
+
+it("should extract source map - 4", () => {
+	const fileData = fs.readFileSync(path.resolve(__dirname, "bundle3.js.map")).toString("utf-8");
+	const { sources } = JSON.parse(fileData);
+	expect(sources.includes("webpack:///antd/./components/button/index.tsx")).toBe(true);
+});

test/configCases/source-map/extract-source-map/lock-files/lock.json

@@ -0,0 +1,4 @@
+{
+  "https://cdnjs.cloudflare.com/ajax/libs/antd/5.27.3/antd.min.js.map": { "integrity": "sha512-dV8AVOy1aVA2CRfd5/ZKySqR8i/VcIZCfJbstL+kyuzioyu/652hEL+KRrqVutRx8y5CTwFskQVQ79rFy5VRLg==", "contentType": "application/octet-stream; charset=utf-8" },
+  "version": 1
+}

test/configCases/source-map/extract-source-map/test4.js

@@ -0,0 +1,3 @@
+const a = 1;
+// comment
+//#sourceMappingURL=https://cdnjs.cloudflare.com/ajax/libs/antd/5.27.3/antd.min.js.map

test/configCases/source-map/extract-source-map/webpack.config.js

@@ -1,5 +1,7 @@
 "use strict";
 
+const path = require("path");
+
 /** @type {import("../../../../").Configuration[]} */
 module.exports = [
 	{
@@ -38,6 +40,26 @@ module.exports = [
 			]
 		}
 	},
+	{
+		target: "node",
+		entry: "./extract4",
+		devtool: "source-map",
+		experiments: {
+			buildHttp: {
+				allowedUris: [() => true],
+				lockfileLocation: path.resolve(__dirname, "./lock-files/lock.json"),
+				cacheLocation: path.resolve(__dirname, "./lock-files/test"),
+				frozen: false
+			}
+		},
+		module: {
+			rules: [
+				{
+					extractSourceMap: true
+				}
+			]
+		}
+	},
 	{
 		entry: "./remove-comment",
 		devtool: "source-map",