chore(build): trim old transformed source files when generating new ones (#37363)

packages/playwright/src/transform/compilationCache.ts

@@ -18,6 +18,7 @@ import fs from 'fs';
 import os from 'os';
 import path from 'path';
 
+import { calculateSha1 } from 'playwright-core/lib/utils';
 import { isWorkerProcess } from '../common/globals';
 import { sourceMapSupport } from '../utilsBundle';
 
@@ -104,7 +105,7 @@ type CompilationCacheLookupResult = {
   addToCache?: (code: string, map: any | undefined | null, data: Map<string, any>) => { serializedCache?: any };
 };
 
-export function getFromCompilationCache(filename: string, hash: string, moduleUrl?: string): CompilationCacheLookupResult {
+export function getFromCompilationCache(filename: string, contentHash: string, moduleUrl?: string): CompilationCacheLookupResult {
   // First check the memory cache by filename, this cache will always work in the worker,
   // because we just compiled this file in the loader.
   const cache = memoryCache.get(filename);
@@ -117,7 +118,10 @@ export function getFromCompilationCache(filename: string, hash: string, moduleUr
   }
 
   // Then do the disk cache, this cache works between the Playwright Test runs.
-  const cachePath = calculateCachePath(filename, hash);
+  const filePathHash = calculateFilePathHash(filename);
+  const hashPrefix = filePathHash + '_' + contentHash.substring(0, 7);
+  const cacheFolderName = filePathHash.substring(0, 2);
+  const cachePath = calculateCachePath(filename, cacheFolderName, hashPrefix);
   const codePath = cachePath + '.js';
   const sourceMapPath = cachePath + '.map';
   const dataPath = cachePath + '.data';
@@ -132,6 +136,8 @@ export function getFromCompilationCache(filename: string, hash: string, moduleUr
     addToCache: (code: string, map: any | undefined | null, data: Map<string, any>) => {
       if (isWorkerProcess())
         return {};
+      // Trim cache. This won't help with deleted files, but it will remove storing multiple copies of the same file
+      clearOldCacheEntries(cacheFolderName, filePathHash);
       fs.mkdirSync(path.dirname(cachePath), { recursive: true });
       if (map)
         fs.writeFileSync(sourceMapPath, JSON.stringify(map), 'utf8');
@@ -168,9 +174,24 @@ export function addToCompilationCache(payload: SerializedCompilationCache) {
   }
 }
 
-function calculateCachePath(filePath: string, hash: string): string {
-  const fileName = path.basename(filePath, path.extname(filePath)).replace(/\W/g, '') + '_' + hash;
-  return path.join(cacheDir, hash[0] + hash[1], fileName);
+function calculateFilePathHash(filePath: string): string {
+  // Larger file path hash allows for fewer collisions compared to content, as we only check file path collision for deleting files
+  return calculateSha1(filePath).substring(0, 10);
+}
+
+function calculateCachePath(filePath: string, cacheFolderName: string, hashPrefix: string): string {
+  const fileName = hashPrefix + '_' + path.basename(filePath, path.extname(filePath)).replace(/\W/g, '');
+  return path.join(cacheDir, cacheFolderName, fileName);
+}
+
+function clearOldCacheEntries(cacheFolderName: string, filePathHash: string) {
+  const cachePath = path.join(cacheDir, cacheFolderName);
+  try {
+    const cachedRelevantFiles = fs.readdirSync(cachePath).filter(file => file.startsWith(filePathHash));
+    for (const file of cachedRelevantFiles)
+      fs.rmSync(path.join(cachePath, file), { force: true });
+  } catch {
+  }
 }
 
 // Since ESM and CJS collect dependencies differently,

tests/playwright-test/cache.spec.ts

@@ -0,0 +1,103 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should clear cache with type:module', async ({ runCLICommand }) => {
+  const result = await runCLICommand({
+    'playwright.config.ts': `
+      import { defineConfig } from '@playwright/test';
+      export default defineConfig({});
+    `,
+    'package.json': `
+      { "type": "module" }
+    `,
+    'a.spec.ts': `
+      import { test } from '@playwright/test';
+      test('example', () => {});
+    `,
+  }, 'clear-cache');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should clear cache for ct', async ({ runCLICommand }) => {
+  const result = await runCLICommand({
+    'playwright.config.ts': `
+      import { defineConfig } from '@playwright/test';
+      export default defineConfig({});
+    `,
+    'a.spec.ts': `
+      import { test } from '@playwright/test';
+      test('example', () => {});
+    `,
+  }, 'clear-cache', []);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should automatically clean cached versions of a changed file', async ({ runInlineTest, writeFiles }) => {
+  const cacheDir = test.info().outputPath('playwright-test-cache');
+  await runInlineTest({
+    'a.spec.ts': `
+      import { test } from '@playwright/test';
+      test('example', () => {});
+    `,
+  }, undefined, {
+    PWTEST_CACHE_DIR: cacheDir
+  });
+
+  const cacheDirectories = await fs.promises.readdir(cacheDir);
+  expect(cacheDirectories).toHaveLength(1);
+  const testCacheDirectory = path.join(cacheDir, cacheDirectories[0]);
+
+  const matchRegex = (extension: string) => new RegExp('^([0-9a-f]{10})_([0-9a-f]{7})_aspec\\.' + extension + '$', 'i');
+
+  let cachedFiles = await fs.promises.readdir(testCacheDirectory);
+  cachedFiles.sort();
+  expect(cachedFiles).toHaveLength(2);
+  expect(cachedFiles[0]).toMatch(matchRegex('js'));
+  expect(cachedFiles[1]).toMatch(matchRegex('map'));
+
+  const initialMatches = cachedFiles[0].match(matchRegex('js'));
+  expect(initialMatches).toHaveLength(3);
+  const firstPathHash = initialMatches[1];
+  const firstTestHash = initialMatches[2];
+
+  await runInlineTest({
+    'a.spec.ts': `
+      import { test } from '@playwright/test';
+      test('modified test', () => {});
+    `,
+  }, undefined, {
+    PWTEST_CACHE_DIR: cacheDir
+  });
+
+  cachedFiles = await fs.promises.readdir(testCacheDirectory);
+  cachedFiles.sort();
+  expect(cachedFiles).toHaveLength(2);
+  expect(cachedFiles[0]).toMatch(matchRegex('js'));
+  expect(cachedFiles[1]).toMatch(matchRegex('map'));
+
+  const finalMatches = cachedFiles[0].match(matchRegex('js'));
+  expect(finalMatches).toHaveLength(3);
+  const finalPathHash = finalMatches[1];
+  const finalTestHash = finalMatches[2];
+
+  expect(finalPathHash).toBe(firstPathHash);
+  expect(finalTestHash).not.toBe(firstTestHash);
+});

tests/playwright-test/clear-cache.spec.ts

@@ -1,48 +0,0 @@
-/**
- * Copyright Microsoft Corporation. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { test, expect } from './playwright-test-fixtures';
-
-test('should clear cache with type:module', async ({ runCLICommand }) => {
-  const result = await runCLICommand({
-    'playwright.config.ts': `
-      import { defineConfig } from '@playwright/test';
-      export default defineConfig({});
-    `,
-    'package.json': `
-      { "type": "module" }
-    `,
-    'a.spec.ts': `
-      import { test } from '@playwright/test';
-      test('example', () => {});
-    `,
-  }, 'clear-cache');
-  expect(result.exitCode).toBe(0);
-});
-
-test('should clear cache for ct', async ({ runCLICommand }) => {
-  const result = await runCLICommand({
-    'playwright.config.ts': `
-      import { defineConfig } from '@playwright/test';
-      export default defineConfig({});
-    `,
-    'a.spec.ts': `
-      import { test } from '@playwright/test';
-      test('example', () => {});
-    `,
-  }, 'clear-cache', []);
-  expect(result.exitCode).toBe(0);
-});

tests/playwright-test/playwright-test-fixtures.ts

@@ -281,7 +281,7 @@ export const test = base
         const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-'));
         await use(async (files: Files, params: Params = {}, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => {
           const baseDir = await writeFiles(testInfo, files, true);
-          return await runPlaywrightTest(childProcess, baseDir, params, { ...env, PWTEST_CACHE_DIR: cacheDir }, options, files, mergeReports, useIntermediateMergeReport);
+          return await runPlaywrightTest(childProcess, baseDir, params, { PWTEST_CACHE_DIR: cacheDir, ...env }, options, files, mergeReports, useIntermediateMergeReport);
         });
         await removeFolders([cacheDir]);
       },
@@ -313,7 +313,7 @@ export const test = base
         let testProcess: TestChildProcess | undefined;
         await use(async (files: Files, params: Params = {}, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => {
           const baseDir = await writeFiles(testInfo, files, true);
-          testProcess = startPlaywrightTest(childProcess, baseDir, params, { ...env, PWTEST_CACHE_DIR: cacheDir }, options);
+          testProcess = startPlaywrightTest(childProcess, baseDir, params, { PWTEST_CACHE_DIR: cacheDir, ...env }, options);
           return testProcess;
         });
         await testProcess?.kill();