Compare commits

...

33 Commits

Author SHA1 Message Date
Sampark Sharma 27f9a7d461
Revert Address review comments 2022-12-07 08:41:56 +00:00
Sampark Sharma d79a09bc0e
Address review comments 2022-12-07 08:27:08 +00:00
Sampark Sharma d1094e1523
Add test 2022-12-07 07:39:41 +00:00
Sampark Sharma c0085d7739
Fix test 2022-12-06 11:52:46 +00:00
Sampark Sharma 1dae855746
Add fallback to gzip compression if cache not found 2022-12-06 11:38:17 +00:00
Sampark Sharma 785599dd2e
Merge pull request #1237 from actions/pdotl/win-bsdtar-zstd
Bsdtar + zstd fallback to prevent zstd hanging issue on windows
2022-12-01 12:14:55 +05:30
Sampark Sharma 8595831452
Fix test 2022-11-30 10:37:32 +00:00
Sampark Sharma afbc5c0a9e
Add await to async function calls 2022-11-30 06:16:19 +00:00
Sampark Sharma 424ae62ee7
Fix tar test 2022-11-30 06:05:34 +00:00
Sampark Sharma 2e5a517460
Fix test 2022-11-29 12:16:17 +00:00
Sampark Sharma 34f0143be2
Address review comments 2022-11-29 10:35:22 +00:00
Sampark Sharma 0fd856d0a0
Refactor code 2022-11-28 10:24:40 +00:00
Sampark Sharma 0822441ee0
Fix lint test 2022-11-23 11:51:46 +00:00
Sampark Sharma 187781e273
Fix tests 2022-11-23 11:40:11 +00:00
Sampark Sharma 1f3371766a
Add new tests 2022-11-23 11:20:24 +00:00
Sampark Sharma 39b7a86b23
Fix old tests 2022-11-23 07:58:46 +00:00
Sampark Sharma 2f73afa843
Separate args 2022-11-23 07:39:15 +00:00
Sampark Sharma ffde3e4bd5
Fix test 2022-11-22 07:33:01 +00:00
Sampark Sharma 32f538163d
Fix windows test 2022-11-21 12:51:07 +00:00
Sampark Sharma 32b95825ba
Add windows bsdtar test 2022-11-21 12:19:44 +00:00
Sampark Sharma 54eb9b8055
Address some comments and correct compression commands 2022-11-21 12:05:03 +00:00
Lovepreet Singh f9dfb05bd2 Temporarily remove thhe condition that prevents zstd usage on windows unless with GNUtar 2022-11-17 12:13:49 +00:00
Lovepreet Singh b3bd482c0f Fix windows gnutar test case 2022-11-17 09:46:01 +00:00
Lovepreet Singh 4fa5b7d133 Fix lint issues 2022-11-17 09:12:53 +00:00
Lovepreet Singh ea9856079f Fix tar tests 2022-11-17 08:53:46 +00:00
Sampark Sharma caadeee090
Merge pull request #1239 from actions/phantsure/gnutar_windows
Add GNUtar as default on windows
2022-11-17 13:12:02 +05:30
Sampark Sharma 964682b5d4
Fix test 2022-11-17 07:34:07 +00:00
Sampark Sharma 6f7397feb6
Add GNUtar as default on windows 2022-11-17 07:26:52 +00:00
Lovepreet Singh 5f89653f1b Fix order of args for tar 2022-11-17 06:22:13 +00:00
Lovepreet Singh e9e146bbf8 Add -v option for testing 2022-11-16 21:04:25 +00:00
Lovepreet Singh 27bf8304ea Fix tar operations 2022-11-16 20:50:04 +00:00
Lovepreet Singh db3517fe3b bsd + zstd fallback implementation 2022-11-16 20:21:40 +00:00
Lovepreet Singh 6349c3ca3a bsd + zstd fallback implementation 2022-11-16 20:18:31 +00:00
7 changed files with 526 additions and 152 deletions

View File

@ -161,6 +161,81 @@ test('restore with gzip compressed cache found', async () => {
expect(getCompressionMock).toHaveBeenCalledTimes(1)
})
test('restore with zstd as default but gzip compressed cache found on windows', async () => {
if (process.platform === 'win32') {
const paths = ['node_modules']
const key = 'node-test'
const cacheEntry: ArtifactCacheEntry = {
cacheKey: key,
scope: 'refs/heads/main',
archiveLocation: 'www.actionscache.test/download'
}
const getCacheMock = jest.spyOn(cacheHttpClient, 'getCacheEntry')
getCacheMock
.mockImplementationOnce(async () => {
throw new Error('Cache not found.')
})
.mockImplementationOnce(async () => {
return Promise.resolve(cacheEntry)
})
const tempPath = '/foo/bar'
const createTempDirectoryMock = jest.spyOn(
cacheUtils,
'createTempDirectory'
)
createTempDirectoryMock.mockImplementation(async () => {
return Promise.resolve(tempPath)
})
const archivePath = path.join(tempPath, CacheFilename.Gzip)
const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')
const fileSize = 142
const getArchiveFileSizeInBytesMock = jest
.spyOn(cacheUtils, 'getArchiveFileSizeInBytes')
.mockReturnValue(fileSize)
const extractTarMock = jest.spyOn(tar, 'extractTar')
const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')
const compression = CompressionMethod.Zstd
const getCompressionMock = jest
.spyOn(cacheUtils, 'getCompressionMethod')
.mockReturnValue(Promise.resolve(compression))
const cacheKey = await restoreCache(paths, key)
expect(cacheKey).toBe(key)
expect(getCacheMock).toHaveBeenNthCalledWith(1, [key], paths, {
compressionMethod: compression
})
expect(getCacheMock).toHaveBeenNthCalledWith(2, [key], paths, {
compressionMethod: CompressionMethod.Gzip
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheMock).toHaveBeenCalledWith(
cacheEntry.archiveLocation,
archivePath,
undefined
)
expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
expect(extractTarMock).toHaveBeenCalledTimes(1)
expect(extractTarMock).toHaveBeenCalledWith(
archivePath,
CompressionMethod.Gzip
)
expect(unlinkFileMock).toHaveBeenCalledTimes(1)
expect(unlinkFileMock).toHaveBeenCalledWith(archivePath)
expect(getCompressionMock).toHaveBeenCalledTimes(1)
}
})
test('restore with zstd compressed cache found', async () => {
const paths = ['node_modules']
const key = 'node-test'

View File

@ -1,7 +1,14 @@
import * as exec from '@actions/exec'
import * as io from '@actions/io'
import * as path from 'path'
import {CacheFilename, CompressionMethod} from '../src/internal/constants'
import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows,
ManifestFilename,
SystemTarPathOnWindows,
TarFilename
} from '../src/internal/constants'
import * as tar from '../src/internal/tar'
import * as utils from '../src/internal/cacheUtils'
// eslint-disable-next-line @typescript-eslint/no-require-imports
@ -13,7 +20,7 @@ jest.mock('@actions/io')
const IS_WINDOWS = process.platform === 'win32'
const IS_MAC = process.platform === 'darwin'
const defaultTarPath = process.platform === 'darwin' ? 'gtar' : 'tar'
const defaultTarPath = IS_MAC ? 'gtar' : 'tar'
function getTempDir(): string {
return path.join(__dirname, '_temp', 'tar')
@ -28,6 +35,10 @@ beforeAll(async () => {
await jest.requireActual('@actions/io').rmRF(getTempDir())
})
beforeEach(async () => {
jest.restoreAllMocks()
})
afterAll(async () => {
delete process.env['GITHUB_WORKSPACE']
await jest.requireActual('@actions/io').rmRF(getTempDir())
@ -41,16 +52,15 @@ test('zstd extract tar', async () => {
? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
`"${tarPath}"`,
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
@ -58,11 +68,48 @@ test('zstd extract tar', async () => {
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' ')
)
})
test('zstd extract tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = SystemTarPathOnWindows
await tar.extractTar(archivePath, CompressionMethod.Zstd)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
'zstd -d --long=30 -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`"${tarPath}"`,
'-xf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/')
].join(' ')
)
}
})
test('gzip extract tar', async () => {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
@ -74,51 +121,48 @@ test('gzip extract tar', async () => {
await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(mkdirMock).toHaveBeenCalledWith(workspace)
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
'-z',
`"${tarPath}"`,
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' ')
)
})
test('gzip extract GNU tar on windows', async () => {
test('gzip extract GNU tar on windows with GNUtar in path', async () => {
if (IS_WINDOWS) {
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
const isGnuMock = jest
.spyOn(utils, 'isGnuTarInstalled')
.mockReturnValue(Promise.resolve(true))
// GNU tar present in path but not at default location
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve('tar'))
const execMock = jest.spyOn(exec, 'exec')
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
const workspace = process.env['GITHUB_WORKSPACE']
await tar.extractTar(archivePath, CompressionMethod.Gzip)
expect(isGnuMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"tar"`,
[
'-z',
`"tar"`,
'-xf',
archivePath.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--force-local'
],
{cwd: undefined}
'--force-local',
'-z'
].join(' ')
)
}
})
@ -134,13 +178,13 @@ test('zstd create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[
`"${tarPath}"`,
'--posix',
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30',
'-cf',
IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd,
'--exclude',
@ -149,16 +193,70 @@ test('zstd create tar', async () => {
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
'manifest.txt'
ManifestFilename
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
])
.join(' '),
undefined, // args
{
cwd: archiveFolder
}
)
})
test('zstd create tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
await fs.promises.mkdir(archiveFolder, {recursive: true})
await tar.createTar(
archiveFolder,
sourceDirectories,
CompressionMethod.Zstd
)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
`"${tarPath}"`,
'--posix',
'-cf',
TarFilename.replace(/\\/g, '/'),
'--exclude',
TarFilename.replace(/\\/g, '/'),
'-P',
'-C',
workspace?.replace(/\\/g, '/'),
'--files-from',
ManifestFilename,
'&&',
'zstd -T0 --long=30 -o',
CacheFilename.Zstd.replace(/\\/g, '/'),
TarFilename.replace(/\\/g, '/')
].join(' '),
undefined, // args
{
cwd: archiveFolder
}
)
}
})
test('gzip create tar', async () => {
const execMock = jest.spyOn(exec, 'exec')
@ -170,16 +268,13 @@ test('gzip create tar', async () => {
await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
`"${tarPath}"`,
'--posix',
'-z',
'-cf',
IS_WINDOWS ? CacheFilename.Gzip.replace(/\\/g, '/') : CacheFilename.Gzip,
'--exclude',
@ -188,8 +283,13 @@ test('gzip create tar', async () => {
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
'manifest.txt'
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
ManifestFilename
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' '),
undefined, // args
{
cwd: archiveFolder
}
@ -205,22 +305,52 @@ test('zstd list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30',
`"${tarPath}"`,
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat([
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
])
.join(' ')
)
})
test('zstd list tar with windows BSDtar', async () => {
if (IS_WINDOWS) {
const execMock = jest.spyOn(exec, 'exec')
jest
.spyOn(utils, 'getGnuTarPathOnWindows')
.mockReturnValue(Promise.resolve(''))
const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
await tar.listTar(archivePath, CompressionMethod.Zstd)
const tarPath = SystemTarPathOnWindows
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
'zstd -d --long=30 -o',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&',
`"${tarPath}"`,
'-tf',
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
].join(' ')
)
}
})
test('zstdWithoutLong list tar', async () => {
const execMock = jest.spyOn(exec, 'exec')
@ -230,19 +360,19 @@ test('zstdWithoutLong list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.ZstdWithoutLong)
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${defaultTarPath}"`,
[
'--use-compress-program',
IS_WINDOWS ? 'zstd -d' : 'unzstd',
`"${tarPath}"`,
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd'])
.join(' ')
)
})
@ -254,18 +384,18 @@ test('gzip list tar', async () => {
await tar.listTar(archivePath, CompressionMethod.Gzip)
const tarPath = IS_WINDOWS
? `${process.env['windir']}\\System32\\tar.exe`
: defaultTarPath
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
`"${tarPath}"`,
[
'-z',
`"${tarPath}"`,
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
].concat(IS_MAC ? ['--delay-directory-restore'] : []),
{cwd: undefined}
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.concat(['-z'])
.join(' ')
)
})

View File

@ -4,6 +4,8 @@ import * as utils from './internal/cacheUtils'
import * as cacheHttpClient from './internal/cacheHttpClient'
import {createTar, extractTar, listTar} from './internal/tar'
import {DownloadOptions, UploadOptions} from './options'
import {CompressionMethod} from './internal/constants'
import {ArtifactCacheEntry} from './internal/contracts'
export class ValidationError extends Error {
constructor(message: string) {
@ -85,19 +87,38 @@ export async function restoreCache(
checkKey(key)
}
const compressionMethod = await utils.getCompressionMethod()
let cacheEntry: ArtifactCacheEntry | null
let compressionMethod = await utils.getCompressionMethod()
let archivePath = ''
try {
// path are needed to compute version
const cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod
})
try {
// path are needed to compute version
cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod
})
} catch (error) {
// This is to support the old cache entry created
// by the old version of the cache action on windows.
if (
process.platform === 'win32' &&
compressionMethod !== CompressionMethod.Gzip
) {
compressionMethod = CompressionMethod.Gzip
cacheEntry = await cacheHttpClient.getCacheEntry(keys, paths, {
compressionMethod
})
if (!cacheEntry?.archiveLocation) {
throw error
}
} else {
throw error
}
}
if (!cacheEntry?.archiveLocation) {
// Cache not found
return undefined
}
archivePath = path.join(
await utils.createTempDirectory(),
utils.getCacheFileName(compressionMethod)

View File

@ -7,7 +7,11 @@ import * as path from 'path'
import * as semver from 'semver'
import * as util from 'util'
import {v4 as uuidV4} from 'uuid'
import {CacheFilename, CompressionMethod} from './constants'
import {
CacheFilename,
CompressionMethod,
GnuTarPathOnWindows
} from './constants'
// From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23
export async function createTempDirectory(): Promise<string> {
@ -90,11 +94,6 @@ async function getVersion(app: string): Promise<string> {
// Use zstandard if possible to maximize cache performance
export async function getCompressionMethod(): Promise<CompressionMethod> {
if (process.platform === 'win32' && !(await isGnuTarInstalled())) {
// Disable zstd due to bug https://github.com/actions/cache/issues/301
return CompressionMethod.Gzip
}
const versionOutput = await getVersion('zstd')
const version = semver.clean(versionOutput)
@ -116,9 +115,12 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string {
: CacheFilename.Zstd
}
export async function isGnuTarInstalled(): Promise<boolean> {
export async function getGnuTarPathOnWindows(): Promise<string> {
if (fs.existsSync(GnuTarPathOnWindows)) {
return GnuTarPathOnWindows
}
const versionOutput = await getVersion('tar')
return versionOutput.toLowerCase().includes('gnu tar')
return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : ''
}
export function assertDefined<T>(name: string, value?: T): T {

View File

@ -11,6 +11,11 @@ export enum CompressionMethod {
Zstd = 'zstd'
}
export enum ArchiveToolType {
GNU = 'gnu',
BSD = 'bsd'
}
// The default number of retry attempts.
export const DefaultRetryAttempts = 2
@ -21,3 +26,13 @@ export const DefaultRetryDelay = 5000
// over the socket during this period, the socket is destroyed and the download
// is aborted.
export const SocketTimeout = 5000
// The default path of GNUtar on hosted Windows runners
export const GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`
// The default path of BSDtar on hosted Windows runners
export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`
export const TarFilename = 'cache.tar'
export const ManifestFilename = 'manifest.txt'

View File

@ -31,3 +31,8 @@ export interface InternalCacheOptions {
compressionMethod?: CompressionMethod
cacheSize?: number
}
export interface ArchiveTool {
path: string
type: string
}

View File

@ -3,25 +3,28 @@ import * as io from '@actions/io'
import {existsSync, writeFileSync} from 'fs'
import * as path from 'path'
import * as utils from './cacheUtils'
import {CompressionMethod} from './constants'
import {ArchiveTool} from './contracts'
import {
CompressionMethod,
SystemTarPathOnWindows,
ArchiveToolType,
TarFilename,
ManifestFilename
} from './constants'
const IS_WINDOWS = process.platform === 'win32'
async function getTarPath(
args: string[],
compressionMethod: CompressionMethod
): Promise<string> {
// Function also mutates the args array. For non-mutation call with passing an empty array.
async function getTarPath(): Promise<ArchiveTool> {
switch (process.platform) {
case 'win32': {
const systemTar = `${process.env['windir']}\\System32\\tar.exe`
if (compressionMethod !== CompressionMethod.Gzip) {
// We only use zstandard compression on windows when gnu tar is installed due to
// a bug with compressing large files with bsdtar + zstd
args.push('--force-local')
const gnuTar = await utils.getGnuTarPathOnWindows()
const systemTar = SystemTarPathOnWindows
if (gnuTar) {
// Use GNUtar as default on windows
return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
} else if (existsSync(systemTar)) {
return systemTar
} else if (await utils.isGnuTarInstalled()) {
args.push('--force-local')
return <ArchiveTool>{path: systemTar, type: ArchiveToolType.BSD}
}
break
}
@ -29,26 +32,120 @@ async function getTarPath(
const gnuTar = await io.which('gtar', false)
if (gnuTar) {
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
args.push('--delay-directory-restore')
return gnuTar
return <ArchiveTool>{path: gnuTar, type: ArchiveToolType.GNU}
} else {
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.BSD
}
}
break
}
default:
break
}
return await io.which('tar', true)
return <ArchiveTool>{
path: await io.which('tar', true),
type: ArchiveToolType.GNU
}
}
async function execTar(
args: string[],
// Return arguments for tar as per tarPath, compressionMethod, method type and os
async function getTarArgs(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod,
cwd?: string
): Promise<void> {
try {
await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {cwd})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
type: string,
archivePath = ''
): Promise<string[]> {
const args = [`"${tarPath.path}"`]
const cacheFileName = utils.getCacheFileName(compressionMethod)
const tarFile = 'cache.tar'
const workingDirectory = getWorkingDirectory()
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
// Method specific args
switch (type) {
case 'create':
args.push(
'--posix',
'-cf',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
BSD_TAR_ZSTD
? tarFile
: cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
ManifestFilename
)
break
case 'extract':
args.push(
'-xf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
)
break
case 'list':
args.push(
'-tf',
BSD_TAR_ZSTD
? tarFile
: archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
)
break
}
// Platform specific args
if (tarPath.type === ArchiveToolType.GNU) {
switch (process.platform) {
case 'win32':
args.push('--force-local')
break
case 'darwin':
args.push('--delay-directory-restore')
break
}
}
return args
}
async function getArgs(
compressionMethod: CompressionMethod,
type: string,
archivePath = ''
): Promise<string> {
const tarPath = await getTarPath()
const tarArgs = await getTarArgs(
tarPath,
compressionMethod,
type,
archivePath
)
const compressionArgs =
type !== 'create'
? await getDecompressionProgram(tarPath, compressionMethod, archivePath)
: await getCompressionProgram(tarPath, compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
if (BSD_TAR_ZSTD && type !== 'create') {
return [...compressionArgs, ...tarArgs].join(' ')
} else {
return [...tarArgs, ...compressionArgs].join(' ')
}
}
@ -57,19 +154,82 @@ function getWorkingDirectory(): string {
}
// Common function for extractTar and listTar to get the compression method
function getCompressionProgram(compressionMethod: CompressionMethod): string[] {
async function getDecompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod,
archivePath: string
): Promise<string[]> {
// -d: Decompress.
// unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) {
case CompressionMethod.Zstd:
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
]
return BSD_TAR_ZSTD
? [
'zstd -d --long=30 -o',
TarFilename,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']
return BSD_TAR_ZSTD
? [
'zstd -d -o',
TarFilename,
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'&&'
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']
default:
return ['-z']
}
}
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
async function getCompressionProgram(
tarPath: ArchiveTool,
compressionMethod: CompressionMethod
): Promise<string[]> {
const cacheFileName = utils.getCacheFileName(compressionMethod)
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
IS_WINDOWS
switch (compressionMethod) {
case CompressionMethod.Zstd:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 --long=30 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: [
'--use-compress-program',
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return BSD_TAR_ZSTD
? [
'&&',
'zstd -T0 -o',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
TarFilename
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']
default:
return ['-z']
}
@ -79,13 +239,12 @@ export async function listTar(
archivePath: string,
compressionMethod: CompressionMethod
): Promise<void> {
const args = [
...getCompressionProgram(compressionMethod),
'-tf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
]
await execTar(args, compressionMethod)
const args = await getArgs(compressionMethod, 'list', archivePath)
try {
await exec(args)
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
}
export async function extractTar(
@ -95,15 +254,12 @@ export async function extractTar(
// Create directory to extract tar into
const workingDirectory = getWorkingDirectory()
await io.mkdirP(workingDirectory)
const args = [
...getCompressionProgram(compressionMethod),
'-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
await execTar(args, compressionMethod)
const args = await getArgs(compressionMethod, 'extract', archivePath)
try {
await exec(args)
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
}
export async function createTar(
@ -112,44 +268,14 @@ export async function createTar(
compressionMethod: CompressionMethod
): Promise<void> {
// Write source directories to manifest.txt to avoid command length limits
const manifestFilename = 'manifest.txt'
const cacheFileName = utils.getCacheFileName(compressionMethod)
writeFileSync(
path.join(archiveFolder, manifestFilename),
path.join(archiveFolder, ManifestFilename),
sourceDirectories.join('\n')
)
const workingDirectory = getWorkingDirectory()
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram(): string[] {
switch (compressionMethod) {
case CompressionMethod.Zstd:
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
]
case CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']
default:
return ['-z']
}
const args = await getArgs(compressionMethod, 'create')
try {
await exec(args, undefined, {cwd: archiveFolder})
} catch (error) {
throw new Error(`Tar failed with error: ${error?.message}`)
}
const args = [
'--posix',
...getCompressionProgram(),
'-cf',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--exclude',
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P',
'-C',
workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'--files-from',
manifestFilename
]
await execTar(args, compressionMethod, archiveFolder)
}