buildah/push/manifest-push: add support for --force-compression
Adds support for `--force-compression` which allows end-users to force push blobs with the selected compresison in `--compression` option, in order to make sure that `blobs` of other compression on registry are not reused. Is equivalent to: `force-compression` here: https://docs.docker.com/build/exporters/#compression Closes: https://github.com/containers/buildah/issues/4613 Also Implements: `--compression-format` and `--compression-level` for `manifest push` just like `podman`'s `manifest push` Signed-off-by: Aditya R <arajan@redhat.com>
This commit is contained in:
Aditya R
parent
3445a775a0
commit
d68d9a237c
|
|
@ -17,6 +17,7 @@ import (
|
|||
"github.com/containers/common/pkg/auth"
|
||||
cp "github.com/containers/image/v5/copy"
|
||||
"github.com/containers/image/v5/manifest"
|
||||
"github.com/containers/image/v5/pkg/compression"
|
||||
"github.com/containers/image/v5/transports"
|
||||
"github.com/containers/image/v5/transports/alltransports"
|
||||
"github.com/containers/image/v5/types"
|
||||
|
|
@ -226,6 +227,9 @@ func init() {
|
|||
flags.StringVar(&manifestPushOpts.certDir, "cert-dir", "", "use certificates at the specified path to access the registry")
|
||||
flags.StringVar(&manifestPushOpts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
|
||||
flags.StringVar(&manifestPushOpts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting digest to the file")
|
||||
flags.BoolVarP(&manifestPushOpts.forceCompressionFormat, "force-compression", "", false, "use the specified compression algorithm if the destination contains a differently-compressed variant already")
|
||||
flags.StringVar(&manifestPushOpts.compressionFormat, "compression-format", "", "compression format to use")
|
||||
flags.IntVar(&manifestPushOpts.compressionLevel, "compression-level", 0, "compression level to use")
|
||||
flags.StringVarP(&manifestPushOpts.format, "format", "f", "", "manifest type (oci or v2s2) to attempt to use when pushing the manifest list (default is manifest type of source)")
|
||||
flags.StringSliceVar(&manifestPushOpts.addCompression, "add-compression", nil, "add instances with selected compression while pushing")
|
||||
flags.BoolVarP(&manifestPushOpts.removeSignatures, "remove-signatures", "", false, "don't copy signatures when pushing images")
|
||||
|
|
@ -864,6 +868,16 @@ func manifestPushCmd(c *cobra.Command, args []string, opts pushOptions) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("building system context: %w", err)
|
||||
}
|
||||
if opts.compressionFormat != "" {
|
||||
algo, err := compression.AlgorithmByName(opts.compressionFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
systemContext.CompressionFormat = &algo
|
||||
}
|
||||
if c.Flag("compression-level").Changed {
|
||||
systemContext.CompressionLevel = &opts.compressionLevel
|
||||
}
|
||||
|
||||
return manifestPush(systemContext, store, listImageSpec, destSpec, opts)
|
||||
}
|
||||
|
|
@ -902,14 +916,15 @@ func manifestPush(systemContext *types.SystemContext, store storage.Store, listI
|
|||
}
|
||||
|
||||
options := manifests.PushOptions{
|
||||
Store: store,
|
||||
SystemContext: systemContext,
|
||||
ImageListSelection: cp.CopySpecificImages,
|
||||
Instances: nil,
|
||||
RemoveSignatures: opts.removeSignatures,
|
||||
SignBy: opts.signBy,
|
||||
ManifestType: manifestType,
|
||||
AddCompression: opts.addCompression,
|
||||
Store: store,
|
||||
SystemContext: systemContext,
|
||||
ImageListSelection: cp.CopySpecificImages,
|
||||
Instances: nil,
|
||||
RemoveSignatures: opts.removeSignatures,
|
||||
SignBy: opts.signBy,
|
||||
ManifestType: manifestType,
|
||||
AddCompression: opts.addCompression,
|
||||
ForceCompressionFormat: opts.forceCompressionFormat,
|
||||
}
|
||||
if opts.all {
|
||||
options.ImageListSelection = cp.CopyAllImages
|
||||
|
|
|
|||
|
|
@ -25,28 +25,29 @@ import (
|
|||
)
|
||||
|
||||
type pushOptions struct {
|
||||
all bool
|
||||
authfile string
|
||||
blobCache string
|
||||
certDir string
|
||||
creds string
|
||||
digestfile string
|
||||
disableCompression bool
|
||||
format string
|
||||
compressionFormat string
|
||||
compressionLevel int
|
||||
retry int
|
||||
retryDelay string
|
||||
rm bool
|
||||
quiet bool
|
||||
removeSignatures bool
|
||||
signaturePolicy string
|
||||
signBy string
|
||||
tlsVerify bool
|
||||
encryptionKeys []string
|
||||
encryptLayers []int
|
||||
insecure bool
|
||||
addCompression []string
|
||||
all bool
|
||||
authfile string
|
||||
blobCache string
|
||||
certDir string
|
||||
creds string
|
||||
digestfile string
|
||||
disableCompression bool
|
||||
format string
|
||||
compressionFormat string
|
||||
compressionLevel int
|
||||
forceCompressionFormat bool
|
||||
retry int
|
||||
retryDelay string
|
||||
rm bool
|
||||
quiet bool
|
||||
removeSignatures bool
|
||||
signaturePolicy string
|
||||
signBy string
|
||||
tlsVerify bool
|
||||
encryptionKeys []string
|
||||
encryptLayers []int
|
||||
insecure bool
|
||||
addCompression []string
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
|
@ -86,6 +87,7 @@ func init() {
|
|||
flags.StringVar(&opts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
|
||||
flags.StringVar(&opts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting image to the file")
|
||||
flags.BoolVarP(&opts.disableCompression, "disable-compression", "D", false, "don't compress layers")
|
||||
flags.BoolVarP(&opts.forceCompressionFormat, "force-compression", "", false, "use the specified compression algorithm if the destination contains a differently-compressed variant already")
|
||||
flags.StringVarP(&opts.format, "format", "f", "", "manifest type (oci, v2s1, or v2s2) to use in the destination (default is manifest type of source, with fallbacks)")
|
||||
flags.StringVar(&opts.compressionFormat, "compression-format", "", "compression format to use")
|
||||
flags.IntVar(&opts.compressionLevel, "compression-level", 0, "compression level to use")
|
||||
|
|
@ -199,18 +201,19 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error {
|
|||
}
|
||||
|
||||
options := buildah.PushOptions{
|
||||
Compression: compress,
|
||||
ManifestType: manifestType,
|
||||
SignaturePolicyPath: iopts.signaturePolicy,
|
||||
Store: store,
|
||||
SystemContext: systemContext,
|
||||
BlobDirectory: iopts.blobCache,
|
||||
RemoveSignatures: iopts.removeSignatures,
|
||||
SignBy: iopts.signBy,
|
||||
MaxRetries: iopts.retry,
|
||||
RetryDelay: pullPushRetryDelay,
|
||||
OciEncryptConfig: encConfig,
|
||||
OciEncryptLayers: encLayers,
|
||||
Compression: compress,
|
||||
ManifestType: manifestType,
|
||||
SignaturePolicyPath: iopts.signaturePolicy,
|
||||
Store: store,
|
||||
SystemContext: systemContext,
|
||||
BlobDirectory: iopts.blobCache,
|
||||
RemoveSignatures: iopts.removeSignatures,
|
||||
SignBy: iopts.signBy,
|
||||
MaxRetries: iopts.retry,
|
||||
RetryDelay: pullPushRetryDelay,
|
||||
OciEncryptConfig: encConfig,
|
||||
OciEncryptLayers: encLayers,
|
||||
ForceCompressionFormat: iopts.forceCompressionFormat,
|
||||
}
|
||||
if !iopts.quiet {
|
||||
options.ReportWriter = os.Stderr
|
||||
|
|
|
|||
|
|
@ -42,6 +42,16 @@ If the authorization state is not found there, $HOME/.docker/config.json is chec
|
|||
Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
|
||||
The default certificates directory is _/etc/containers/certs.d_.
|
||||
|
||||
**--compression-format** *format*
|
||||
|
||||
Specifies the compression format to use. Supported values are: `gzip`, `zstd` and `zstd:chunked`.
|
||||
|
||||
**--compression-level** *level*
|
||||
|
||||
Specify the compression level used with the compression.
|
||||
|
||||
Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive).
|
||||
|
||||
**--creds** *creds*
|
||||
|
||||
The [username[:password]] to use to authenticate with the registry if required.
|
||||
|
|
@ -52,6 +62,10 @@ value can be entered. The password is entered without echo.
|
|||
|
||||
After copying the image, write the digest of the resulting image to the file.
|
||||
|
||||
**--force-compression**
|
||||
|
||||
Use the specified compression algorithm even if the destination contains a differently-compressed variant already.
|
||||
|
||||
**--format**, **-f**
|
||||
|
||||
Manifest list type (oci or v2s2) to use when pushing the list (default is oci).
|
||||
|
|
|
|||
|
|
@ -70,6 +70,10 @@ Layer(s) to encrypt: 0-indexed layer indices with support for negative indexing
|
|||
|
||||
The [protocol:keyfile] specifies the encryption protocol, which can be JWE (RFC7516), PGP (RFC4880), and PKCS7 (RFC2315) and the key material required for image encryption. For instance, jwe:/path/to/key.pem or pgp:admin@example.com or pkcs7:/path/to/x509-file.
|
||||
|
||||
**--force-compression**
|
||||
|
||||
Use the specified compression algorithm even if the destination contains a differently-compressed variant already.
|
||||
|
||||
**--format**, **-f**
|
||||
|
||||
Manifest Type (oci, v2s2, or v2s1) to use when pushing an image. (default is manifest type of the source image, with fallbacks)
|
||||
|
|
|
|||
5
push.go
5
push.go
|
|
@ -95,6 +95,10 @@ type PushOptions struct {
|
|||
CompressionFormat *compression.Algorithm
|
||||
// CompressionLevel specifies what compression level is used
|
||||
CompressionLevel *int
|
||||
// ForceCompressionFormat ensures that the compression algorithm set in
|
||||
// CompressionFormat is used exclusively, and blobs of other compression
|
||||
// algorithms are not reused.
|
||||
ForceCompressionFormat bool
|
||||
}
|
||||
|
||||
// Push copies the contents of the image to a new location.
|
||||
|
|
@ -110,6 +114,7 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options
|
|||
libimageOptions.OciEncryptLayers = options.OciEncryptLayers
|
||||
libimageOptions.CompressionFormat = options.CompressionFormat
|
||||
libimageOptions.CompressionLevel = options.CompressionLevel
|
||||
libimageOptions.ForceCompressionFormat = options.ForceCompressionFormat
|
||||
libimageOptions.PolicyAllowStorage = true
|
||||
|
||||
if options.Quiet {
|
||||
|
|
|
|||
|
|
@ -39,6 +39,72 @@ _EOF
|
|||
validate_instance_compression "3" "$list" "arm64" "zstd"
|
||||
}
|
||||
|
||||
@test "bud: build manifest list with --add-compression zstd, --compression and --force-compression" {
|
||||
local contextdir=${TEST_SCRATCH_DIR}/bud/platform
|
||||
mkdir -p $contextdir
|
||||
|
||||
cat > $contextdir/Dockerfile1 << _EOF
|
||||
FROM alpine
|
||||
_EOF
|
||||
|
||||
start_registry
|
||||
run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT}
|
||||
run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1
|
||||
run_buildah build $WITH_POLICY_JSON -t image2 --platform linux/arm64 -f $contextdir/Dockerfile1
|
||||
|
||||
run_buildah manifest create foo
|
||||
run_buildah manifest add foo image1
|
||||
run_buildah manifest add foo image2
|
||||
|
||||
run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list
|
||||
|
||||
run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list
|
||||
list="$output"
|
||||
|
||||
validate_instance_compression "0" "$list" "amd64" "gzip"
|
||||
validate_instance_compression "1" "$list" "arm64" "gzip"
|
||||
validate_instance_compression "2" "$list" "amd64" "zstd"
|
||||
validate_instance_compression "3" "$list" "arm64" "zstd"
|
||||
|
||||
# Pushing again should keep every thing intact if original compression is `gzip` and `--force-compression` is specified
|
||||
run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --compression-format gzip --force-compression --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list
|
||||
|
||||
run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list
|
||||
list="$output"
|
||||
|
||||
validate_instance_compression "0" "$list" "amd64" "gzip"
|
||||
validate_instance_compression "1" "$list" "arm64" "gzip"
|
||||
validate_instance_compression "2" "$list" "amd64" "zstd"
|
||||
validate_instance_compression "3" "$list" "arm64" "zstd"
|
||||
}
|
||||
|
||||
@test "bud: build push with --force-compression" {
|
||||
skip_if_no_podman
|
||||
local contextdir=${TEST_SCRATCH_DIR}/bud/platform
|
||||
mkdir -p $contextdir
|
||||
|
||||
cat > $contextdir/Dockerfile1 << _EOF
|
||||
FROM alpine
|
||||
_EOF
|
||||
|
||||
start_registry
|
||||
run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT}
|
||||
run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1
|
||||
|
||||
run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format gzip image1 docker://localhost:${REGISTRY_PORT}/image
|
||||
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
|
||||
# layers should have no trace of zstd since push was with --compression-format gzip
|
||||
assert "$output" !~ "zstd" "zstd found in layers where push was with --compression-format gzip"
|
||||
run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format zstd image1 docker://localhost:${REGISTRY_PORT}/image
|
||||
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
|
||||
# layers should have no trace of zstd since push is without --force-compression
|
||||
assert "$output" !~ "zstd" "zstd found even though push was without --force-compression"
|
||||
run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format zstd --force-compression image1 docker://localhost:${REGISTRY_PORT}/image
|
||||
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
|
||||
# layers should container `zstd`
|
||||
expect_output --substring "zstd" "layers must contain zstd compression"
|
||||
}
|
||||
|
||||
@test "bud with --dns* flags" {
|
||||
_prefetch alpine
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue