From d68d9a237ce230038c5a2bb65703fb0515da23c8 Mon Sep 17 00:00:00 2001 From: Aditya R Date: Fri, 11 Aug 2023 23:47:11 +0530 Subject: [PATCH] buildah/push/manifest-push: add support for --force-compression Adds support for `--force-compression` which allows end-users to force push blobs with the selected compresison in `--compression` option, in order to make sure that `blobs` of other compression on registry are not reused. Is equivalent to: `force-compression` here: https://docs.docker.com/build/exporters/#compression Closes: https://github.com/containers/buildah/issues/4613 Also Implements: `--compression-format` and `--compression-level` for `manifest push` just like `podman`'s `manifest push` Signed-off-by: Aditya R --- cmd/buildah/manifest.go | 31 ++++++++++---- cmd/buildah/push.go | 71 +++++++++++++++++---------------- docs/buildah-manifest-push.1.md | 14 +++++++ docs/buildah-push.1.md | 4 ++ push.go | 5 +++ tests/bud.bats | 66 ++++++++++++++++++++++++++++++ 6 files changed, 149 insertions(+), 42 deletions(-) diff --git a/cmd/buildah/manifest.go b/cmd/buildah/manifest.go index bb73cacd7..8476a882c 100644 --- a/cmd/buildah/manifest.go +++ b/cmd/buildah/manifest.go @@ -17,6 +17,7 @@ import ( "github.com/containers/common/pkg/auth" cp "github.com/containers/image/v5/copy" "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/pkg/compression" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" @@ -226,6 +227,9 @@ func init() { flags.StringVar(&manifestPushOpts.certDir, "cert-dir", "", "use certificates at the specified path to access the registry") flags.StringVar(&manifestPushOpts.creds, "creds", "", "use `[username[:password]]` for accessing the registry") flags.StringVar(&manifestPushOpts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting digest to the file") + flags.BoolVarP(&manifestPushOpts.forceCompressionFormat, "force-compression", "", false, "use the specified compression algorithm if the destination contains a differently-compressed variant already") + flags.StringVar(&manifestPushOpts.compressionFormat, "compression-format", "", "compression format to use") + flags.IntVar(&manifestPushOpts.compressionLevel, "compression-level", 0, "compression level to use") flags.StringVarP(&manifestPushOpts.format, "format", "f", "", "manifest type (oci or v2s2) to attempt to use when pushing the manifest list (default is manifest type of source)") flags.StringSliceVar(&manifestPushOpts.addCompression, "add-compression", nil, "add instances with selected compression while pushing") flags.BoolVarP(&manifestPushOpts.removeSignatures, "remove-signatures", "", false, "don't copy signatures when pushing images") @@ -864,6 +868,16 @@ func manifestPushCmd(c *cobra.Command, args []string, opts pushOptions) error { if err != nil { return fmt.Errorf("building system context: %w", err) } + if opts.compressionFormat != "" { + algo, err := compression.AlgorithmByName(opts.compressionFormat) + if err != nil { + return err + } + systemContext.CompressionFormat = &algo + } + if c.Flag("compression-level").Changed { + systemContext.CompressionLevel = &opts.compressionLevel + } return manifestPush(systemContext, store, listImageSpec, destSpec, opts) } @@ -902,14 +916,15 @@ func manifestPush(systemContext *types.SystemContext, store storage.Store, listI } options := manifests.PushOptions{ - Store: store, - SystemContext: systemContext, - ImageListSelection: cp.CopySpecificImages, - Instances: nil, - RemoveSignatures: opts.removeSignatures, - SignBy: opts.signBy, - ManifestType: manifestType, - AddCompression: opts.addCompression, + Store: store, + SystemContext: systemContext, + ImageListSelection: cp.CopySpecificImages, + Instances: nil, + RemoveSignatures: opts.removeSignatures, + SignBy: opts.signBy, + ManifestType: manifestType, + AddCompression: opts.addCompression, + ForceCompressionFormat: opts.forceCompressionFormat, } if opts.all { options.ImageListSelection = cp.CopyAllImages diff --git a/cmd/buildah/push.go b/cmd/buildah/push.go index 4fc815576..f3d535a3a 100644 --- a/cmd/buildah/push.go +++ b/cmd/buildah/push.go @@ -25,28 +25,29 @@ import ( ) type pushOptions struct { - all bool - authfile string - blobCache string - certDir string - creds string - digestfile string - disableCompression bool - format string - compressionFormat string - compressionLevel int - retry int - retryDelay string - rm bool - quiet bool - removeSignatures bool - signaturePolicy string - signBy string - tlsVerify bool - encryptionKeys []string - encryptLayers []int - insecure bool - addCompression []string + all bool + authfile string + blobCache string + certDir string + creds string + digestfile string + disableCompression bool + format string + compressionFormat string + compressionLevel int + forceCompressionFormat bool + retry int + retryDelay string + rm bool + quiet bool + removeSignatures bool + signaturePolicy string + signBy string + tlsVerify bool + encryptionKeys []string + encryptLayers []int + insecure bool + addCompression []string } func init() { @@ -86,6 +87,7 @@ func init() { flags.StringVar(&opts.creds, "creds", "", "use `[username[:password]]` for accessing the registry") flags.StringVar(&opts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting image to the file") flags.BoolVarP(&opts.disableCompression, "disable-compression", "D", false, "don't compress layers") + flags.BoolVarP(&opts.forceCompressionFormat, "force-compression", "", false, "use the specified compression algorithm if the destination contains a differently-compressed variant already") flags.StringVarP(&opts.format, "format", "f", "", "manifest type (oci, v2s1, or v2s2) to use in the destination (default is manifest type of source, with fallbacks)") flags.StringVar(&opts.compressionFormat, "compression-format", "", "compression format to use") flags.IntVar(&opts.compressionLevel, "compression-level", 0, "compression level to use") @@ -199,18 +201,19 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error { } options := buildah.PushOptions{ - Compression: compress, - ManifestType: manifestType, - SignaturePolicyPath: iopts.signaturePolicy, - Store: store, - SystemContext: systemContext, - BlobDirectory: iopts.blobCache, - RemoveSignatures: iopts.removeSignatures, - SignBy: iopts.signBy, - MaxRetries: iopts.retry, - RetryDelay: pullPushRetryDelay, - OciEncryptConfig: encConfig, - OciEncryptLayers: encLayers, + Compression: compress, + ManifestType: manifestType, + SignaturePolicyPath: iopts.signaturePolicy, + Store: store, + SystemContext: systemContext, + BlobDirectory: iopts.blobCache, + RemoveSignatures: iopts.removeSignatures, + SignBy: iopts.signBy, + MaxRetries: iopts.retry, + RetryDelay: pullPushRetryDelay, + OciEncryptConfig: encConfig, + OciEncryptLayers: encLayers, + ForceCompressionFormat: iopts.forceCompressionFormat, } if !iopts.quiet { options.ReportWriter = os.Stderr diff --git a/docs/buildah-manifest-push.1.md b/docs/buildah-manifest-push.1.md index 66f67217e..2aed1bba5 100644 --- a/docs/buildah-manifest-push.1.md +++ b/docs/buildah-manifest-push.1.md @@ -42,6 +42,16 @@ If the authorization state is not found there, $HOME/.docker/config.json is chec Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry. The default certificates directory is _/etc/containers/certs.d_. +**--compression-format** *format* + +Specifies the compression format to use. Supported values are: `gzip`, `zstd` and `zstd:chunked`. + +**--compression-level** *level* + +Specify the compression level used with the compression. + +Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive). + **--creds** *creds* The [username[:password]] to use to authenticate with the registry if required. @@ -52,6 +62,10 @@ value can be entered. The password is entered without echo. After copying the image, write the digest of the resulting image to the file. +**--force-compression** + +Use the specified compression algorithm even if the destination contains a differently-compressed variant already. + **--format**, **-f** Manifest list type (oci or v2s2) to use when pushing the list (default is oci). diff --git a/docs/buildah-push.1.md b/docs/buildah-push.1.md index 7df91f3c4..1260e8c4b 100644 --- a/docs/buildah-push.1.md +++ b/docs/buildah-push.1.md @@ -70,6 +70,10 @@ Layer(s) to encrypt: 0-indexed layer indices with support for negative indexing The [protocol:keyfile] specifies the encryption protocol, which can be JWE (RFC7516), PGP (RFC4880), and PKCS7 (RFC2315) and the key material required for image encryption. For instance, jwe:/path/to/key.pem or pgp:admin@example.com or pkcs7:/path/to/x509-file. +**--force-compression** + +Use the specified compression algorithm even if the destination contains a differently-compressed variant already. + **--format**, **-f** Manifest Type (oci, v2s2, or v2s1) to use when pushing an image. (default is manifest type of the source image, with fallbacks) diff --git a/push.go b/push.go index 1f4439147..2e2b9498a 100644 --- a/push.go +++ b/push.go @@ -95,6 +95,10 @@ type PushOptions struct { CompressionFormat *compression.Algorithm // CompressionLevel specifies what compression level is used CompressionLevel *int + // ForceCompressionFormat ensures that the compression algorithm set in + // CompressionFormat is used exclusively, and blobs of other compression + // algorithms are not reused. + ForceCompressionFormat bool } // Push copies the contents of the image to a new location. @@ -110,6 +114,7 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options libimageOptions.OciEncryptLayers = options.OciEncryptLayers libimageOptions.CompressionFormat = options.CompressionFormat libimageOptions.CompressionLevel = options.CompressionLevel + libimageOptions.ForceCompressionFormat = options.ForceCompressionFormat libimageOptions.PolicyAllowStorage = true if options.Quiet { diff --git a/tests/bud.bats b/tests/bud.bats index 36b2e6c14..196a933e8 100644 --- a/tests/bud.bats +++ b/tests/bud.bats @@ -39,6 +39,72 @@ _EOF validate_instance_compression "3" "$list" "arm64" "zstd" } +@test "bud: build manifest list with --add-compression zstd, --compression and --force-compression" { + local contextdir=${TEST_SCRATCH_DIR}/bud/platform + mkdir -p $contextdir + + cat > $contextdir/Dockerfile1 << _EOF +FROM alpine +_EOF + + start_registry + run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT} + run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1 + run_buildah build $WITH_POLICY_JSON -t image2 --platform linux/arm64 -f $contextdir/Dockerfile1 + + run_buildah manifest create foo + run_buildah manifest add foo image1 + run_buildah manifest add foo image2 + + run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list + + run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list + list="$output" + + validate_instance_compression "0" "$list" "amd64" "gzip" + validate_instance_compression "1" "$list" "arm64" "gzip" + validate_instance_compression "2" "$list" "amd64" "zstd" + validate_instance_compression "3" "$list" "arm64" "zstd" + + # Pushing again should keep every thing intact if original compression is `gzip` and `--force-compression` is specified + run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --compression-format gzip --force-compression --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list + + run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list + list="$output" + + validate_instance_compression "0" "$list" "amd64" "gzip" + validate_instance_compression "1" "$list" "arm64" "gzip" + validate_instance_compression "2" "$list" "amd64" "zstd" + validate_instance_compression "3" "$list" "arm64" "zstd" +} + +@test "bud: build push with --force-compression" { + skip_if_no_podman + local contextdir=${TEST_SCRATCH_DIR}/bud/platform + mkdir -p $contextdir + + cat > $contextdir/Dockerfile1 << _EOF +FROM alpine +_EOF + + start_registry + run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT} + run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1 + + run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format gzip image1 docker://localhost:${REGISTRY_PORT}/image + run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image + # layers should have no trace of zstd since push was with --compression-format gzip + assert "$output" !~ "zstd" "zstd found in layers where push was with --compression-format gzip" + run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format zstd image1 docker://localhost:${REGISTRY_PORT}/image + run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image + # layers should have no trace of zstd since push is without --force-compression + assert "$output" !~ "zstd" "zstd found even though push was without --force-compression" + run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format zstd --force-compression image1 docker://localhost:${REGISTRY_PORT}/image + run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image + # layers should container `zstd` + expect_output --substring "zstd" "layers must contain zstd compression" +} + @test "bud with --dns* flags" { _prefetch alpine