root
/
buildah
mirror of https://github.com/containers/buildah.git
1
0
Fork 0
Code Issues Actions 3 Packages Projects Releases Wiki Activity

buildah/push/manifest-push: add support for --force-compression 

Adds support for `--force-compression` which allows end-users to force
push blobs with the selected compresison in `--compression` option, in
order to make sure that `blobs` of other compression on registry are not
reused.

Is equivalent to: `force-compression` here: https://docs.docker.com/build/exporters/#compression
Closes: https://github.com/containers/buildah/issues/4613

Also Implements:
`--compression-format` and `--compression-level` for `manifest push` just like
`podman`'s `manifest push`

Signed-off-by: Aditya R <arajan@redhat.com>
This commit is contained in:
Aditya R 2023-08-11 23:47:11 +05:30
parent 3445a775a0
commit d68d9a237c
No known key found for this signature in database
GPG Key ID: 8E5A8A19DF7C8673
6 changed files with 149 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/buildah/manifest.go
View File

@ -17,6 +17,7 @@ import (
"github.com/containers/common/pkg/auth"
cp "github.com/containers/image/v5/copy"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/compression"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
@ -226,6 +227,9 @@ func init() {
flags.StringVar(&manifestPushOpts.certDir, "cert-dir", "", "use certificates at the specified path to access the registry")
flags.StringVar(&manifestPushOpts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
flags.StringVar(&manifestPushOpts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting digest to the file")
flags.BoolVarP(&manifestPushOpts.forceCompressionFormat, "force-compression", "", false, "use the specified compression algorithm if the destination contains a differently-compressed variant already")
flags.StringVar(&manifestPushOpts.compressionFormat, "compression-format", "", "compression format to use")
flags.IntVar(&manifestPushOpts.compressionLevel, "compression-level", 0, "compression level to use")
flags.StringVarP(&manifestPushOpts.format, "format", "f", "", "manifest type (oci or v2s2) to attempt to use when pushing the manifest list (default is manifest type of source)")
flags.StringSliceVar(&manifestPushOpts.addCompression, "add-compression", nil, "add instances with selected compression while pushing")
flags.BoolVarP(&manifestPushOpts.removeSignatures, "remove-signatures", "", false, "don't copy signatures when pushing images")
@ -864,6 +868,16 @@ func manifestPushCmd(c *cobra.Command, args []string, opts pushOptions) error {
if err != nil {
return fmt.Errorf("building system context: %w", err)
}
if opts.compressionFormat != "" {
algo, err := compression.AlgorithmByName(opts.compressionFormat)
if err != nil {
return err
}
systemContext.CompressionFormat = &algo
}
if c.Flag("compression-level").Changed {
systemContext.CompressionLevel = &opts.compressionLevel
}
return manifestPush(systemContext, store, listImageSpec, destSpec, opts)
}
@ -910,6 +924,7 @@ func manifestPush(systemContext *types.SystemContext, store storage.Store, listI
SignBy: opts.signBy,
ManifestType: manifestType,
AddCompression: opts.addCompression,
ForceCompressionFormat: opts.forceCompressionFormat,
}
if opts.all {
options.ImageListSelection = cp.CopyAllImages

3
cmd/buildah/push.go
View File

@ -35,6 +35,7 @@ type pushOptions struct {
format string
compressionFormat string
compressionLevel int
forceCompressionFormat bool
retry int
retryDelay string
rm bool
@ -86,6 +87,7 @@ func init() {
flags.StringVar(&opts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
flags.StringVar(&opts.digestfile, "digestfile", "", "after copying the image, write the digest of the resulting image to the file")
flags.BoolVarP(&opts.disableCompression, "disable-compression", "D", false, "don't compress layers")
flags.BoolVarP(&opts.forceCompressionFormat, "force-compression", "", false, "use the specified compression algorithm if the destination contains a differently-compressed variant already")
flags.StringVarP(&opts.format, "format", "f", "", "manifest type (oci, v2s1, or v2s2) to use in the destination (default is manifest type of source, with fallbacks)")
flags.StringVar(&opts.compressionFormat, "compression-format", "", "compression format to use")
flags.IntVar(&opts.compressionLevel, "compression-level", 0, "compression level to use")
@ -211,6 +213,7 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error {
RetryDelay: pullPushRetryDelay,
OciEncryptConfig: encConfig,
OciEncryptLayers: encLayers,
ForceCompressionFormat: iopts.forceCompressionFormat,
}
if !iopts.quiet {
options.ReportWriter = os.Stderr

14
docs/buildah-manifest-push.1.md
View File

@ -42,6 +42,16 @@ If the authorization state is not found there, $HOME/.docker/config.json is chec
Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
The default certificates directory is _/etc/containers/certs.d_.
**--compression-format** *format*
Specifies the compression format to use. Supported values are: `gzip`, `zstd` and `zstd:chunked`.
**--compression-level** *level*
Specify the compression level used with the compression.
Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive).
**--creds** *creds*
The [username[:password]] to use to authenticate with the registry if required.
@ -52,6 +62,10 @@ value can be entered. The password is entered without echo.
After copying the image, write the digest of the resulting image to the file.
**--force-compression**
Use the specified compression algorithm even if the destination contains a differently-compressed variant already.
**--format**, **-f**
Manifest list type (oci or v2s2) to use when pushing the list (default is oci).

4
docs/buildah-push.1.md
View File

@ -70,6 +70,10 @@ Layer(s) to encrypt: 0-indexed layer indices with support for negative indexing
The [protocol:keyfile] specifies the encryption protocol, which can be JWE (RFC7516), PGP (RFC4880), and PKCS7 (RFC2315) and the key material required for image encryption. For instance, jwe:/path/to/key.pem or pgp:admin@example.com or pkcs7:/path/to/x509-file.
**--force-compression**
Use the specified compression algorithm even if the destination contains a differently-compressed variant already.
**--format**, **-f**
Manifest Type (oci, v2s2, or v2s1) to use when pushing an image. (default is manifest type of the source image, with fallbacks)

5
push.go
View File

@ -95,6 +95,10 @@ type PushOptions struct {
CompressionFormat *compression.Algorithm
// CompressionLevel specifies what compression level is used
CompressionLevel *int
// ForceCompressionFormat ensures that the compression algorithm set in
// CompressionFormat is used exclusively, and blobs of other compression
// algorithms are not reused.
ForceCompressionFormat bool
}
// Push copies the contents of the image to a new location.
@ -110,6 +114,7 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options
libimageOptions.OciEncryptLayers = options.OciEncryptLayers
libimageOptions.CompressionFormat = options.CompressionFormat
libimageOptions.CompressionLevel = options.CompressionLevel
libimageOptions.ForceCompressionFormat = options.ForceCompressionFormat
libimageOptions.PolicyAllowStorage = true
if options.Quiet {

66
tests/bud.bats
View File

@ -39,6 +39,72 @@ _EOF
validate_instance_compression "3" "$list" "arm64" "zstd"
}
@test "bud: build manifest list with --add-compression zstd, --compression and --force-compression" {
local contextdir=${TEST_SCRATCH_DIR}/bud/platform
mkdir -p $contextdir
cat > $contextdir/Dockerfile1 << _EOF
FROM alpine
_EOF
start_registry
run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT}
run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1
run_buildah build $WITH_POLICY_JSON -t image2 --platform linux/arm64 -f $contextdir/Dockerfile1
run_buildah manifest create foo
run_buildah manifest add foo image1
run_buildah manifest add foo image2
run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list
run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list
list="$output"
validate_instance_compression "0" "$list" "amd64" "gzip"
validate_instance_compression "1" "$list" "arm64" "gzip"
validate_instance_compression "2" "$list" "amd64" "zstd"
validate_instance_compression "3" "$list" "arm64" "zstd"
# Pushing again should keep every thing intact if original compression is `gzip` and `--force-compression` is specified
run_buildah manifest push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --all --add-compression zstd --compression-format gzip --force-compression --tls-verify=false foo docker://localhost:${REGISTRY_PORT}/list
run_buildah manifest inspect --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false localhost:${REGISTRY_PORT}/list
list="$output"
validate_instance_compression "0" "$list" "amd64" "gzip"
validate_instance_compression "1" "$list" "arm64" "gzip"
validate_instance_compression "2" "$list" "amd64" "zstd"
validate_instance_compression "3" "$list" "arm64" "zstd"
}
@test "bud: build push with --force-compression" {
skip_if_no_podman
local contextdir=${TEST_SCRATCH_DIR}/bud/platform
mkdir -p $contextdir
cat > $contextdir/Dockerfile1 << _EOF
FROM alpine
_EOF
start_registry
run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT}
run_buildah build $WITH_POLICY_JSON -t image1 --platform linux/amd64 -f $contextdir/Dockerfile1
run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format gzip image1 docker://localhost:${REGISTRY_PORT}/image
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
# layers should have no trace of zstd since push was with --compression-format gzip
assert "$output" !~ "zstd" "zstd found in layers where push was with --compression-format gzip"
run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format zstd image1 docker://localhost:${REGISTRY_PORT}/image
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
# layers should have no trace of zstd since push is without --force-compression
assert "$output" !~ "zstd" "zstd found even though push was without --force-compression"
run_buildah push $WITH_POLICY_JSON --authfile ${TEST_SCRATCH_DIR}/test.auth --tls-verify=false --compression-format zstd --force-compression image1 docker://localhost:${REGISTRY_PORT}/image
run podman run --rm --mount type=bind,src=${TEST_SCRATCH_DIR}/test.auth,target=/test.auth,Z --net host quay.io/skopeo/stable inspect --authfile=/test.auth --tls-verify=false --raw docker://localhost:${REGISTRY_PORT}/image
# layers should container `zstd`
expect_output --substring "zstd" "layers must contain zstd compression"
}
@test "bud with --dns* flags" {
_prefetch alpine