Implement ADD checksum flag #5135

See https://docs.docker.com/build/dockerfile/release-notes/#160.

Fixes #5135

Signed-off-by: Jean-Francois Roy <jf@devklog.net>

add.go

@@ -22,6 +22,7 @@ import (
 	"github.com/containers/storage/pkg/fileutils"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/hashicorp/go-multierror"
+	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/runc/libcontainer/userns"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
@@ -35,6 +36,9 @@ type AddAndCopyOptions struct {
 	// newly-added content, potentially overriding permissions which would
 	// otherwise be set to 0:0.
 	Chown string
+	// Checksum is a standard container digest string (e.g. <algorithm>:<digest>)
+	// and is the expected hash of the content being copied.
+	Checksum string
 	// PreserveOwnership, if Chown is not set, tells us to avoid setting
 	// ownership of copied items to 0:0, instead using whatever ownership
 	// information is already set.  Not meaningful for remote sources or
@@ -77,7 +81,7 @@ func sourceIsRemote(source string) bool {
 }
 
 // getURL writes a tar archive containing the named content
-func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string, writer io.Writer, chmod *os.FileMode) error {
+func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string, writer io.Writer, chmod *os.FileMode, srcDigest digest.Digest) error {
 	url, err := url.Parse(src)
 	if err != nil {
 		return err
@@ -110,7 +114,7 @@ func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string,
 	}
 	// Figure out the size of the content.
 	size := response.ContentLength
-	responseBody := response.Body
+	var responseBody io.Reader = response.Body
 	if size < 0 {
 		// Create a temporary file and copy the content to it, so that
 		// we can figure out how much content there is.
@@ -130,6 +134,11 @@ func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string,
 		}
 		responseBody = f
 	}
+	var digester digest.Digester
+	if srcDigest != "" {
+		digester = srcDigest.Algorithm().Digester()
+		responseBody = io.TeeReader(responseBody, digester.Hash())
+	}
 	// Write the output archive.  Set permissions for compatibility.
 	tw := tar.NewWriter(writer)
 	defer tw.Close()
@@ -161,6 +170,12 @@ func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string,
 		return fmt.Errorf("writing content from %q to tar stream: %w", src, err)
 	}
 
+	if digester != nil {
+		if responseDigest := digester.Digest(); responseDigest != srcDigest {
+			return fmt.Errorf("unexpected response digest for %q: %s, want %s", src, responseDigest, srcDigest)
+		}
+	}
+
 	return nil
 }
 
@@ -392,9 +407,16 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		var wg sync.WaitGroup
 		if sourceIsRemote(src) {
 			pipeReader, pipeWriter := io.Pipe()
+			var srcDigest digest.Digest
+			if options.Checksum != "" {
+				srcDigest, err = digest.Parse(options.Checksum)
+				if err != nil {
+					return fmt.Errorf("invalid checksum flag: %w", err)
+				}
+			}
 			wg.Add(1)
 			go func() {
-				getErr = getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles)
+				getErr = getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest)
 				pipeWriter.Close()
 				wg.Done()
 			}()
@@ -441,6 +463,10 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 			continue
 		}
 
+		if options.Checksum != "" {
+			return fmt.Errorf("checksum flag is not supported for local sources")
+		}
+
 		// Dig out the result of running glob+stat on this source spec.
 		var localSourceStat *copier.StatsForGlob
 		for _, st := range localSourceStats {

cmd/buildah/addcopy.go

@@ -22,6 +22,7 @@ type addCopyResults struct {
 	addHistory       bool
 	chmod            string
 	chown            string
+	checksum         string
 	quiet            bool
 	ignoreFile       string
 	contextdir       string
@@ -67,6 +68,7 @@ func applyFlagVars(flags *pflag.FlagSet, opts *addCopyResults) {
 	if err := flags.MarkHidden("cert-dir"); err != nil {
 		panic(fmt.Sprintf("error marking cert-dir as hidden: %v", err))
 	}
+	flags.StringVar(&opts.checksum, "checksum", "", "checksum the HTTP source content")
 	flags.StringVar(&opts.chown, "chown", "", "set the user and group ownership of the destination content")
 	flags.StringVar(&opts.chmod, "chmod", "", "set the access permissions of the destination content")
 	flags.StringVar(&opts.creds, "creds", "", "use `[username[:password]]` for accessing registries when pulling images")
@@ -235,6 +237,7 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 	options := buildah.AddAndCopyOptions{
 		Chmod:            iopts.chmod,
 		Chown:            iopts.chown,
+		Checksum:         iopts.checksum,
 		ContextDir:       contextdir,
 		IDMappingOptions: idMappingOptions,
 	}

docs/buildah-add.1.md

@@ -23,6 +23,11 @@ Defaults to false.
 Note: You can also override the default value of --add-history by setting the
 BUILDAH\_HISTORY environment variable. `export BUILDAH_HISTORY=true`
 
+**--checksum** *checksum*
+
+Checksum the source content. The value of *checksum* must be a standard
+container digest string. Only supported for HTTP sources.
+
 **--chmod** *permissions*
 
 Sets the access permissions of the destination content. Accepts the numerical format.

docs/buildah-copy.1.md

@@ -21,6 +21,11 @@ Defaults to false.
 Note: You can also override the default value of --add-history by setting the
 BUILDAH\_HISTORY environment variable. `export BUILDAH_HISTORY=true`
 
+**--checksum** *checksum*
+
+Checksum the source content. The value of *checksum* must be a standard
+container digest string. Only supported for HTTP sources.
+
 **--chmod** *permissions*
 
 Sets the access permissions of the destination content. Accepts the numerical format.

imagebuildah/stage_executor.go

@@ -473,6 +473,7 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err
 		options := buildah.AddAndCopyOptions{
 			Chmod:             copy.Chmod,
 			Chown:             copy.Chown,
+			Checksum:          copy.Checksum,
 			PreserveOwnership: preserveOwnership,
 			ContextDir:        contextDir,
 			Excludes:          copyExcludes,
@@ -1118,8 +1119,8 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 			if command == "COPY" && (flag == "--chmod" || flag == "--chown" || flag == "--from") {
 				return "", nil, false, fmt.Errorf("COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image|stage> flags")
 			}
-			if command == "ADD" && (flag == "--chmod" || flag == "--chown") {
+			if command == "ADD" && (flag == "--chmod" || flag == "--chown" || flag == "--checksum") {
-				return "", nil, false, fmt.Errorf("ADD only supports the --chmod=<permissions> and the --chown=<uid:gid> flags")
+				return "", nil, false, fmt.Errorf("ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> flags")
 			}
 			if strings.Contains(flag, "--from") && command == "COPY" {
 				arr := strings.Split(flag, "=")

tests/add.bats

@@ -281,3 +281,28 @@ stuff/mystuff"
   cmp $ubuntu/etc/passwd ${croot}/tmp/passwd
   cmp $ubuntu/etc/passwd ${croot}/tmp/passwd2
 }
+
+@test "add url with checksum flag" {
+  _prefetch busybox
+  run_buildah from --quiet $WITH_POLICY_JSON busybox
+  cid=$output
+  run_buildah add --checksum=sha256:4fd3aed66b5488b45fe83dd11842c2324fadcc38e1217bb45fbd28d660afdd39 $cid https://raw.githubusercontent.com/containers/buildah/bf3b55ba74102cc2503eccbaeffe011728d46b20/README.md /
+  run_buildah run $cid ls /README.md
+}
+
+@test "add url with bad checksum" {
+  _prefetch busybox
+  run_buildah from --quiet $WITH_POLICY_JSON busybox
+  cid=$output
+  run_buildah 125 add --checksum=sha256:0000000000000000000000000000000000000000000000000000000000000000 $cid https://raw.githubusercontent.com/containers/buildah/bf3b55ba74102cc2503eccbaeffe011728d46b20/README.md /
+  expect_output --substring "unexpected response digest for \"https://raw.githubusercontent.com/containers/buildah/bf3b55ba74102cc2503eccbaeffe011728d46b20/README.md\": sha256:4fd3aed66b5488b45fe83dd11842c2324fadcc38e1217bb45fbd28d660afdd39, want sha256:0000000000000000000000000000000000000000000000000000000000000000"
+}
+
+@test "add path with checksum flag" {
+  _prefetch busybox
+  createrandom ${TEST_SCRATCH_DIR}/randomfile
+  run_buildah from --quiet $WITH_POLICY_JSON busybox
+  cid=$output
+  run_buildah 125 add --checksum=sha256:0000000000000000000000000000000000000000000000000000000000000000 $cid ${TEST_SCRATCH_DIR}/randomfile /
+  expect_output --substring "checksum flag is not supported for local sources"
+}

tests/bud.bats

@@ -3088,7 +3088,7 @@ _EOF
   imgName=alpine-image
   ctrName=alpine-chown
   run_buildah 125 build $WITH_POLICY_JSON --layers -t ${imgName} -f $BUDFILES/add-chown/Dockerfile.bad $BUDFILES/add-chown
-  expect_output --substring "ADD only supports the --chmod=<permissions> and the --chown=<uid:gid> flags"
+  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> flags"
 }
 
 @test "bud with chmod add with bad chmod flag in Dockerfile with --layers" {
@@ -3096,7 +3096,30 @@ _EOF
   imgName=alpine-image
   ctrName=alpine-chmod
   run_buildah 125 build $WITH_POLICY_JSON --layers -t ${imgName} -f $BUDFILES/add-chmod/Dockerfile.bad $BUDFILES/add-chmod
-  expect_output --substring "ADD only supports the --chmod=<permissions> and the --chown=<uid:gid> flags"
+  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> flags"
+}
+
+@test "bud with ADD with checksum flag" {
+  _prefetch alpine
+  target=alpine-image
+  run_buildah build $WITH_POLICY_JSON -t alpine-image -f $BUDFILES/add-checksum/Containerfile $BUDFILES/add-checksum
+  run_buildah from --quiet $WITH_POLICY_JSON --name alpine-ctr alpine-image
+  run_buildah run alpine-ctr -- ls -l /README.md
+  expect_output --substring "README.md"
+}
+
+@test "bud with ADD with bad checksum" {
+  _prefetch alpine
+  target=alpine-image
+  run_buildah 125 build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/add-checksum/Containerfile.bad-checksum $BUDFILES/add-checksum
+  expect_output --substring "unexpected response digest for \"https://raw.githubusercontent.com/containers/buildah/bf3b55ba74102cc2503eccbaeffe011728d46b20/README.md\": sha256:4fd3aed66b5488b45fe83dd11842c2324fadcc38e1217bb45fbd28d660afdd39, want sha256:0000000000000000000000000000000000000000000000000000000000000000"
+}
+
+@test "bud with ADD with bad checksum flag" {
+  _prefetch alpine
+  target=alpine-image
+  run_buildah 125 build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/add-checksum/Containerfile.bad $BUDFILES/add-checksum
+  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> flags"
 }
 
 @test "bud with ADD file construct" {

tests/bud/add-checksum/Containerfile

@@ -0,0 +1,2 @@
+FROM alpine
+ADD --checksum=sha256:4fd3aed66b5488b45fe83dd11842c2324fadcc38e1217bb45fbd28d660afdd39 https://raw.githubusercontent.com/containers/buildah/bf3b55ba74102cc2503eccbaeffe011728d46b20/README.md /

tests/bud/add-checksum/Containerfile.bad

@@ -0,0 +1,2 @@
+FROM alpine
+ADD --checksum https://raw.githubusercontent.com/containers/buildah/bf3b55ba74102cc2503eccbaeffe011728d46b20/README.md /

tests/bud/add-checksum/Containerfile.bad-checksum

@@ -0,0 +1,2 @@
+FROM alpine
+ADD --checksum=sha256:0000000000000000000000000000000000000000000000000000000000000000 https://raw.githubusercontent.com/containers/buildah/bf3b55ba74102cc2503eccbaeffe011728d46b20/README.md /