add: add a DryRun flag to AddAndCopyOptions

Add a DryRun flag to AddAndCopyOptions, so that we can "copy" content to
digest it.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

Closes: #1792
Approved by: TomSweeneyRedHat

add.go

@@ -35,57 +35,68 @@ type AddAndCopyOptions struct {
 	Hasher io.Writer
 	// Excludes is the contents of the .dockerignore file
 	Excludes []string
-	// The base directory for Excludes and data to copy in
+	// ContextDir is the base directory for Excludes for content being copied
 	ContextDir string
 	// ID mapping options to use when contents to be copied are part of
 	// another container, and need ownerships to be mapped from the host to
 	// that container's values before copying them into the container.
 	IDMappingOptions *IDMappingOptions
+	// DryRun indicates that the content should be digested, but not actually
+	// copied into the container.
+	DryRun bool
 }
 
 // addURL copies the contents of the source URL to the destination.  This is
 // its own function so that deferred closes happen after we're done pulling
 // down each item of potentially many.
-func addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer) error {
+func addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer, dryRun bool) error {
-	logrus.Debugf("saving %q to %q", srcurl, destination)
 	resp, err := http.Get(srcurl)
 	if err != nil {
 		return errors.Wrapf(err, "error getting %q", srcurl)
 	}
 	defer resp.Body.Close()
-	f, err := os.Create(destination)
+
-	if err != nil {
+	thisWriter := hasher
-		return errors.Wrapf(err, "error creating %q", destination)
+
-	}
+	if !dryRun {
-	if err = f.Chown(owner.UID, owner.GID); err != nil {
+		logrus.Debugf("saving %q to %q", srcurl, destination)
-		return errors.Wrapf(err, "error setting owner of %q to %d:%d", destination, owner.UID, owner.GID)
+		f, err := os.Create(destination)
-	}
+		if err != nil {
-	if last := resp.Header.Get("Last-Modified"); last != "" {
+			return errors.Wrapf(err, "error creating %q", destination)
-		if mtime, err2 := time.Parse(time.RFC1123, last); err2 != nil {
+		}
-			logrus.Debugf("error parsing Last-Modified time %q: %v", last, err2)
+		defer f.Close()
+		if err = f.Chown(owner.UID, owner.GID); err != nil {
+			return errors.Wrapf(err, "error setting owner of %q to %d:%d", destination, owner.UID, owner.GID)
+		}
+		if last := resp.Header.Get("Last-Modified"); last != "" {
+			if mtime, err2 := time.Parse(time.RFC1123, last); err2 != nil {
+				logrus.Debugf("error parsing Last-Modified time %q: %v", last, err2)
+			} else {
+				defer func() {
+					if err3 := os.Chtimes(destination, time.Now(), mtime); err3 != nil {
+						logrus.Debugf("error setting mtime on %q to Last-Modified time %q: %v", destination, last, err3)
+					}
+				}()
+			}
+		}
+		defer func() {
+			if err2 := f.Chmod(0600); err2 != nil {
+				logrus.Debugf("error setting permissions on %q: %v", destination, err2)
+			}
+		}()
+		if thisWriter != nil {
+			thisWriter = io.MultiWriter(f, thisWriter)
 		} else {
-			defer func() {
+			thisWriter = f
-				if err3 := os.Chtimes(destination, time.Now(), mtime); err3 != nil {
-					logrus.Debugf("error setting mtime on %q to Last-Modified time %q: %v", destination, last, err3)
-				}
-			}()
 		}
 	}
-	defer f.Close()
+	n, err := io.Copy(thisWriter, resp.Body)
-	bodyReader := io.Reader(resp.Body)
-	if hasher != nil {
-		bodyReader = io.TeeReader(bodyReader, hasher)
-	}
-	n, err := io.Copy(f, bodyReader)
 	if err != nil {
 		return errors.Wrapf(err, "error reading contents for %q from %q", destination, srcurl)
 	}
 	if resp.ContentLength >= 0 && n != resp.ContentLength {
 		return errors.Errorf("error reading contents for %q from %q: wrong length (%d != %d)", destination, srcurl, n, resp.ContentLength)
 	}
-	if err := f.Chmod(0600); err != nil {
-		return errors.Wrapf(err, "error setting permissions on %q", destination)
-	}
 	return nil
 }
 
@@ -118,31 +129,34 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 	}
 	hostOwner := idtools.IDPair{UID: int(hostUID), GID: int(hostGID)}
 	dest := mountPoint
-	if destination != "" && filepath.IsAbs(destination) {
+	if !options.DryRun {
-		dir := filepath.Dir(destination)
+		// Resolve the destination if it was specified as a relative path.
-		if dir != "." && dir != "/" {
+		if destination != "" && filepath.IsAbs(destination) {
-			if err = idtools.MkdirAllAndChownNew(filepath.Join(dest, dir), 0755, hostOwner); err != nil {
+			dir := filepath.Dir(destination)
-				return errors.Wrapf(err, "error creating directory %q", filepath.Join(dest, dir))
+			if dir != "." && dir != "/" {
+				if err = idtools.MkdirAllAndChownNew(filepath.Join(dest, dir), 0755, hostOwner); err != nil {
+					return errors.Wrapf(err, "error creating directory %q", filepath.Join(dest, dir))
+				}
+			}
+			dest = filepath.Join(dest, destination)
+		} else {
+			if err = idtools.MkdirAllAndChownNew(filepath.Join(dest, b.WorkDir()), 0755, hostOwner); err != nil {
+				return errors.Wrapf(err, "error creating directory %q", filepath.Join(dest, b.WorkDir()))
+			}
+			dest = filepath.Join(dest, b.WorkDir(), destination)
+		}
+		// If the destination was explicitly marked as a directory by ending it
+		// with a '/', create it so that we can be sure that it's a directory,
+		// and any files we're copying will be placed in the directory.
+		if len(destination) > 0 && destination[len(destination)-1] == os.PathSeparator {
+			if err = idtools.MkdirAllAndChownNew(dest, 0755, hostOwner); err != nil {
+				return errors.Wrapf(err, "error creating directory %q", dest)
 			}
 		}
-		dest = filepath.Join(dest, destination)
+		// Make sure the destination's parent directory is usable.
-	} else {
+		if destpfi, err2 := os.Stat(filepath.Dir(dest)); err2 == nil && !destpfi.IsDir() {
-		if err = idtools.MkdirAllAndChownNew(filepath.Join(dest, b.WorkDir()), 0755, hostOwner); err != nil {
+			return errors.Errorf("%q already exists, but is not a subdirectory)", filepath.Dir(dest))
-			return errors.Wrapf(err, "error creating directory %q", filepath.Join(dest, b.WorkDir()))
 		}
-		dest = filepath.Join(dest, b.WorkDir(), destination)
-	}
-	// If the destination was explicitly marked as a directory by ending it
-	// with a '/', create it so that we can be sure that it's a directory,
-	// and any files we're copying will be placed in the directory.
-	if len(destination) > 0 && destination[len(destination)-1] == os.PathSeparator {
-		if err = idtools.MkdirAllAndChownNew(dest, 0755, hostOwner); err != nil {
-			return errors.Wrapf(err, "error creating directory %q", dest)
-		}
-	}
-	// Make sure the destination's parent directory is usable.
-	if destpfi, err2 := os.Stat(filepath.Dir(dest)); err2 == nil && !destpfi.IsDir() {
-		return errors.Errorf("%q already exists, but is not a subdirectory)", filepath.Dir(dest))
 	}
 	// Now look at the destination itself.
 	destfi, err := os.Stat(dest)
@@ -155,9 +169,9 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 	if len(source) > 1 && (destfi == nil || !destfi.IsDir()) {
 		return errors.Errorf("destination %q is not a directory", dest)
 	}
-	copyFileWithTar := b.copyFileWithTar(options.IDMappingOptions, &containerOwner, options.Hasher)
+	copyFileWithTar := b.copyFileWithTar(options.IDMappingOptions, &containerOwner, options.Hasher, options.DryRun)
-	copyWithTar := b.copyWithTar(options.IDMappingOptions, &containerOwner, options.Hasher)
+	copyWithTar := b.copyWithTar(options.IDMappingOptions, &containerOwner, options.Hasher, options.DryRun)
-	untarPath := b.untarPath(nil, options.Hasher)
+	untarPath := b.untarPath(nil, options.Hasher, options.DryRun)
 	err = addHelper(excludes, extract, dest, destfi, hostOwner, options, copyFileWithTar, copyWithTar, untarPath, source...)
 	if err != nil {
 		return err
@@ -245,7 +259,7 @@ func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, de
 			if destfi != nil && destfi.IsDir() {
 				d = filepath.Join(dest, path.Base(url.Path))
 			}
-			if err = addURL(d, src, hostOwner, options.Hasher); err != nil {
+			if err = addURL(d, src, hostOwner, options.Hasher, options.DryRun); err != nil {
 				return err
 			}
 			continue
@@ -273,8 +287,10 @@ func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, de
 				// the source directory into the target directory.  Try
 				// to create it first, so that if there's a problem,
 				// we'll discover why that won't work.
-				if err = idtools.MkdirAllAndChownNew(dest, 0755, hostOwner); err != nil {
+				if !options.DryRun {
-					return errors.Wrapf(err, "error creating directory %q", dest)
+					if err = idtools.MkdirAllAndChownNew(dest, 0755, hostOwner); err != nil {
+						return errors.Wrapf(err, "error creating directory %q", dest)
+					}
 				}
 				logrus.Debugf("copying %q to %q", esrc+string(os.PathSeparator)+"*", dest+string(os.PathSeparator)+"*")
 				if excludes == nil || !excludes.Exclusions() {

run_linux.go

@@ -431,7 +431,7 @@ func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath st
 
 	// Add temporary copies of the contents of volume locations at the
 	// volume locations, unless we already have something there.
-	copyWithTar := b.copyWithTar(nil, nil, nil)
+	copyWithTar := b.copyWithTar(nil, nil, nil, false)
 	builtins, err := runSetupBuiltinVolumes(b.MountLabel, mountPoint, cdir, copyWithTar, builtinVolumes, int(rootUID), int(rootGID))
 	if err != nil {
 		return err

util.go

@@ -3,6 +3,7 @@ package buildah
 import (
 	"archive/tar"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -112,7 +113,7 @@ func convertRuntimeIDMaps(UIDMap, GIDMap []rspec.LinuxIDMapping) ([]idtools.IDMa
 // of any container, or another container, into our working container, mapping
 // read permissions using the passed-in ID maps, writing using the container's
 // ID mappings, possibly overridden using the passed-in chownOpts
-func (b *Builder) copyFileWithTar(tarIDMappingOptions *IDMappingOptions, chownOpts *idtools.IDPair, hasher io.Writer) func(src, dest string) error {
+func (b *Builder) copyFileWithTar(tarIDMappingOptions *IDMappingOptions, chownOpts *idtools.IDPair, hasher io.Writer, dryRun bool) func(src, dest string) error {
 	if tarIDMappingOptions == nil {
 		tarIDMappingOptions = &IDMappingOptions{
 			HostUIDMapping: true,
@@ -207,7 +208,7 @@ func (b *Builder) copyFileWithTar(tarIDMappingOptions *IDMappingOptions, chownOp
 			pipeWriter = nil
 		}(f)
 
-		untar := b.untar(chownOpts, hasher)
+		untar := b.untar(chownOpts, hasher, dryRun)
 		err = untar(pipeReader, b.MountPoint)
 		if err == nil {
 			err = copyErr
@@ -224,9 +225,9 @@ func (b *Builder) copyFileWithTar(tarIDMappingOptions *IDMappingOptions, chownOp
 // our container or from another container, into our working container, mapping
 // permissions at read-time using the container's ID maps, with ownership at
 // write-time possibly overridden using the passed-in chownOpts
-func (b *Builder) copyWithTar(tarIDMappingOptions *IDMappingOptions, chownOpts *idtools.IDPair, hasher io.Writer) func(src, dest string) error {
+func (b *Builder) copyWithTar(tarIDMappingOptions *IDMappingOptions, chownOpts *idtools.IDPair, hasher io.Writer, dryRun bool) func(src, dest string) error {
 	tar := b.tarPath(tarIDMappingOptions)
-	untar := b.untar(chownOpts, hasher)
+	untar := b.untar(chownOpts, hasher, dryRun)
 	return func(src, dest string) error {
 		rc, err := tar(src)
 		if err != nil {
@@ -239,8 +240,22 @@ func (b *Builder) copyWithTar(tarIDMappingOptions *IDMappingOptions, chownOpts *
 // untarPath returns a function which extracts an archive in a specified
 // location into our working container, mapping permissions using the
 // container's ID maps, possibly overridden using the passed-in chownOpts
-func (b *Builder) untarPath(chownOpts *idtools.IDPair, hasher io.Writer) func(src, dest string) error {
+func (b *Builder) untarPath(chownOpts *idtools.IDPair, hasher io.Writer, dryRun bool) func(src, dest string) error {
 	convertedUIDMap, convertedGIDMap := convertRuntimeIDMaps(b.IDMappingOptions.UIDMap, b.IDMappingOptions.GIDMap)
+	if dryRun {
+		return func(src, dest string) error {
+			if hasher == nil {
+				hasher = ioutil.Discard
+			}
+			f, err := os.Open(src)
+			if err != nil {
+				return errors.Wrapf(err, "error opening %q", src)
+			}
+			defer f.Close()
+			_, err = io.Copy(hasher, f)
+			return err
+		}
+	}
 	return chrootarchive.UntarPathAndChown(chownOpts, hasher, convertedUIDMap, convertedGIDMap)
 }
 
@@ -272,7 +287,7 @@ func (b *Builder) tarPath(idMappingOptions *IDMappingOptions) func(path string)
 // untar returns a function which extracts an archive stream to a specified
 // location in the container's filesystem, mapping permissions using the
 // container's ID maps, possibly overridden using the passed-in chownOpts
-func (b *Builder) untar(chownOpts *idtools.IDPair, hasher io.Writer) func(tarArchive io.ReadCloser, dest string) error {
+func (b *Builder) untar(chownOpts *idtools.IDPair, hasher io.Writer, dryRun bool) func(tarArchive io.ReadCloser, dest string) error {
 	convertedUIDMap, convertedGIDMap := convertRuntimeIDMaps(b.IDMappingOptions.UIDMap, b.IDMappingOptions.GIDMap)
 	untarMappings := idtools.NewIDMappingsFromMaps(convertedUIDMap, convertedGIDMap)
 	options := &archive.TarOptions{
@@ -281,6 +296,14 @@ func (b *Builder) untar(chownOpts *idtools.IDPair, hasher io.Writer) func(tarArc
 		ChownOpts: chownOpts,
 	}
 	untar := chrootarchive.Untar
+	if dryRun {
+		untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
+			if _, err := io.Copy(ioutil.Discard, tarArchive); err != nil {
+				return errors.Wrapf(err, "error digesting tar stream")
+			}
+			return nil
+		}
+	}
 	if hasher != nil {
 		originalUntar := untar
 		untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {