Merge pull request #6178 from nalind/add-timestamp

add: add a new --timestamp flag

add.go

@@ -95,8 +95,13 @@ type AddAndCopyOptions struct {
 	// RetryDelay is how long to wait before retrying attempts to retrieve
 	// remote contents.
 	RetryDelay time.Duration
-	// Parents preserve parent directories of source content
+	// Parents specifies that we should preserve either all of the parent
+	// directories of source locations, or the ones which follow "/./" in
+	// the source paths for source locations which include such a
+	// component.
 	Parents bool
+	// Timestamp is a timestamp to override on all content as it is being read.
+	Timestamp *time.Time
 }
 
 // gitURLFragmentSuffix matches fragments to use as Git reference and build
@@ -123,7 +128,7 @@ func sourceIsRemote(source string) bool {
 }
 
 // getURL writes a tar archive containing the named content
-func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string, writer io.Writer, chmod *os.FileMode, srcDigest digest.Digest, certPath string, insecureSkipTLSVerify types.OptionalBool) error {
+func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string, writer io.Writer, chmod *os.FileMode, srcDigest digest.Digest, certPath string, insecureSkipTLSVerify types.OptionalBool, timestamp *time.Time) error {
 	url, err := url.Parse(src)
 	if err != nil {
 		return err
@@ -154,15 +159,19 @@ func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string,
 		name = path.Base(url.Path)
 	}
 	// If there's a date on the content, use it.  If not, use the Unix epoch
-	// for compatibility.
+	// or a specified value for compatibility.
 	date := time.Unix(0, 0).UTC()
-	lastModified := response.Header.Get("Last-Modified")
+	if timestamp != nil {
-	if lastModified != "" {
+		date = timestamp.UTC()
-		d, err := time.Parse(time.RFC1123, lastModified)
+	} else {
-		if err != nil {
+		lastModified := response.Header.Get("Last-Modified")
-			return fmt.Errorf("parsing last-modified time: %w", err)
+		if lastModified != "" {
+			d, err := time.Parse(time.RFC1123, lastModified)
+			if err != nil {
+				return fmt.Errorf("parsing last-modified time %q: %w", lastModified, err)
+			}
+			date = d.UTC()
 		}
-		date = d
 	}
 	// Figure out the size of the content.
 	size := response.ContentLength
@@ -532,6 +541,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 						StripSetuidBit: options.StripSetuidBit,
 						StripSetgidBit: options.StripSetgidBit,
 						StripStickyBit: options.StripStickyBit,
+						Timestamp:      options.Timestamp,
 					}
 					writer := io.WriteCloser(pipeWriter)
 					repositoryDir := filepath.Join(cloneDir, subdir)
@@ -540,7 +550,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 			} else {
 				go func() {
 					getErr = retry.IfNecessary(context.TODO(), func() error {
-						return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify)
+						return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify, options.Timestamp)
 					}, &retry.Options{
 						MaxRetry: options.MaxRetries,
 						Delay:    options.RetryDelay,
@@ -696,6 +706,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 					StripSetgidBit: options.StripSetgidBit,
 					StripStickyBit: options.StripStickyBit,
 					Parents:        options.Parents,
+					Timestamp:      options.Timestamp,
 				}
 				getErr = copier.Get(contextDir, contextDir, getOptions, []string{globbedToGlobbable(globbed)}, writer)
 				closeErr = writer.Close()

cmd/buildah/addcopy.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -39,6 +40,7 @@ type addCopyResults struct {
 	retryDelay       string
 	excludes         []string
 	parents          bool
+	timestamp        string
 }
 
 func createCommand(addCopy string, desc string, short string, opts *addCopyResults) *cobra.Command {
@@ -95,6 +97,7 @@ func applyFlagVars(flags *pflag.FlagSet, opts *addCopyResults) {
 	if err := flags.MarkHidden("signature-policy"); err != nil {
 		panic(fmt.Sprintf("error marking signature-policy as hidden: %v", err))
 	}
+	flags.StringVar(&opts.timestamp, "timestamp", "", "set timestamps on new content to `seconds` after the epoch")
 }
 
 func init() {
@@ -235,6 +238,16 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 
 	builder.ContentDigester.Restart()
 
+	var timestamp *time.Time
+	if iopts.timestamp != "" {
+		u, err := strconv.ParseInt(iopts.timestamp, 10, 64)
+		if err != nil {
+			return fmt.Errorf("parsing timestamp value %q: %w", iopts.timestamp, err)
+		}
+		t := time.Unix(u, 0).UTC()
+		timestamp = &t
+	}
+
 	options := buildah.AddAndCopyOptions{
 		Chmod:             iopts.chmod,
 		Chown:             iopts.chown,
@@ -249,6 +262,7 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 		InsecureSkipTLSVerify: systemContext.DockerInsecureSkipTLSVerify,
 		MaxRetries:            iopts.retry,
 		Parents:               iopts.parents,
+		Timestamp:             timestamp,
 	}
 	if iopts.contextdir != "" {
 		var excludes []string

copier/copier.go

@@ -391,6 +391,7 @@ type GetOptions struct {
 	NoDerefSymlinks    bool              // don't follow symlinks when globs match them
 	IgnoreUnreadable   bool              // ignore errors reading items, instead of returning an error
 	NoCrossDevice      bool              // if a subdirectory is a mountpoint with a different device number, include it but skip its contents
+	Timestamp          *time.Time        // timestamp to force on all contents
 }
 
 // Get produces an archive containing items that match the specified glob
@@ -1633,6 +1634,16 @@ func copierHandlerGetOne(srcfi os.FileInfo, symlinkTarget, name, contentPath str
 				if options.Rename != nil {
 					hdr.Name = handleRename(options.Rename, hdr.Name)
 				}
+				if options.Timestamp != nil {
+					timestamp := options.Timestamp.UTC()
+					hdr.ModTime = timestamp
+					if !hdr.AccessTime.IsZero() {
+						hdr.AccessTime = timestamp
+					}
+					if !hdr.ChangeTime.IsZero() {
+						hdr.ChangeTime = timestamp
+					}
+				}
 				if err = tw.WriteHeader(hdr); err != nil {
 					return fmt.Errorf("writing tar header from %q to pipe: %w", contentPath, err)
 				}
@@ -1711,6 +1722,16 @@ func copierHandlerGetOne(srcfi os.FileInfo, symlinkTarget, name, contentPath str
 		}
 		defer f.Close()
 	}
+	if options.Timestamp != nil {
+		timestamp := options.Timestamp.UTC()
+		hdr.ModTime = timestamp
+		if !hdr.AccessTime.IsZero() {
+			hdr.AccessTime = timestamp
+		}
+		if !hdr.ChangeTime.IsZero() {
+			hdr.ChangeTime = timestamp
+		}
+	}
 	// output the header
 	if err = tw.WriteHeader(hdr); err != nil {
 		return fmt.Errorf("writing header for %s (%s): %w", contentPath, hdr.Name, err)

copier/copier_test.go

@@ -177,7 +177,8 @@ type enumeratedFile struct {
 }
 
 var (
-	testDate = time.Unix(1485449953, 0)
+	testDate       = time.Unix(1485449953, 0)
+	secondTestDate = time.Unix(1485449953*2, 0)
 
 	uid = os.Getuid()
 
@@ -890,6 +891,7 @@ func testGetMultiple(t *testing.T) {
 		renames            map[string]string
 		noDerefSymlinks    bool
 		parents            bool
+		timestamp          *time.Time
 	}
 	getTestArchives := []struct {
 		name              string
@@ -997,6 +999,16 @@ func testGetMultiple(t *testing.T) {
 						"subdir-f/hlink-b", // from subdir-e
 					},
 				},
+				{
+					name:    "timestamped",
+					pattern: "file*",
+					items: []string{
+						"file-0",
+						"file-a",
+						"file-b",
+					},
+					timestamp: &secondTestDate,
+				},
 				{
 					name:    "dot-with-wildcard-includes-and-excludes",
 					pattern: ".",
@@ -1520,6 +1532,7 @@ func testGetMultiple(t *testing.T) {
 					Rename:             testCase.renames,
 					NoDerefSymlinks:    testCase.noDerefSymlinks,
 					Parents:            testCase.parents,
+					Timestamp:          testCase.timestamp,
 				}
 
 				t.Run(fmt.Sprintf("topdir=%s,archive=%s,case=%s,pattern=%s", topdir, testArchive.name, testCase.name, testCase.pattern), func(t *testing.T) {
@@ -1535,15 +1548,18 @@ func testGetMultiple(t *testing.T) {
 					var wg sync.WaitGroup
 					wg.Add(1)
 					go func() {
+						defer wg.Done()
 						getErr = Get(root, topdir, getOptions, []string{testCase.pattern}, pipeWriter)
 						pipeWriter.Close()
-						wg.Done()
 					}()
 					tr := tar.NewReader(pipeReader)
 					hdr, err := tr.Next()
 					actualContents := []string{}
 					for err == nil {
 						actualContents = append(actualContents, filepath.FromSlash(hdr.Name))
+						if testCase.timestamp != nil {
+							assert.Truef(t, testCase.timestamp.Equal(hdr.ModTime), "timestamp was supposed to be forced for %q", hdr.Name)
+						}
 						hdr, err = tr.Next()
 					}
 					pipeReader.Close()

docs/buildah-add.1.md

@@ -83,6 +83,15 @@ from registries or retrieving content from HTTPS URLs.
 
 Defaults to `2s`.
 
+**--timestamp** *seconds*
+
+Set the timestamp ("mtime") for added content to exactly this number of seconds
+since the epoch (Unix time 0, i.e., 00:00:00 UTC on 1 January 1970) to help
+allow for deterministic builds.
+
+The destination directory into which the content is being copied will most
+likely reflect the time at which the content was added to it.
+
 **--tls-verify** *bool-value*
 
 Require verification of certificates when retrieving sources from HTTPS

docs/buildah-copy.1.md

@@ -87,6 +87,15 @@ Duration of delay between retry attempts in case of failure when performing pull
 
 Defaults to `2s`.
 
+**--timestamp** *seconds*
+
+Set the timestamp ("mtime") for added content to exactly this number of seconds
+since the epoch (Unix time 0, i.e., 00:00:00 UTC on 1 January 1970) to help
+allow for deterministic builds.
+
+The destination directory into which the content is being copied will most
+likely reflect the time at which the content was added to it.
+
 **--tls-verify** *bool-value*
 
 Require verification of certificates when pulling images referred to with the

tests/add.bats

@@ -334,3 +334,61 @@ stuff/mystuff"
     # Only that the add was actually successful.
     run_buildah add $cid /usr/libexec/catatonit/catatonit /catatonit
 }
+
+@test "add-with-timestamp" {
+  _prefetch busybox
+  url=https://raw.githubusercontent.com/containers/buildah/main/tests/bud/from-scratch/Dockerfile
+  timestamp=60
+  mkdir -p $TEST_SCRATCH_DIR/context
+  createrandom $TEST_SCRATCH_DIR/context/randomfile1
+  createrandom $TEST_SCRATCH_DIR/context/randomfile2
+  run_buildah from -q busybox
+  cid="$output"
+  # Add the content with more or less contemporary timestamps.
+  run_buildah copy "$cid" $TEST_SCRATCH_DIR/context/randomfile* /default
+  # Add a second copy that should get the same contemporary timestamps.
+  run_buildah copy "$cid" $TEST_SCRATCH_DIR/context/randomfile* /default2
+  # Add a third copy that we explicitly force timestamps for.
+  run_buildah copy --timestamp=$timestamp "$cid" $TEST_SCRATCH_DIR/context/randomfile* /explicit
+  run_buildah add --timestamp=$timestamp "$cid" "$url" /explicit
+  # Add a fourth copy that we forced the timestamps for out of band.
+  cp -v "${BUDFILES}"/from-scratch/Dockerfile $TEST_SCRATCH_DIR/context/
+  tar -cf $TEST_SCRATCH_DIR/tarball -C $TEST_SCRATCH_DIR/context randomfile1 randomfile2 Dockerfile
+  touch -d @$timestamp $TEST_SCRATCH_DIR/context/*
+  run_buildah copy "$cid" $TEST_SCRATCH_DIR/context/* /touched
+  # Add a fifth copy that we forced the timestamps for, from an archive.
+  run_buildah add --timestamp=$timestamp "$cid" $TEST_SCRATCH_DIR/tarball /archive
+  # Build the script to verify this inside of the rootfs.
+  cat > $TEST_SCRATCH_DIR/context/check-dates.sh <<-EOF
+  # Okay, at this point, default, default2, explicit, touched, and archive
+  # should all contain randomfile1, randomfile2, and Dockerfile.
+  # The copies in default and default2 should have contemporary timestamps for
+  # the random files, and a server-supplied timestamp or the epoch for the
+  # Dockerfile.
+  # The copies in explicit, touched, and archive should all have the same
+  # very old timestamps.
+  touch -d @$timestamp /tmp/reference-file
+  for f in /default/* /default2/* ; do
+    if test \$f -ot /tmp/reference-file ; then
+      echo expected \$f to be newer than /tmp/reference-file, but it was not
+      ls -l \$f /tmp/reference-file
+      exit 1
+    fi
+  done
+  for f in /explicit/* /touched/* /archive/* ; do
+    if test \$f -nt /tmp/reference-file ; then
+      echo expected \$f and /tmp/reference-file to have the same datestamp
+      ls -l \$f /tmp/reference-file
+      exit 1
+    fi
+    if test \$f -ot /tmp/reference-file ; then
+      echo expected \$f and /tmp/reference-file to have the same datestamp
+      ls -l \$f /tmp/reference-file
+      exit 1
+    fi
+  done
+  exit 0
+EOF
+  run_buildah copy --chmod=0755 "$cid" $TEST_SCRATCH_DIR/context/check-dates.sh /
+  run_buildah run "$cid" sh -x /check-dates.sh
+}

tests/helpers.bash

@@ -727,6 +727,9 @@ function skip_if_no_docker() {
   fi
 }
 
+########################
+#  skip_if_no_unshare  #
+########################
 function skip_if_no_unshare() {
   run which ${UNSHARE_BINARY:-unshare}
   if [[ $status -ne 0 ]]; then
@@ -749,6 +752,9 @@ function skip_if_no_unshare() {
   fi
 }
 
+######################
+#  start_git_daemon  #
+######################
 function start_git_daemon() {
   daemondir=${TEST_SCRATCH_DIR}/git-daemon
   mkdir -p ${daemondir}/repo
@@ -772,6 +778,9 @@ function start_git_daemon() {
   GITPORT=$(cat ${TEST_SCRATCH_DIR}/git-daemon/port)
 }
 
+#####################
+#  stop_git_daemon  #
+#####################
 function stop_git_daemon() {
   if test -s ${TEST_SCRATCH_DIR}/git-daemon/pid ; then
     kill $(cat ${TEST_SCRATCH_DIR}/git-daemon/pid)
@@ -779,6 +788,9 @@ function stop_git_daemon() {
   fi
 }
 
+####################
+#  start_registry  #
+####################
 # Bring up a registry server using buildah with vfs and chroot as a cheap
 # substitute for podman, accessible only to user $1 using password $2 on the
 # local system at a dynamically-allocated port.
@@ -872,6 +884,9 @@ auth:
   fi
 }
 
+###################
+#  stop_registry  #
+###################
 function stop_registry() {
   if test -n "${REGISTRY_PID}" ; then
     kill "${REGISTRY_PID}"
@@ -885,3 +900,188 @@ function stop_registry() {
   fi
   unset REGISTRY_DIR
 }
+
+###############################
+#  oci_image_manifest_digest  #
+###############################
+# prints the digest of the form "sha256:xxx" of the manifest for the main image
+# in an OCI layout in "$1"
+function oci_image_manifest_digest() {
+  run jq -r '.manifests[0].digest' "$1"/index.json
+  assert $status = 0 "looking for the digest of the image manifest"
+  assert "$output" != ""
+  echo "$output"
+}
+
+#############################
+#  oci_image_config_digest  #
+#############################
+# prints the digest of the form "sha256:xxx" of the config blob for the main
+# image in an OCI layout in "$1"
+function oci_image_config_digest() {
+  local digest=$(oci_image_manifest_digest "$1")
+  local alg=${digest%%:*}
+  local val=${digest##*:}
+  run jq -r '.config.digest' "$1"/blobs/"$alg"/"$val"
+  assert $status = 0 "looking for the digest of the image config"
+  assert "$output" != ""
+  echo "$output"
+}
+
+######################
+#  oci_image_config  #
+######################
+# prints the relative path of the config blob for the main image in an OCI
+# layout in "$1"
+function oci_image_config() {
+  local diff_id=$(oci_image_config_digest "$@")
+  local alg=${diff_id%%:*}
+  local val=${diff_id##*:}
+  echo blobs/"$alg"/"$val"
+}
+
+########################
+#  oci_image_diff_ids  #
+########################
+# prints the list of digests of the diff IDs for the main image in an OCI
+# layout in "$1"
+function oci_image_diff_ids() {
+  local digest=$(oci_image_config_digest "$1")
+  local alg=${digest%%:*}
+  local val=${digest##*:}
+  run jq -r '.rootfs.diff_ids[]' "$1"/blobs/"$alg"/"$val"
+  assert $status = 0 "looking for the diff IDs in the image config"
+  assert "$output" != ""
+  echo "$output"
+}
+
+#######################
+#  oci_image_diff_id  #
+#######################
+# prints a single diff ID for the main image in an OCI layout in "$1", choosing
+# which one to print based on an index and arithmetic operands passed in
+# subsequent arguments
+function oci_image_diff_id() {
+  local diff_ids=($(oci_image_diff_ids "$1"))
+  shift
+  case "$*" in
+      -*) echo ${diff_ids[$((${#diff_ids[@]} "$@"))]} ;;
+      *) echo ${diff_ids[$(("$@"))]} ;;
+  esac
+}
+
+############################
+#  oci_image_last_diff_id  #
+############################
+# prints the diff ID of the most recent layer for the main image in an OCI
+# layout in "$1"
+function oci_image_last_diff_id() {
+  local diff_id=($(oci_image_diff_id "$1" - 1))
+  echo "$diff_id"
+}
+
+####################
+#  oci_image_diff  #
+####################
+# prints the relative path of a single layer diff for the main image in an OCI
+# layout in "$1", choosing which one to print based on an index and arithmetic
+# operands passed in subsequent arguments
+function oci_image_diff() {
+  local diff_id=$(oci_image_diff_id "$@")
+  local alg=${diff_id%%:*}
+  local val=${diff_id##*:}
+  echo blobs/"$alg"/"$val"
+}
+
+#########################
+#  oci_image_last_diff  #
+#########################
+# prints the relative path of the most recent layer for the main image in an
+# OCI layout in "$1"
+function oci_image_last_diff() {
+  local output=$(oci_image_diff "$1" - 1)
+  echo "$output"
+}
+
+#############################
+#  dir_image_config_digest  #
+#############################
+# prints the digest of the form "sha256:xxx" of the config blob for the "dir"
+# image in "$1"
+function dir_image_config_digest() {
+  run jq -r '.config.digest' "$1"/manifest.json
+  assert $status = 0 "looking for the digest of the image config"
+  assert "$output" != ""
+  echo "$output"
+}
+
+########################
+#  dir_image_diff_ids  #
+########################
+# prints the list of digests of the diff IDs for the "dir" image in "$1"
+function dir_image_diff_ids() {
+  local digest=$(dir_image_config_digest "$1")
+  local alg=${digest%%:*}
+  local val=${digest##*:}
+  run jq -r '.rootfs.diff_ids[]' "$1"/"$val"
+  assert $status = 0 "looking for the diff IDs in the image config"
+  assert "$output" != ""
+  echo "$output"
+}
+
+#######################
+#  dir_image_diff_id  #
+#######################
+# prints a single diff ID for the "dir" image in "$1", choosing which one to
+# print based on an index and arithmetic operands passed in subsequent
+# arguments
+function dir_image_diff_id() {
+  local diff_ids=($(dir_image_diff_ids "$1"))
+  shift
+  case "$*" in
+      -*) echo ${diff_ids[$((${#diff_ids[@]} "$@"))]} ;;
+      *) echo ${diff_ids[$(("$@"))]} ;;
+  esac
+}
+
+############################
+#  dir_image_last_diff_id  #
+############################
+# prints the diff ID of the most recent layer for "dir" image in "$1"
+function dir_image_last_diff_id() {
+  local diff_id=($(dir_image_diff_id "$1" - 1))
+  echo "$diff_id"
+}
+
+######################
+#  dir_image_config  #
+######################
+# prints the relative path of the config blob for the "dir" image in "$1"
+function dir_image_config() {
+  local diff_id=$(dir_image_config_digest "$@")
+  local alg=${diff_id%%:*}
+  local val=${diff_id##*:}
+  echo "$val"
+}
+
+####################
+#  dir_image_diff  #
+####################
+# prints the relative path of a single layer diff for the "dir" image in "$1",
+# choosing which one to print based on an index and arithmetic operands passed
+# in subsequent arguments
+function dir_image_diff() {
+  local diff_id=$(dir_image_diff_id "$@")
+  local alg=${diff_id%%:*}
+  local val=${diff_id##*:}
+  echo "$val"
+}
+
+#########################
+#  dir_image_last_diff  #
+#########################
+# prints the relative path of the most recent layer for "dir" image in "$1"
+function dir_image_last_diff() {
+  local output=$(dir_image_diff "$1")
+  echo "$output"
+}