root
/
buildah
mirror of https://github.com/containers/buildah.git
1
0
Fork 0
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #6178 from nalind/add-timestamp 

add: add a new --timestamp flag
This commit is contained in:
openshift-merge-bot[bot] 2025-05-28 19:16:00 +00:00 committed by GitHub
parent 5168237d2b b9a65a9d86
commit 9986534eea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 350 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
add.go
View File

@ -95,8 +95,13 @@ type AddAndCopyOptions struct {
// RetryDelay is how long to wait before retrying attempts to retrieve
// remote contents.
RetryDelay time.Duration
// Parents preserve parent directories of source content
// Parents specifies that we should preserve either all of the parent
// directories of source locations, or the ones which follow "/./" in
// the source paths for source locations which include such a
// component.
Parents bool
// Timestamp is a timestamp to override on all content as it is being read.
Timestamp *time.Time
}
// gitURLFragmentSuffix matches fragments to use as Git reference and build
@ -123,7 +128,7 @@ func sourceIsRemote(source string) bool {
}
// getURL writes a tar archive containing the named content
func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string, writer io.Writer, chmod *os.FileMode, srcDigest digest.Digest, certPath string, insecureSkipTLSVerify types.OptionalBool) error {
func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string, writer io.Writer, chmod *os.FileMode, srcDigest digest.Digest, certPath string, insecureSkipTLSVerify types.OptionalBool, timestamp *time.Time) error {
url, err := url.Parse(src)
if err != nil {
return err
@ -154,15 +159,19 @@ func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string,
name = path.Base(url.Path)
}
// If there's a date on the content, use it. If not, use the Unix epoch
// for compatibility.
// or a specified value for compatibility.
date := time.Unix(0, 0).UTC()
lastModified := response.Header.Get("Last-Modified")
if lastModified != "" {
d, err := time.Parse(time.RFC1123, lastModified)
if err != nil {
return fmt.Errorf("parsing last-modified time: %w", err)
if timestamp != nil {
date = timestamp.UTC()
} else {
lastModified := response.Header.Get("Last-Modified")
if lastModified != "" {
d, err := time.Parse(time.RFC1123, lastModified)
if err != nil {
return fmt.Errorf("parsing last-modified time %q: %w", lastModified, err)
}
date = d.UTC()
}
date = d
}
// Figure out the size of the content.
size := response.ContentLength
@ -532,6 +541,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
StripSetuidBit: options.StripSetuidBit,
StripSetgidBit: options.StripSetgidBit,
StripStickyBit: options.StripStickyBit,
Timestamp: options.Timestamp,
}
writer := io.WriteCloser(pipeWriter)
repositoryDir := filepath.Join(cloneDir, subdir)
@ -540,7 +550,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
} else {
go func() {
getErr = retry.IfNecessary(context.TODO(), func() error {
return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify)
return getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest, options.CertPath, options.InsecureSkipTLSVerify, options.Timestamp)
}, &retry.Options{
MaxRetry: options.MaxRetries,
Delay: options.RetryDelay,
@ -696,6 +706,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
StripSetgidBit: options.StripSetgidBit,
StripStickyBit: options.StripStickyBit,
Parents: options.Parents,
Timestamp: options.Timestamp,
}
getErr = copier.Get(contextDir, contextDir, getOptions, []string{globbedToGlobbable(globbed)}, writer)
closeErr = writer.Close()

14
cmd/buildah/addcopy.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@ -39,6 +40,7 @@ type addCopyResults struct {
retryDelay string
excludes []string
parents bool
timestamp string
}
func createCommand(addCopy string, desc string, short string, opts *addCopyResults) *cobra.Command {
@ -95,6 +97,7 @@ func applyFlagVars(flags *pflag.FlagSet, opts *addCopyResults) {
if err := flags.MarkHidden("signature-policy"); err != nil {
panic(fmt.Sprintf("error marking signature-policy as hidden: %v", err))
}
flags.StringVar(&opts.timestamp, "timestamp", "", "set timestamps on new content to `seconds` after the epoch")
}
func init() {
@ -235,6 +238,16 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
builder.ContentDigester.Restart()
var timestamp *time.Time
if iopts.timestamp != "" {
u, err := strconv.ParseInt(iopts.timestamp, 10, 64)
if err != nil {
return fmt.Errorf("parsing timestamp value %q: %w", iopts.timestamp, err)
}
t := time.Unix(u, 0).UTC()
timestamp = &t
}
options := buildah.AddAndCopyOptions{
Chmod: iopts.chmod,
Chown: iopts.chown,
@ -249,6 +262,7 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
InsecureSkipTLSVerify: systemContext.DockerInsecureSkipTLSVerify,
MaxRetries: iopts.retry,
Parents: iopts.parents,
Timestamp: timestamp,
}
if iopts.contextdir != "" {
var excludes []string

21
copier/copier.go
View File

@ -391,6 +391,7 @@ type GetOptions struct {
NoDerefSymlinks bool // don't follow symlinks when globs match them
IgnoreUnreadable bool // ignore errors reading items, instead of returning an error
NoCrossDevice bool // if a subdirectory is a mountpoint with a different device number, include it but skip its contents
Timestamp *time.Time // timestamp to force on all contents
}
// Get produces an archive containing items that match the specified glob
@ -1633,6 +1634,16 @@ func copierHandlerGetOne(srcfi os.FileInfo, symlinkTarget, name, contentPath str
if options.Rename != nil {
hdr.Name = handleRename(options.Rename, hdr.Name)
}
if options.Timestamp != nil {
timestamp := options.Timestamp.UTC()
hdr.ModTime = timestamp
if !hdr.AccessTime.IsZero() {
hdr.AccessTime = timestamp
}
if !hdr.ChangeTime.IsZero() {
hdr.ChangeTime = timestamp
}
}
if err = tw.WriteHeader(hdr); err != nil {
return fmt.Errorf("writing tar header from %q to pipe: %w", contentPath, err)
}
@ -1711,6 +1722,16 @@ func copierHandlerGetOne(srcfi os.FileInfo, symlinkTarget, name, contentPath str
}
defer f.Close()
}
if options.Timestamp != nil {
timestamp := options.Timestamp.UTC()
hdr.ModTime = timestamp
if !hdr.AccessTime.IsZero() {
hdr.AccessTime = timestamp
}
if !hdr.ChangeTime.IsZero() {
hdr.ChangeTime = timestamp
}
}
// output the header
if err = tw.WriteHeader(hdr); err != nil {
return fmt.Errorf("writing header for %s (%s): %w", contentPath, hdr.Name, err)

20
copier/copier_test.go
View File

@ -177,7 +177,8 @@ type enumeratedFile struct {
}
var (
testDate = time.Unix(1485449953, 0)
testDate = time.Unix(1485449953, 0)
secondTestDate = time.Unix(1485449953*2, 0)
uid = os.Getuid()
@ -890,6 +891,7 @@ func testGetMultiple(t *testing.T) {
renames map[string]string
noDerefSymlinks bool
parents bool
timestamp *time.Time
}
getTestArchives := []struct {
name string
@ -997,6 +999,16 @@ func testGetMultiple(t *testing.T) {
"subdir-f/hlink-b", // from subdir-e
},
},
{
name: "timestamped",
pattern: "file*",
items: []string{
"file-0",
"file-a",
"file-b",
},
timestamp: &secondTestDate,
},
{
name: "dot-with-wildcard-includes-and-excludes",
pattern: ".",
@ -1520,6 +1532,7 @@ func testGetMultiple(t *testing.T) {
Rename: testCase.renames,
NoDerefSymlinks: testCase.noDerefSymlinks,
Parents: testCase.parents,
Timestamp: testCase.timestamp,
}
t.Run(fmt.Sprintf("topdir=%s,archive=%s,case=%s,pattern=%s", topdir, testArchive.name, testCase.name, testCase.pattern), func(t *testing.T) {
@ -1535,15 +1548,18 @@ func testGetMultiple(t *testing.T) {
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
getErr = Get(root, topdir, getOptions, []string{testCase.pattern}, pipeWriter)
pipeWriter.Close()
wg.Done()
}()
tr := tar.NewReader(pipeReader)
hdr, err := tr.Next()
actualContents := []string{}
for err == nil {
actualContents = append(actualContents, filepath.FromSlash(hdr.Name))
if testCase.timestamp != nil {
assert.Truef(t, testCase.timestamp.Equal(hdr.ModTime), "timestamp was supposed to be forced for %q", hdr.Name)
}
hdr, err = tr.Next()
}
pipeReader.Close()

9
docs/buildah-add.1.md
View File

@ -83,6 +83,15 @@ from registries or retrieving content from HTTPS URLs.
Defaults to `2s`.
**--timestamp** *seconds*
Set the timestamp ("mtime") for added content to exactly this number of seconds
since the epoch (Unix time 0, i.e., 00:00:00 UTC on 1 January 1970) to help
allow for deterministic builds.
The destination directory into which the content is being copied will most
likely reflect the time at which the content was added to it.
**--tls-verify** *bool-value*
Require verification of certificates when retrieving sources from HTTPS

9
docs/buildah-copy.1.md
View File

@ -87,6 +87,15 @@ Duration of delay between retry attempts in case of failure when performing pull
Defaults to `2s`.
**--timestamp** *seconds*
Set the timestamp ("mtime") for added content to exactly this number of seconds
since the epoch (Unix time 0, i.e., 00:00:00 UTC on 1 January 1970) to help
allow for deterministic builds.
The destination directory into which the content is being copied will most
likely reflect the time at which the content was added to it.
**--tls-verify** *bool-value*
Require verification of certificates when pulling images referred to with the

58
tests/add.bats
View File

@ -334,3 +334,61 @@ stuff/mystuff"
# Only that the add was actually successful.
run_buildah add $cid /usr/libexec/catatonit/catatonit /catatonit
}
@test "add-with-timestamp" {
_prefetch busybox
url=https://raw.githubusercontent.com/containers/buildah/main/tests/bud/from-scratch/Dockerfile
timestamp=60
mkdir -p $TEST_SCRATCH_DIR/context
createrandom $TEST_SCRATCH_DIR/context/randomfile1
createrandom $TEST_SCRATCH_DIR/context/randomfile2
run_buildah from -q busybox
cid="$output"
# Add the content with more or less contemporary timestamps.
run_buildah copy "$cid" $TEST_SCRATCH_DIR/context/randomfile* /default
# Add a second copy that should get the same contemporary timestamps.
run_buildah copy "$cid" $TEST_SCRATCH_DIR/context/randomfile* /default2
# Add a third copy that we explicitly force timestamps for.
run_buildah copy --timestamp=$timestamp "$cid" $TEST_SCRATCH_DIR/context/randomfile* /explicit
run_buildah add --timestamp=$timestamp "$cid" "$url" /explicit
# Add a fourth copy that we forced the timestamps for out of band.
cp -v "${BUDFILES}"/from-scratch/Dockerfile $TEST_SCRATCH_DIR/context/
tar -cf $TEST_SCRATCH_DIR/tarball -C $TEST_SCRATCH_DIR/context randomfile1 randomfile2 Dockerfile
touch -d @$timestamp $TEST_SCRATCH_DIR/context/*
run_buildah copy "$cid" $TEST_SCRATCH_DIR/context/* /touched
# Add a fifth copy that we forced the timestamps for, from an archive.
run_buildah add --timestamp=$timestamp "$cid" $TEST_SCRATCH_DIR/tarball /archive
# Build the script to verify this inside of the rootfs.
cat > $TEST_SCRATCH_DIR/context/check-dates.sh <<-EOF
# Okay, at this point, default, default2, explicit, touched, and archive
# should all contain randomfile1, randomfile2, and Dockerfile.
# The copies in default and default2 should have contemporary timestamps for
# the random files, and a server-supplied timestamp or the epoch for the
# Dockerfile.
# The copies in explicit, touched, and archive should all have the same
# very old timestamps.
touch -d @$timestamp /tmp/reference-file
for f in /default/* /default2/* ; do
if test \$f -ot /tmp/reference-file ; then
echo expected \$f to be newer than /tmp/reference-file, but it was not
ls -l \$f /tmp/reference-file
exit 1
fi
done
for f in /explicit/* /touched/* /archive/* ; do
if test \$f -nt /tmp/reference-file ; then
echo expected \$f and /tmp/reference-file to have the same datestamp
ls -l \$f /tmp/reference-file
exit 1
fi
if test \$f -ot /tmp/reference-file ; then
echo expected \$f and /tmp/reference-file to have the same datestamp
ls -l \$f /tmp/reference-file
exit 1
fi
done
exit 0
EOF
run_buildah copy --chmod=0755 "$cid" $TEST_SCRATCH_DIR/context/check-dates.sh /
run_buildah run "$cid" sh -x /check-dates.sh
}

200
tests/helpers.bash
View File

@ -727,6 +727,9 @@ function skip_if_no_docker() {
fi
}
########################
# skip_if_no_unshare #
########################
function skip_if_no_unshare() {
run which ${UNSHARE_BINARY:-unshare}
if [[ $status -ne 0 ]]; then
@ -749,6 +752,9 @@ function skip_if_no_unshare() {
fi
}
######################
# start_git_daemon #
######################
function start_git_daemon() {
daemondir=${TEST_SCRATCH_DIR}/git-daemon
mkdir -p ${daemondir}/repo
@ -772,6 +778,9 @@ function start_git_daemon() {
GITPORT=$(cat ${TEST_SCRATCH_DIR}/git-daemon/port)
}
#####################
# stop_git_daemon #
#####################
function stop_git_daemon() {
if test -s ${TEST_SCRATCH_DIR}/git-daemon/pid ; then
kill $(cat ${TEST_SCRATCH_DIR}/git-daemon/pid)
@ -779,6 +788,9 @@ function stop_git_daemon() {
fi
}
####################
# start_registry #
####################
# Bring up a registry server using buildah with vfs and chroot as a cheap
# substitute for podman, accessible only to user $1 using password $2 on the
# local system at a dynamically-allocated port.
@ -872,6 +884,9 @@ auth:
fi
}
###################
# stop_registry #
###################
function stop_registry() {
if test -n "${REGISTRY_PID}" ; then
kill "${REGISTRY_PID}"
@ -885,3 +900,188 @@ function stop_registry() {
fi
unset REGISTRY_DIR
}
###############################
# oci_image_manifest_digest #
###############################
# prints the digest of the form "sha256:xxx" of the manifest for the main image
# in an OCI layout in "$1"
function oci_image_manifest_digest() {
run jq -r '.manifests[0].digest' "$1"/index.json
assert $status = 0 "looking for the digest of the image manifest"
assert "$output" != ""
echo "$output"
}
#############################
# oci_image_config_digest #
#############################
# prints the digest of the form "sha256:xxx" of the config blob for the main
# image in an OCI layout in "$1"
function oci_image_config_digest() {
local digest=$(oci_image_manifest_digest "$1")
local alg=${digest%%:*}
local val=${digest##*:}
run jq -r '.config.digest' "$1"/blobs/"$alg"/"$val"
assert $status = 0 "looking for the digest of the image config"
assert "$output" != ""
echo "$output"
}
######################
# oci_image_config #
######################
# prints the relative path of the config blob for the main image in an OCI
# layout in "$1"
function oci_image_config() {
local diff_id=$(oci_image_config_digest "$@")
local alg=${diff_id%%:*}
local val=${diff_id##*:}
echo blobs/"$alg"/"$val"
}
########################
# oci_image_diff_ids #
########################
# prints the list of digests of the diff IDs for the main image in an OCI
# layout in "$1"
function oci_image_diff_ids() {
local digest=$(oci_image_config_digest "$1")
local alg=${digest%%:*}
local val=${digest##*:}
run jq -r '.rootfs.diff_ids[]' "$1"/blobs/"$alg"/"$val"
assert $status = 0 "looking for the diff IDs in the image config"
assert "$output" != ""
echo "$output"
}
#######################
# oci_image_diff_id #
#######################
# prints a single diff ID for the main image in an OCI layout in "$1", choosing
# which one to print based on an index and arithmetic operands passed in
# subsequent arguments
function oci_image_diff_id() {
local diff_ids=($(oci_image_diff_ids "$1"))
shift
case "$*" in
-*) echo ${diff_ids[$((${#diff_ids[@]} "$@"))]} ;;
*) echo ${diff_ids[$(("$@"))]} ;;
esac
}
############################
# oci_image_last_diff_id #
############################
# prints the diff ID of the most recent layer for the main image in an OCI
# layout in "$1"
function oci_image_last_diff_id() {
local diff_id=($(oci_image_diff_id "$1" - 1))
echo "$diff_id"
}
####################
# oci_image_diff #
####################
# prints the relative path of a single layer diff for the main image in an OCI
# layout in "$1", choosing which one to print based on an index and arithmetic
# operands passed in subsequent arguments
function oci_image_diff() {
local diff_id=$(oci_image_diff_id "$@")
local alg=${diff_id%%:*}
local val=${diff_id##*:}
echo blobs/"$alg"/"$val"
}
#########################
# oci_image_last_diff #
#########################
# prints the relative path of the most recent layer for the main image in an
# OCI layout in "$1"
function oci_image_last_diff() {
local output=$(oci_image_diff "$1" - 1)
echo "$output"
}
#############################
# dir_image_config_digest #
#############################
# prints the digest of the form "sha256:xxx" of the config blob for the "dir"
# image in "$1"
function dir_image_config_digest() {
run jq -r '.config.digest' "$1"/manifest.json
assert $status = 0 "looking for the digest of the image config"
assert "$output" != ""
echo "$output"
}
########################
# dir_image_diff_ids #
########################
# prints the list of digests of the diff IDs for the "dir" image in "$1"
function dir_image_diff_ids() {
local digest=$(dir_image_config_digest "$1")
local alg=${digest%%:*}
local val=${digest##*:}
run jq -r '.rootfs.diff_ids[]' "$1"/"$val"
assert $status = 0 "looking for the diff IDs in the image config"
assert "$output" != ""
echo "$output"
}
#######################
# dir_image_diff_id #
#######################
# prints a single diff ID for the "dir" image in "$1", choosing which one to
# print based on an index and arithmetic operands passed in subsequent
# arguments
function dir_image_diff_id() {
local diff_ids=($(dir_image_diff_ids "$1"))
shift
case "$*" in
-*) echo ${diff_ids[$((${#diff_ids[@]} "$@"))]} ;;
*) echo ${diff_ids[$(("$@"))]} ;;
esac
}
############################
# dir_image_last_diff_id #
############################
# prints the diff ID of the most recent layer for "dir" image in "$1"
function dir_image_last_diff_id() {
local diff_id=($(dir_image_diff_id "$1" - 1))
echo "$diff_id"
}
######################
# dir_image_config #
######################
# prints the relative path of the config blob for the "dir" image in "$1"
function dir_image_config() {
local diff_id=$(dir_image_config_digest "$@")
local alg=${diff_id%%:*}
local val=${diff_id##*:}
echo "$val"
}
####################
# dir_image_diff #
####################
# prints the relative path of a single layer diff for the "dir" image in "$1",
# choosing which one to print based on an index and arithmetic operands passed
# in subsequent arguments
function dir_image_diff() {
local diff_id=$(dir_image_diff_id "$@")
local alg=${diff_id%%:*}
local val=${diff_id##*:}
echo "$val"
}
#########################
# dir_image_last_diff #
#########################
# prints the relative path of the most recent layer for "dir" image in "$1"
function dir_image_last_diff() {
local output=$(dir_image_diff "$1")
echo "$output"
}