root
/
buildah
mirror of https://github.com/containers/buildah.git
1
0
Fork 0
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Use content digests in ADD/COPY history entries 

Use digests of the added content in history entries that we create for
ADD and COPY instructions, tightening up cache checking just a little
bit more.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

Closes: #1792
Approved by: TomSweeneyRedHat
This commit is contained in:
Nalin Dahyabhai 2019-08-09 16:21:24 -04:00 committed by Atomic Bot
parent db2b3e48ac
commit ebf6f518d0
9 changed files with 306 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
add.go
View File

@ -49,14 +49,21 @@ type AddAndCopyOptions struct {
// addURL copies the contents of the source URL to the destination. This is
// its own function so that deferred closes happen after we're done pulling
// down each item of potentially many.
func addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer, dryRun bool) error {
func (b *Builder) addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer, dryRun bool) error {
resp, err := http.Get(srcurl)
if err != nil {
return errors.Wrapf(err, "error getting %q", srcurl)
}
defer resp.Body.Close()
thisWriter := hasher
thisHasher := hasher
if thisHasher != nil && b.ContentDigester.Hash() != nil {
thisHasher = io.MultiWriter(thisHasher, b.ContentDigester.Hash())
}
if thisHasher == nil {
thisHasher = b.ContentDigester.Hash()
}
thisWriter := thisHasher
if !dryRun {
logrus.Debugf("saving %q to %q", srcurl, destination)
@ -84,12 +91,9 @@ func addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer,
logrus.Debugf("error setting permissions on %q: %v", destination, err2)
}
}()
if thisWriter != nil {
thisWriter = io.MultiWriter(f, thisWriter)
} else {
thisWriter = f
}
thisWriter = io.MultiWriter(f, thisWriter)
}
n, err := io.Copy(thisWriter, resp.Body)
if err != nil {
return errors.Wrapf(err, "error reading contents for %q from %q", destination, srcurl)
@ -172,7 +176,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
copyFileWithTar := b.copyFileWithTar(options.IDMappingOptions, &containerOwner, options.Hasher, options.DryRun)
copyWithTar := b.copyWithTar(options.IDMappingOptions, &containerOwner, options.Hasher, options.DryRun)
untarPath := b.untarPath(nil, options.Hasher, options.DryRun)
err = addHelper(excludes, extract, dest, destfi, hostOwner, options, copyFileWithTar, copyWithTar, untarPath, source...)
err = b.addHelper(excludes, extract, dest, destfi, hostOwner, options, copyFileWithTar, copyWithTar, untarPath, source...)
if err != nil {
return err
}
@ -243,9 +247,10 @@ func dockerIgnoreMatcher(lines []string, contextDir string) (*fileutils.PatternM
return matcher, nil
}
func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error {
for _, src := range source {
func (b *Builder) addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error {
for n, src := range source {
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
b.ContentDigester.Start("")
// We assume that source is a file, and we're copying
// it to the destination. If the destination is
// already a directory, create a file inside of it.
@ -259,7 +264,7 @@ func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, de
if destfi != nil && destfi.IsDir() {
d = filepath.Join(dest, path.Base(url.Path))
}
if err = addURL(d, src, hostOwner, options.Hasher, options.DryRun); err != nil {
if err = b.addURL(d, src, hostOwner, options.Hasher, options.DryRun); err != nil {
return err
}
continue
@ -283,6 +288,7 @@ func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, de
return errors.Wrapf(err, "error reading %q", esrc)
}
if srcfi.IsDir() {
b.ContentDigester.Start("dir")
// The source is a directory, so copy the contents of
// the source directory into the target directory. Try
// to create it first, so that if there's a problem,
@ -292,7 +298,7 @@ func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, de
return errors.Wrapf(err, "error creating directory %q", dest)
}
}
logrus.Debugf("copying %q to %q", esrc+string(os.PathSeparator)+"*", dest+string(os.PathSeparator)+"*")
logrus.Debugf("copying[%d] %q to %q", n, esrc+string(os.PathSeparator)+"*", dest+string(os.PathSeparator)+"*")
if excludes == nil || !excludes.Exclusions() {
if err = copyWithTar(esrc, dest); err != nil {
return errors.Wrapf(err, "error copying %q to %q", esrc, dest)
@ -326,6 +332,8 @@ func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, de
continue
}
b.ContentDigester.Start("file")
if !extract || !archive.IsArchivePath(esrc) {
// This source is a file, and either it's not an
// archive, or we don't care whether or not it's an
@ -335,7 +343,7 @@ func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, de
d = filepath.Join(dest, filepath.Base(gsrc))
}
// Copy the file, preserving attributes.
logrus.Debugf("copying %q to %q", esrc, d)
logrus.Debugf("copying[%d] %q to %q", n, esrc, d)
if err = copyFileWithTar(esrc, d); err != nil {
return errors.Wrapf(err, "error copying %q to %q", esrc, d)
}
@ -343,7 +351,7 @@ func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, de
}
// We're extracting an archive into the destination directory.
logrus.Debugf("extracting contents of %q into %q", esrc, dest)
logrus.Debugf("extracting contents[%d] of %q into %q", n, esrc, dest)
if err = untarPath(esrc, dest); err != nil {
return errors.Wrapf(err, "error extracting %q into %q", esrc, dest)
}

2
buildah.go
View File

@ -196,6 +196,8 @@ type Builder struct {
Format string
// TempVolumes are temporary mount points created during container runs
TempVolumes map[string]bool
// ContentDigester counts the digest of all Add()ed content
ContentDigester CompositeDigester
}
// BuilderInfo are used as objects to display container information

64
digester.go Normal file
View File

@ -0,0 +1,64 @@
package buildah
import (
"hash"
"strings"
digest "github.com/opencontainers/go-digest"
)
type singleDigester struct {
digester digest.Digester
prefix string
}
// CompositeDigester can compute a digest over multiple items.
type CompositeDigester struct {
digesters []singleDigester
}
// Restart clears all state, so that the composite digester can start over.
func (c *CompositeDigester) Restart() {
c.digesters = nil
}
// Start starts recording the digest for a new item. The caller should call
// Hash() immediately after to retrieve the new io.Writer.
func (c *CompositeDigester) Start(prefix string) {
prefix = strings.TrimSuffix(prefix, ":")
c.digesters = append(c.digesters, singleDigester{digester: digest.Canonical.Digester(), prefix: prefix})
}
// Hash returns the hasher for the current item.
func (c *CompositeDigester) Hash() hash.Hash {
num := len(c.digesters)
if num == 0 {
return nil
}
return c.digesters[num-1].digester.Hash()
}
// Digest returns the prefix and a composite digest over everything that's been
// digested.
func (c *CompositeDigester) Digest() (string, digest.Digest) {
num := len(c.digesters)
switch num {
case 0:
return "", ""
case 1:
return c.digesters[0].prefix, c.digesters[0].digester.Digest()
default:
content := ""
for i, digester := range c.digesters {
if i > 0 {
content += ","
}
prefix := digester.prefix
if digester.prefix != "" {
digester.prefix += ":"
}
content += prefix + digester.digester.Digest().Encoded()
}
return "multi", digest.Canonical.FromString(content)
}
}

115
imagebuildah/chroot_symlink_linux.go
View File

@ -6,7 +6,6 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/containers/storage/pkg/reexec"
"github.com/pkg/errors"
@ -15,13 +14,11 @@ import (
const (
symlinkChrootedCommand = "chrootsymlinks-resolve"
symlinkModifiedTime = "modtimesymlinks-resolve"
maxSymlinksResolved = 40
)
func init() {
reexec.Register(symlinkChrootedCommand, resolveChrootedSymlinks)
reexec.Register(symlinkModifiedTime, resolveSymlinkTimeModified)
}
// resolveSymlink uses a child subprocess to resolve any symlinks in filename
@ -71,118 +68,6 @@ func resolveChrootedSymlinks() {
os.Exit(status)
}
// main() for grandparent subprocess. Its main job is to shuttle stdio back
// and forth, managing a pseudo-terminal if we want one, for our child, the
// parent subprocess.
func resolveSymlinkTimeModified() {
status := 0
flag.Parse()
if len(flag.Args()) < 1 {
os.Exit(1)
}
// Our first parameter is the directory to chroot into.
if err := unix.Chdir(flag.Arg(0)); err != nil {
fmt.Fprintf(os.Stderr, "chdir(): %v\n", err)
os.Exit(1)
}
if err := unix.Chroot(flag.Arg(0)); err != nil {
fmt.Fprintf(os.Stderr, "chroot(): %v\n", err)
os.Exit(1)
}
// Our second parameter is the path name to evaluate for symbolic links.
// Our third parameter is the time the cached intermediate image was created.
// We check whether the modified time of the filepath we provide is after the time the cached image was created.
timeIsGreater, err := modTimeIsGreater(flag.Arg(0), flag.Arg(1), flag.Arg(2))
if err != nil {
fmt.Fprintf(os.Stderr, "error checking if modified time of resolved symbolic link is greater: %v\n", err)
os.Exit(1)
}
if _, err := os.Stdout.WriteString(fmt.Sprintf("%v", timeIsGreater)); err != nil {
fmt.Fprintf(os.Stderr, "error writing string to stdout: %v\n", err)
os.Exit(1)
}
os.Exit(status)
}
// resolveModifiedTime (in the grandparent process) checks filename for any symlinks,
// resolves it and compares the modified time of the file with historyTime, which is
// the creation time of the cached image. It returns true if filename was modified after
// historyTime, otherwise returns false.
func resolveModifiedTime(rootdir, filename, historyTime string) (bool, error) {
// The child process expects a chroot and one path that
// will be consulted relative to the chroot directory and evaluated
// for any symbolic links present.
cmd := reexec.Command(symlinkModifiedTime, rootdir, filename, historyTime)
output, err := cmd.CombinedOutput()
if err != nil {
return false, errors.Wrapf(err, string(output))
}
// Hand back true/false depending on in the file was modified after the caches image was created.
return string(output) == "true", nil
}
// modTimeIsGreater goes through the files added/copied in using the Dockerfile and
// checks the time stamp (follows symlinks) with the time stamp of when the cached
// image was created. IT compares the two and returns true if the file was modified
// after the cached image was created, otherwise it returns false.
func modTimeIsGreater(rootdir, path string, historyTime string) (bool, error) {
var timeIsGreater bool
// Convert historyTime from string to time.Time for comparison
histTime, err := time.Parse(time.RFC3339Nano, historyTime)
if err != nil {
return false, errors.Wrapf(err, "error converting string to time.Time %q", historyTime)
}
// Since we are chroot in rootdir, we want a relative path, i.e (path - rootdir)
relPath, err := filepath.Rel(rootdir, path)
if err != nil {
return false, errors.Wrapf(err, "error making path %q relative to %q", path, rootdir)
}
// Walk the file tree and check the time stamps.
err = filepath.Walk(relPath, func(path string, info os.FileInfo, err error) error {
// If using cached images, it is possible for files that are being copied to come from
// previous build stages. But if using cached images, then the copied file won't exist
// since a container won't have been created for the previous build stage and info will be nil.
// In that case just return nil and continue on with using the cached image for the whole build process.
if info == nil {
return nil
}
modTime := info.ModTime()
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
// Evaluate any symlink that occurs to get updated modified information
resolvedPath, err := filepath.EvalSymlinks(path)
if err != nil && os.IsNotExist(err) {
return errors.Wrapf(errDanglingSymlink, "%q", path)
}
if err != nil {
return errors.Wrapf(err, "error evaluating symlink %q", path)
}
fileInfo, err := os.Stat(resolvedPath)
if err != nil {
return errors.Wrapf(err, "error getting file info %q", resolvedPath)
}
modTime = fileInfo.ModTime()
}
if modTime.After(histTime) {
timeIsGreater = true
return nil
}
return nil
})
if err != nil {
// if error is due to dangling symlink, ignore error and return nil
if errors.Cause(err) == errDanglingSymlink {
return false, nil
}
return false, errors.Wrapf(err, "error walking file tree %q", path)
}
return timeIsGreater, err
}
// getSymbolic link goes through each part of the path and continues resolving symlinks as they appear.
// Returns what the whole target path for what "path" resolves to.
func getSymbolicLink(path string) (string, error) {

7
imagebuildah/errors.go
View File

@ -1,7 +0,0 @@
package imagebuildah
import "errors"
var (
errDanglingSymlink = errors.New("error evaluating dangling symlink")
)

26
imagebuildah/executor.go
View File

@ -42,8 +42,8 @@ var builtinAllowedBuildArgs = map[string]bool{
}
// Executor is a buildah-based implementation of the imagebuilder.Executor
// interface. It coordinates the entire build by using one StageExecutors to
// handle each stage of the build.
// interface. It coordinates the entire build by using one or more
// StageExecutors to handle each stage of the build.
type Executor struct {
stages map[string]*StageExecutor
store storage.Store
@ -248,26 +248,36 @@ func (b *Executor) getImageHistory(ctx context.Context, imageID string) ([]v1.Hi
return oci.History, nil
}
// getCreatedBy returns the command the image at node will be created by.
func (b *Executor) getCreatedBy(node *parser.Node) string {
// getCreatedBy returns the command the image at node will be created by. If
// the passed-in CompositeDigester is not nil, it is assumed to have the digest
// information for the content if the node is ADD or COPY.
func (b *Executor) getCreatedBy(node *parser.Node, addedContentDigest string) string {
if node == nil {
return "/bin/sh"
}
if node.Value == "run" {
switch strings.ToUpper(node.Value) {
case "RUN":
buildArgs := b.getBuildArgs()
if buildArgs != "" {
return "|" + strconv.Itoa(len(strings.Split(buildArgs, " "))) + " " + buildArgs + " /bin/sh -c " + node.Original[4:]
}
return "/bin/sh -c " + node.Original[4:]
case "ADD", "COPY":
destination := node
for destination.Next != nil {
destination = destination.Next
}
return "/bin/sh -c #(nop) " + strings.ToUpper(node.Value) + " " + addedContentDigest + " in " + destination.Value + " "
default:
return "/bin/sh -c #(nop) " + node.Original
}
return "/bin/sh -c #(nop) " + node.Original
}
// historyMatches returns true if a candidate history matches the history of our
// base image (if we have one), plus the current instruction.
// Used to verify whether a cache of the intermediate image exists and whether
// to run the build again.
func (b *Executor) historyMatches(baseHistory []v1.History, child *parser.Node, history []v1.History) bool {
func (b *Executor) historyMatches(baseHistory []v1.History, child *parser.Node, history []v1.History, addedContentDigest string) bool {
if len(baseHistory) >= len(history) {
return false
}
@ -297,7 +307,7 @@ func (b *Executor) historyMatches(baseHistory []v1.History, child *parser.Node,
return false
}
}
return history[len(baseHistory)].CreatedBy == b.getCreatedBy(child)
return history[len(baseHistory)].CreatedBy == b.getCreatedBy(child, addedContentDigest)
}
// getBuildArgs returns a string of the build-args specified during the build process

310
imagebuildah/stage_executor.go
View File

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
@ -249,9 +248,112 @@ func (s *StageExecutor) volumeCacheRestore() error {
return nil
}
// digestContent digests any content that this next instruction would add to
// the image, returning the digester if there is any, or nil otherwise. We
// don't care about the details of where in the filesystem the content actually
// goes, because we're not actually going to add it here, so this is less
// involved than Copy().
func (s *StageExecutor) digestSpecifiedContent(node *parser.Node) (string, error) {
// No instruction: done.
if node == nil {
return "", nil
}
// Not adding content: done.
switch strings.ToUpper(node.Value) {
default:
return "", nil
case "ADD", "COPY":
}
// Pull out everything except the first node (the instruction) and the
// last node (the destination).
var srcs []string
destination := node
for destination.Next != nil {
destination = destination.Next
if destination.Next != nil {
srcs = append(srcs, destination.Value)
}
}
var sources []string
var idMappingOptions *buildah.IDMappingOptions
contextDir := s.executor.contextDir
for _, flag := range node.Flags {
if strings.HasPrefix(flag, "--from=") {
// Flag says to read the content from another
// container. Update the ID mappings and
// all-content-comes-from-below-this-directory value.
from := strings.TrimPrefix(flag, "--from=")
if other, ok := s.executor.stages[from]; ok {
contextDir = other.mountPoint
idMappingOptions = &other.builder.IDMappingOptions
} else if builder, ok := s.executor.containerMap[from]; ok {
contextDir = builder.MountPoint
idMappingOptions = &builder.IDMappingOptions
} else {
return "", errors.Errorf("the stage %q has not been built", from)
}
}
}
for _, src := range srcs {
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
// Source is a URL. TODO: cache this content
// somewhere, so that we can avoid pulling it down
// again if we end up needing to drop it into the
// filesystem.
sources = append(sources, src)
} else {
// Source is not a URL, so it's a location relative to
// the all-content-comes-from-below-this-directory
// directory.
contextSrc, err := securejoin.SecureJoin(contextDir, src)
if err != nil {
return "", errors.Wrapf(err, "error joining %q and %q", contextDir, src)
}
sources = append(sources, contextSrc)
}
}
// If the all-content-comes-from-below-this-directory is the build
// context, read its .dockerignore.
var excludes []string
if contextDir == s.executor.contextDir {
var err error
if excludes, err = imagebuilder.ParseDockerignore(contextDir); err != nil {
return "", errors.Wrapf(err, "error parsing .dockerignore in %s", contextDir)
}
}
// Restart the digester and have it do a dry-run copy to compute the
// digest information.
options := buildah.AddAndCopyOptions{
Excludes: excludes,
ContextDir: contextDir,
IDMappingOptions: idMappingOptions,
DryRun: true,
}
s.builder.ContentDigester.Restart()
download := strings.ToUpper(node.Value) == "ADD"
err := s.builder.Add(destination.Value, download, options, sources...)
if err != nil {
return "", errors.Wrapf(err, "error dry-running %q", node.Original)
}
// Return the formatted version of the digester's result.
contentDigest := ""
prefix, digest := s.builder.ContentDigester.Digest()
if prefix != "" {
prefix += ":"
}
if digest.Validate() == nil {
contentDigest = prefix + digest.Encoded()
}
return contentDigest, nil
}
// Copy copies data into the working tree. The "Download" field is how
// imagebuilder tells us the instruction was "ADD" and not "COPY"
// imagebuilder tells us the instruction was "ADD" and not "COPY".
func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) error {
s.builder.ContentDigester.Restart()
for _, copy := range copies {
// Check the file and see if part of it is a symlink.
// Convert it to the target if so. To be ultrasafe
@ -283,41 +385,52 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err
if err := s.volumeCacheInvalidate(copy.Dest); err != nil {
return err
}
sources := []string{}
var sources []string
// The From field says to read the content from another
// container. Update the ID mappings and
// all-content-comes-from-below-this-directory value.
var idMappingOptions *buildah.IDMappingOptions
var copyExcludes []string
contextDir := s.executor.contextDir
if len(copy.From) > 0 {
if other, ok := s.executor.stages[copy.From]; ok && other.index < s.index {
contextDir = other.mountPoint
idMappingOptions = &other.builder.IDMappingOptions
} else if builder, ok := s.executor.containerMap[copy.From]; ok {
contextDir = builder.MountPoint
idMappingOptions = &builder.IDMappingOptions
} else {
return errors.Errorf("the stage %q has not been built", copy.From)
}
copyExcludes = excludes
} else {
copyExcludes = append(s.executor.excludes, excludes...)
}
for _, src := range copy.Src {
contextDir := s.executor.contextDir
copyExcludes := excludes
var idMappingOptions *buildah.IDMappingOptions
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
// Source is a URL.
sources = append(sources, src)
} else if len(copy.From) > 0 {
var srcRoot string
if other, ok := s.executor.stages[copy.From]; ok && other.index < s.index {
srcRoot = other.mountPoint
contextDir = other.mountPoint
idMappingOptions = &other.builder.IDMappingOptions
} else if builder, ok := s.executor.containerMap[copy.From]; ok {
srcRoot = builder.MountPoint
contextDir = builder.MountPoint
idMappingOptions = &builder.IDMappingOptions
} else {
return errors.Errorf("the stage %q has not been built", copy.From)
}
srcSecure, err := securejoin.SecureJoin(srcRoot, src)
} else {
// Treat the source, which is not a URL, as a
// location relative to the
// all-content-comes-from-below-this-directory
// directory.
srcSecure, err := securejoin.SecureJoin(contextDir, src)
if err != nil {
return err
}
// If destination is a folder, we need to take extra care to
// ensure that files are copied with correct names (since
// resolving a symlink may result in a different name).
if hadFinalPathSeparator {
// If destination is a folder, we need to take extra care to
// ensure that files are copied with correct names (since
// resolving a symlink may result in a different name).
_, srcName := filepath.Split(src)
_, srcNameSecure := filepath.Split(srcSecure)
if srcName != srcNameSecure {
options := buildah.AddAndCopyOptions{
Chown: copy.Chown,
ContextDir: contextDir,
Excludes: copyExcludes,
Chown: copy.Chown,
ContextDir: contextDir,
Excludes: copyExcludes,
IDMappingOptions: idMappingOptions,
}
if err := s.builder.Add(filepath.Join(copy.Dest, srcName), copy.Download, options, srcSecure); err != nil {
return err
@ -326,21 +439,17 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err
}
}
sources = append(sources, srcSecure)
} else {
sources = append(sources, filepath.Join(s.executor.contextDir, src))
copyExcludes = append(s.executor.excludes, excludes...)
}
options := buildah.AddAndCopyOptions{
Chown: copy.Chown,
ContextDir: contextDir,
Excludes: copyExcludes,
IDMappingOptions: idMappingOptions,
}
if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil {
return err
}
}
options := buildah.AddAndCopyOptions{
Chown: copy.Chown,
ContextDir: contextDir,
Excludes: copyExcludes,
IDMappingOptions: idMappingOptions,
}
if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil {
return err
}
}
return nil
}
@ -645,7 +754,7 @@ func (s *StageExecutor) Execute(ctx context.Context, stage imagebuilder.Stage, b
// squash the contents of the base image. Whichever is
// the case, we need to commit() to create a new image.
logCommit(s.output, -1)
if imgID, ref, err = s.commit(ctx, ib, s.executor.getCreatedBy(nil), false, s.output); err != nil {
if imgID, ref, err = s.commit(ctx, ib, s.executor.getCreatedBy(nil, ""), false, s.output); err != nil {
return "", nil, errors.Wrapf(err, "error committing base container")
}
} else {
@ -711,13 +820,18 @@ func (s *StageExecutor) Execute(ctx context.Context, stage imagebuilder.Stage, b
logrus.Debugf("%v", errors.Wrapf(err, "error building at step %+v", *step))
return "", nil, errors.Wrapf(err, "error building at STEP \"%s\"", step.Message)
}
// In case we added content, retrieve its digest.
addedContentDigest, err := s.digestSpecifiedContent(node)
if err != nil {
return "", nil, err
}
if moreInstructions {
// There are still more instructions to process
// for this stage. Make a note of the
// instruction in the history that we'll write
// for the image when we eventually commit it.
now := time.Now()
s.builder.AddPrependedEmptyLayer(&now, s.executor.getCreatedBy(node), "", "")
s.builder.AddPrependedEmptyLayer(&now, s.executor.getCreatedBy(node, addedContentDigest), "", "")
continue
} else {
// This is the last instruction for this stage,
@ -726,7 +840,7 @@ func (s *StageExecutor) Execute(ctx context.Context, stage imagebuilder.Stage, b
// if it's used as the basis for a later stage.
if lastStage || imageIsUsedLater {
logCommit(s.output, i)
imgID, ref, err = s.commit(ctx, ib, s.executor.getCreatedBy(node), false, s.output)
imgID, ref, err = s.commit(ctx, ib, s.executor.getCreatedBy(node, addedContentDigest), false, s.output)
if err != nil {
return "", nil, errors.Wrapf(err, "error committing container for step %+v", *step)
}
@ -756,7 +870,11 @@ func (s *StageExecutor) Execute(ctx context.Context, stage imagebuilder.Stage, b
// cached images so far, look for one that matches what we
// expect to produce for this instruction.
if checkForLayers && !(s.executor.squash && lastInstruction && lastStage) {
cacheID, err = s.layerExists(ctx, node)
addedContentDigest, err := s.digestSpecifiedContent(node)
if err != nil {
return "", nil, err
}
cacheID, err = s.intermediateImageExists(ctx, node, addedContentDigest)
if err != nil {
return "", nil, errors.Wrap(err, "error checking if cached image exists from a previous build")
}
@ -809,9 +927,14 @@ func (s *StageExecutor) Execute(ctx context.Context, stage imagebuilder.Stage, b
logrus.Debugf("%v", errors.Wrapf(err, "error building at step %+v", *step))
return "", nil, errors.Wrapf(err, "error building at STEP \"%s\"", step.Message)
}
// In case we added content, retrieve its digest.
addedContentDigest, err := s.digestSpecifiedContent(node)
if err != nil {
return "", nil, err
}
// Create a new image, maybe with a new layer.
logCommit(s.output, i)
imgID, ref, err = s.commit(ctx, ib, s.executor.getCreatedBy(node), !s.stepRequiresLayer(step), commitName)
imgID, ref, err = s.commit(ctx, ib, s.executor.getCreatedBy(node, addedContentDigest), !s.stepRequiresLayer(step), commitName)
if err != nil {
return "", nil, errors.Wrapf(err, "error committing container for step %+v", *step)
}
@ -899,9 +1022,9 @@ func (s *StageExecutor) tagExistingImage(ctx context.Context, cacheID, output st
return img.ID, ref, nil
}
// layerExists returns true if an intermediate image of currNode exists in the image store from a previous build.
// intermediateImageExists returns true if an intermediate image of currNode exists in the image store from a previous build.
// It verifies this by checking the parent of the top layer of the image and the history.
func (s *StageExecutor) layerExists(ctx context.Context, currNode *parser.Node) (string, error) {
func (s *StageExecutor) intermediateImageExists(ctx context.Context, currNode *parser.Node, addedContentDigest string) (string, error) {
// Get the list of images available in the image store
images, err := s.executor.store.Images()
if err != nil {
@ -932,85 +1055,14 @@ func (s *StageExecutor) layerExists(ctx context.Context, currNode *parser.Node)
return "", errors.Wrapf(err, "error getting history of %q", image.ID)
}
// children + currNode is the point of the Dockerfile we are currently at.
if s.executor.historyMatches(baseHistory, currNode, history) {
// This checks if the files copied during build have been changed if the node is
// a COPY or ADD command.
filesMatch, err := s.copiedFilesMatch(currNode, history[len(history)-1].Created)
if err != nil {
return "", errors.Wrapf(err, "error checking if copied files match")
}
if filesMatch {
return image.ID, nil
}
if s.executor.historyMatches(baseHistory, currNode, history, addedContentDigest) {
return image.ID, nil
}
}
}
return "", nil
}
// getFilesToCopy goes through node to get all the src files that are copied, added or downloaded.
// It is possible for the Dockerfile to have src as hom*, which means all files that have hom as a prefix.
// Another format is hom?.txt, which means all files that have that name format with the ? replaced by another character.
func (s *StageExecutor) getFilesToCopy(node *parser.Node) ([]string, error) {
currNode := node.Next
var src []string
for currNode.Next != nil {
if strings.HasPrefix(currNode.Value, "http://") || strings.HasPrefix(currNode.Value, "https://") {
src = append(src, currNode.Value)
currNode = currNode.Next
continue
}
matches, err := filepath.Glob(filepath.Join(s.copyFrom, currNode.Value))
if err != nil {
return nil, errors.Wrapf(err, "error finding match for pattern %q", currNode.Value)
}
src = append(src, matches...)
currNode = currNode.Next
}
return src, nil
}
// copiedFilesMatch checks to see if the node instruction is a COPY or ADD.
// If it is either of those two it checks the timestamps on all the files copied/added
// by the dockerfile. If the host version has a time stamp greater than the time stamp
// of the build, the build will not use the cached version and will rebuild.
func (s *StageExecutor) copiedFilesMatch(node *parser.Node, historyTime *time.Time) (bool, error) {
if node.Value != "add" && node.Value != "copy" {
return true, nil
}
src, err := s.getFilesToCopy(node)
if err != nil {
return false, err
}
for _, item := range src {
// for urls, check the Last-Modified field in the header.
if strings.HasPrefix(item, "http://") || strings.HasPrefix(item, "https://") {
urlContentNew, err := urlContentModified(item, historyTime)
if err != nil {
return false, err
}
if urlContentNew {
return false, nil
}
continue
}
// Walks the file tree for local files and uses chroot to ensure we don't escape out of the allowed path
// when resolving any symlinks.
// Change the time format to ensure we don't run into a parsing error when converting again from string
// to time.Time. It is a known Go issue that the conversions cause errors sometimes, so specifying a particular
// time format here when converting to a string.
timeIsGreater, err := resolveModifiedTime(s.copyFrom, item, historyTime.Format(time.RFC3339Nano))
if err != nil {
return false, errors.Wrapf(err, "error resolving symlinks and comparing modified times: %q", item)
}
if timeIsGreater {
return false, nil
}
}
return true, nil
}
// commit writes the container's contents to an image, using a passed-in tag as
// the name if there is one, generating a unique ID-based one otherwise.
func (s *StageExecutor) commit(ctx context.Context, ib *imagebuilder.Builder, createdBy string, emptyLayer bool, output string) (string, reference.Canonical, error) {
@ -1134,23 +1186,3 @@ func (s *StageExecutor) EnsureContainerPath(path string) error {
}
return nil
}
// urlContentModified sends a get request to the url and checks if the header has a value in
// Last-Modified, and if it does compares the time stamp to that of the history of the cached image.
// returns true if there is no Last-Modified value in the header.
func urlContentModified(url string, historyTime *time.Time) (bool, error) {
resp, err := http.Get(url)
if err != nil {
return false, errors.Wrapf(err, "error getting %q", url)
}
defer resp.Body.Close()
if lastModified := resp.Header.Get("Last-Modified"); lastModified != "" {
lastModifiedTime, err := time.Parse(time.RFC1123, lastModified)
if err != nil {
return false, errors.Wrapf(err, "error parsing time for %q", url)
}
return lastModifiedTime.After(*historyTime), nil
}
logrus.Debugf("Response header did not have Last-Modified %q, will rebuild.", url)
return true, nil
}

1
tests/bud/dest-symlink-dangling/Dockerfile
View File

@ -3,3 +3,4 @@ FROM ubuntu
RUN mkdir /symlink
RUN ln -s /symlink /src && rm -rf /symlink
COPY Dockerfile /src/
RUN test -s /symlink/Dockerfile

34
util.go
View File

@ -227,8 +227,15 @@ func (b *Builder) copyFileWithTar(tarIDMappingOptions *IDMappingOptions, chownOp
// write-time possibly overridden using the passed-in chownOpts
func (b *Builder) copyWithTar(tarIDMappingOptions *IDMappingOptions, chownOpts *idtools.IDPair, hasher io.Writer, dryRun bool) func(src, dest string) error {
tar := b.tarPath(tarIDMappingOptions)
untar := b.untar(chownOpts, hasher, dryRun)
return func(src, dest string) error {
thisHasher := hasher
if thisHasher != nil && b.ContentDigester.Hash() != nil {
thisHasher = io.MultiWriter(thisHasher, b.ContentDigester.Hash())
}
if thisHasher == nil {
thisHasher = b.ContentDigester.Hash()
}
untar := b.untar(chownOpts, thisHasher, dryRun)
rc, err := tar(src)
if err != nil {
return errors.Wrapf(err, "error archiving %q for copy", src)
@ -241,6 +248,12 @@ func (b *Builder) copyWithTar(tarIDMappingOptions *IDMappingOptions, chownOpts *
// location into our working container, mapping permissions using the
// container's ID maps, possibly overridden using the passed-in chownOpts
func (b *Builder) untarPath(chownOpts *idtools.IDPair, hasher io.Writer, dryRun bool) func(src, dest string) error {
if hasher != nil && b.ContentDigester.Hash() != nil {
hasher = io.MultiWriter(hasher, b.ContentDigester.Hash())
}
if hasher == nil {
hasher = b.ContentDigester.Hash()
}
convertedUIDMap, convertedGIDMap := convertRuntimeIDMaps(b.IDMappingOptions.UIDMap, b.IDMappingOptions.GIDMap)
if dryRun {
return func(src, dest string) error {
@ -304,14 +317,23 @@ func (b *Builder) untar(chownOpts *idtools.IDPair, hasher io.Writer, dryRun bool
return nil
}
}
if hasher != nil {
originalUntar := untar
untar = func(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
return originalUntar(io.TeeReader(tarArchive, hasher), dest, options)
originalUntar := untar
untarWithHasher := func(tarArchive io.Reader, dest string, options *archive.TarOptions, untarHasher io.Writer) error {
reader := tarArchive
if untarHasher != nil {
reader = io.TeeReader(tarArchive, untarHasher)
}
return originalUntar(reader, dest, options)
}
return func(tarArchive io.ReadCloser, dest string) error {
err := untar(tarArchive, dest, options)
thisHasher := hasher
if thisHasher != nil && b.ContentDigester.Hash() != nil {
thisHasher = io.MultiWriter(thisHasher, b.ContentDigester.Hash())
}
if thisHasher == nil {
thisHasher = b.ContentDigester.Hash()
}
err := untarWithHasher(tarArchive, dest, options, thisHasher)
if err2 := tarArchive.Close(); err2 != nil {
if err == nil {
err = err2