buildkit: add support for custom build output with --output

Allows end-users to export final build content or rootfs to external formats.

By default, a local container image is created from the build result. The --output (or -o) flag allows you to override this behavior, and a specify a custom exporter. For example, custom exporters allow you to export the build artifacts as files on the local filesystem instead of a Container image, which can be useful for generating local binaries, code generation etc.

The value for --output is a CSV-formatted string defining the exporter type and options. Currently, local and tar exporters are supported. The local exporter writes the resulting build files to a directory on the client side. The tar exporter is similar but writes the files as a single tarball (.tar).

```console
buildah build --output type=local,dest=dir .
buildah build --output type=tar,dest=rootfs.tar .
buildah build -o dir .
```
Reference: https://docs.docker.com/engine/reference/commandline/build/#custom-build-outputs

Signed-off-by: Aditya R <arajan@redhat.com>

cmd/buildah/build.go

@@ -328,6 +328,15 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error {
 		t := time.Unix(iopts.Timestamp, 0).UTC()
 		timestamp = &t
 	}
+	if c.Flag("output").Changed {
+		buildOption, err := parse.GetBuildOutput(iopts.BuildOutput)
+		if err != nil {
+			return err
+		}
+		if buildOption.IsStdout {
+			iopts.Quiet = true
+		}
+	}
 	options := define.BuildOptions{
 		AddCapabilities:         iopts.CapAdd,
 		AdditionalTags:          tags,
@@ -363,6 +372,7 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error {
 		OS:                      systemContext.OSChoice,
 		Out:                     stdout,
 		Output:                  output,
+		BuildOutput:             iopts.BuildOutput,
 		OutputFormat:            format,
 		PullPolicy:              pullPolicy,
 		PullPushRetryDelay:      pullPushRetryDelay,

commit.go

@@ -229,6 +229,7 @@ func (b *Builder) addManifest(ctx context.Context, manifestName string, imageSpe
 func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options CommitOptions) (string, reference.Canonical, digest.Digest, error) {
 	var (
 		imgID string
+		src   types.ImageReference
 	)
 
 	// If we weren't given a name, build a destination reference using a
@@ -300,7 +301,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	logrus.Debugf("committing image with reference %q is allowed by policy", transports.ImageName(dest))
 
 	// Build an image reference from which we can copy the finished image.
-	src, err := b.makeImageRef(options)
+	src, err = b.makeContainerImageRef(options)
 	if err != nil {
 		return imgID, nil, "", errors.Wrapf(err, "error computing layer digests and building metadata for container %q", b.ContainerID)
 	}

define/build.go

@@ -123,6 +123,10 @@ type BuildOptions struct {
 	Args map[string]string
 	// Name of the image to write to.
 	Output string
+	// BuildOutput specifies if any custom build output is selected for following build.
+	// It allows end user to export recently built rootfs into a directory or tar.
+	// See the documentation of 'buildah build --output' for the details of the format.
+	BuildOutput string
 	// Additional tags to add to the image that we write, if we know of a
 	// way to add them.
 	AdditionalTags []string

define/types.go

@@ -96,6 +96,13 @@ type Secret struct {
 	SourceType string
 }
 
+// BuildOutputOptions contains the the outcome of parsing the value of a build --output flag
+type BuildOutputOption struct {
+	Path     string // Only valid if !IsStdout
+	IsDir    bool
+	IsStdout bool
+}
+
 // TempDirForURL checks if the passed-in string looks like a URL or -.  If it is,
 // TempDirForURL creates a temporary directory, arranges for its contents to be
 // the contents of that URL, and returns the temporary directory's path, along

docs/buildah-build.1.md

@@ -411,6 +411,25 @@ By default, Buildah manages _/etc/hosts_, adding the container's own IP address.
 
 Set the OS of the image to be built, and that of the base image to be pulled, if the build uses one, instead of using the current operating system of the host.
 
+**--output**, **-o**=""
+
+Output destination (format: type=local,dest=path)
+
+The --output (or -o) option extends the default behavior of building a container image by allowing users to export the contents of the image as files on the local filesystem, which can be useful for generating local binaries, code generation, etc.
+
+The value for --output is a comma-separated sequence of key=value pairs, defining the output type and options.
+
+Supported _keys_ are:
+- **dest**: Destination path for exported output. Valid value is absolute or relative path, `-` means the standard output.
+- **type**: Defines the type of output to be used. Valid values is documented below.
+
+Valid _type_ values are:
+- **local**: write the resulting build files to a directory on the client-side.
+- **tar**: write the resulting files as a single tarball (.tar).
+
+If no type is specified, the value defaults to **local**.
+Alternatively, instead of a comma-separated sequence, the value of **--output** can be just a destination (in the `**dest** format) (e.g. `--output some-path`, `--output -`) where `--output some-path` is treated as if **type=local** and `--output -` is treated as if **type=tar**.
+
 **--pid** *how*
 
 Sets the configuration for PID namespaces when handling `RUN` instructions.
@@ -824,6 +843,16 @@ buildah bud --platform linux/arm64 --platform linux/amd64 --manifest myimage /tm
 
 buildah bud --all-platforms --manifest myimage /tmp/mysrc
 
+### Building an image using (--output) custom build output
+
+buildah build -o out .
+
+buildah build --output type=local,dest=out .
+
+buildah build --output type=tar,dest=out.tar .
+
+buildah build -o - . > out.tar
+
 ### Building an image using a URL
 
   This will clone the specified GitHub repository from the URL and use it as context. The Containerfile or Dockerfile at the root of the repository is used as the context of the build. This only works if the GitHub repository is a dedicated repository.

image.go

@@ -43,6 +43,15 @@ const (
 	Dockerv2ImageManifest = define.Dockerv2ImageManifest
 )
 
+// ExtractRootfsOptions is consumed by ExtractRootfs() which allows
+// users to preserve nature of various modes like setuid, setgid and xattrs
+// over the extracted file system objects.
+type ExtractRootfsOptions struct {
+	StripSetuidBit bool // strip the setuid bit off of items being extracted.
+	StripSetgidBit bool // strip the setgid bit off of items being extracted.
+	StripXattrs    bool // don't record extended attributes of items being extracted.
+}
+
 type containerImageRef struct {
 	fromImageName         string
 	fromImageID           string
@@ -150,7 +159,10 @@ func computeLayerMIMEType(what string, layerCompression archive.Compression) (om
 }
 
 // Extract the container's whole filesystem as if it were a single layer.
-func (i *containerImageRef) extractRootfs() (io.ReadCloser, chan error, error) {
+// Takes ExtractRootfsOptions as argument which allows caller to configure
+// preserve nature of setuid,setgid,sticky and extended attributes
+// on extracted rootfs.
+func (i *containerImageRef) extractRootfs(opts ExtractRootfsOptions) (io.ReadCloser, chan error, error) {
 	var uidMap, gidMap []idtools.IDMap
 	mountPoint, err := i.store.Mount(i.containerID, i.mountLabel)
 	if err != nil {
@@ -164,8 +176,11 @@ func (i *containerImageRef) extractRootfs() (io.ReadCloser, chan error, error) {
 			uidMap, gidMap = convertRuntimeIDMaps(i.idMappingOptions.UIDMap, i.idMappingOptions.GIDMap)
 		}
 		copierOptions := copier.GetOptions{
-			UIDMap: uidMap,
+			UIDMap:         uidMap,
-			GIDMap: gidMap,
+			GIDMap:         gidMap,
+			StripSetuidBit: opts.StripSetuidBit,
+			StripSetgidBit: opts.StripSetgidBit,
+			StripXattrs:    opts.StripXattrs,
 		}
 		err = copier.Get(mountPoint, mountPoint, copierOptions, []string{"."}, pipeWriter)
 		errChan <- err
@@ -376,7 +391,7 @@ func (i *containerImageRef) NewImageSource(ctx context.Context, sc *types.System
 		var errChan chan error
 		if i.squash {
 			// Extract the root filesystem as a single layer.
-			rc, errChan, err = i.extractRootfs()
+			rc, errChan, err = i.extractRootfs(ExtractRootfsOptions{})
 			if err != nil {
 				return nil, err
 			}
@@ -738,7 +753,7 @@ func (i *containerImageSource) GetBlob(ctx context.Context, blob types.BlobInfo,
 	return ioutils.NewReadCloserWrapper(layerReadCloser, closer), size, nil
 }
 
-func (b *Builder) makeImageRef(options CommitOptions) (types.ImageReference, error) {
+func (b *Builder) makeContainerImageRef(options CommitOptions) (*containerImageRef, error) {
 	var name reference.Named
 	container, err := b.store.Container(b.ContainerID)
 	if err != nil {
@@ -813,3 +828,12 @@ func (b *Builder) makeImageRef(options CommitOptions) (types.ImageReference, err
 	}
 	return ref, nil
 }
+
+// Extract the container's whole filesystem as if it were a single layer from current builder instance
+func (b *Builder) ExtractRootfs(options CommitOptions, opts ExtractRootfsOptions) (io.ReadCloser, chan error, error) {
+	src, err := b.makeContainerImageRef(options)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "error creating image reference for container %q to extract its contents", b.ContainerID)
+	}
+	return src.extractRootfs(opts)
+}

imagebuildah/executor.go

@@ -133,6 +133,7 @@ type Executor struct {
 	unsetEnvs               []string
 	processLabel            string // Shares processLabel of first stage container with containers of other stages in same build
 	mountLabel              string // Shares mountLabel of first stage container with containers of other stages in same build
+	buildOutput             string // Specifies instructions for any custom build output
 }
 
 type imageTypeAndHistoryAndDiffIDs struct {
@@ -276,6 +277,7 @@ func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, o
 		sshsources:                     sshsources,
 		logPrefix:                      logPrefix,
 		unsetEnvs:                      options.UnsetEnvs,
+		buildOutput:                    options.BuildOutput,
 	}
 	if exec.err == nil {
 		exec.err = os.Stderr

imagebuildah/stage_executor.go

@@ -16,6 +16,7 @@ import (
 	"github.com/containers/buildah/define"
 	buildahdocker "github.com/containers/buildah/docker"
 	"github.com/containers/buildah/internal"
+	internalUtil "github.com/containers/buildah/internal/util"
 	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/buildah/pkg/rusage"
 	"github.com/containers/buildah/util"
@@ -28,6 +29,7 @@ import (
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/chrootarchive"
+	"github.com/containers/storage/pkg/unshare"
 	docker "github.com/fsouza/go-dockerclient"
 	digest "github.com/opencontainers/go-digest"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -1447,8 +1449,18 @@ func (s *StageExecutor) intermediateImageExists(ctx context.Context, currNode *p
 
 // commit writes the container's contents to an image, using a passed-in tag as
 // the name if there is one, generating a unique ID-based one otherwise.
+// or commit via any custom exporter if specified.
 func (s *StageExecutor) commit(ctx context.Context, createdBy string, emptyLayer bool, output string) (string, reference.Canonical, error) {
 	ib := s.stage.Builder
+	var buildOutputOption define.BuildOutputOption
+	if s.executor.buildOutput != "" {
+		var err error
+		logrus.Debugf("Generating custom build output with options %q", s.executor.buildOutput)
+		buildOutputOption, err = parse.GetBuildOutput(s.executor.buildOutput)
+		if err != nil {
+			return "", nil, errors.Wrapf(err, "failed to parse build output")
+		}
+	}
 	var imageRef types.ImageReference
 	if output != "" {
 		imageRef2, err := s.executor.resolveNameToImageRef(output)
@@ -1556,6 +1568,39 @@ func (s *StageExecutor) commit(ctx context.Context, createdBy string, emptyLayer
 		Manifest:              s.executor.manifest,
 		UnsetEnvs:             s.executor.unsetEnvs,
 	}
+	// generate build output
+	if s.executor.buildOutput != "" {
+		extractRootfsOpts := buildah.ExtractRootfsOptions{}
+		if unshare.IsRootless() {
+			// In order to maintain as much parity as possible
+			// with buildkit's version of --output and to avoid
+			// unsafe invocation of exported executables it was
+			// decided to strip setuid,setgid and extended attributes.
+			// Since modes like setuid,setgid leaves room for executable
+			// to get invoked with different file-system permission its safer
+			// to strip them off for unpriviledged invocation.
+			// See: https://github.com/containers/buildah/pull/3823#discussion_r829376633
+			extractRootfsOpts.StripSetuidBit = true
+			extractRootfsOpts.StripSetgidBit = true
+			extractRootfsOpts.StripXattrs = true
+		}
+		rc, errChan, err := s.builder.ExtractRootfs(options, extractRootfsOpts)
+		if err != nil {
+			return "", nil, errors.Wrapf(err, "failed to extract rootfs from given container image")
+		}
+		defer rc.Close()
+		err = internalUtil.ExportFromReader(rc, buildOutputOption)
+		if err != nil {
+			return "", nil, errors.Wrapf(err, "failed to export build output")
+		}
+		if errChan != nil {
+			err = <-errChan
+			if err != nil {
+				return "", nil, err
+			}
+		}
+
+	}
 	imgID, _, manifestDigest, err := s.builder.Commit(ctx, imageRef, options)
 	if err != nil {
 		return "", nil, err

internal/util/util.go

@@ -1,9 +1,18 @@
 package util
 
 import (
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/containers/buildah/define"
 	"github.com/containers/common/libimage"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage"
+	"github.com/containers/storage/pkg/archive"
+	"github.com/containers/storage/pkg/chrootarchive"
+	"github.com/containers/storage/pkg/unshare"
+	"github.com/pkg/errors"
 )
 
 // LookupImage returns *Image to corresponding imagename or id
@@ -22,3 +31,51 @@ func LookupImage(ctx *types.SystemContext, store storage.Store, image string) (*
 	}
 	return localImage, nil
 }
+
+// ExportFromReader reads bytes from given reader and exports to external tar, directory or stdout.
+func ExportFromReader(input io.Reader, opts define.BuildOutputOption) error {
+	var err error
+	if !filepath.IsAbs(opts.Path) {
+		opts.Path, err = filepath.Abs(opts.Path)
+		if err != nil {
+			return err
+		}
+	}
+	if opts.IsDir {
+		// In order to keep this feature as close as possible to
+		// buildkit it was decided to preserve ownership when
+		// invoked as root since caller already has access to artifacts
+		// therefore we can preserve ownership as is, however for rootless users
+		// ownership has to be changed so exported artifacts can still
+		// be accessible by unpriviledged users.
+		// See: https://github.com/containers/buildah/pull/3823#discussion_r829376633
+		noLChown := false
+		if unshare.IsRootless() {
+			noLChown = true
+		}
+
+		err = os.MkdirAll(opts.Path, 0700)
+		if err != nil {
+			return errors.Wrapf(err, "failed while creating the destination path %q", opts.Path)
+		}
+
+		err = chrootarchive.Untar(input, opts.Path, &archive.TarOptions{NoLchown: noLChown})
+		if err != nil {
+			return errors.Wrapf(err, "failed while performing untar at %q", opts.Path)
+		}
+	} else {
+		outFile := os.Stdout
+		if !opts.IsStdout {
+			outFile, err = os.Create(opts.Path)
+			if err != nil {
+				return errors.Wrapf(err, "failed while creating destination tar at %q", opts.Path)
+			}
+			defer outFile.Close()
+		}
+		_, err = io.Copy(outFile, input)
+		if err != nil {
+			return errors.Wrapf(err, "failed while performing copy to %q", opts.Path)
+		}
+	}
+	return nil
+}

pkg/cli/common.go

@@ -85,6 +85,7 @@ type BudResults struct {
 	Squash              bool
 	Stdin               bool
 	Tag                 []string
+	BuildOutput         string
 	Target              string
 	TLSVerify           bool
 	Jobs                int
@@ -242,6 +243,7 @@ func GetBudFlags(flags *BudResults) pflag.FlagSet {
 	fs.StringArrayVar(&flags.SSH, "ssh", []string{}, "SSH agent socket or keys to expose to the build. (format: default|<id>[=<socket>|<key>[,<key>]])")
 	fs.BoolVar(&flags.Stdin, "stdin", false, "pass stdin into containers")
 	fs.StringArrayVarP(&flags.Tag, "tag", "t", []string{}, "tagged `name` to apply to the built image")
+	fs.StringVarP(&flags.BuildOutput, "output", "o", "", "output destination (format: type=local,dest=path)")
 	fs.StringVar(&flags.Target, "target", "", "set the target build stage to build")
 	fs.Int64Var(&flags.Timestamp, "timestamp", 0, "set created timestamp to the specified epoch seconds to allow for deterministic builds, defaults to current time")
 	fs.BoolVar(&flags.TLSVerify, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry")
@@ -281,6 +283,7 @@ func GetBudFlagsCompletions() commonComp.FlagCompletions {
 	flagCompletion["timestamp"] = commonComp.AutocompleteNone
 	flagCompletion["variant"] = commonComp.AutocompleteNone
 	flagCompletion["unsetenv"] = commonComp.AutocompleteNone
+	flagCompletion["output"] = commonComp.AutocompleteNone
 	return flagCompletion
 }
 

pkg/parse/parse.go

@@ -510,6 +510,73 @@ func AuthConfig(creds string) (*types.DockerAuthConfig, error) {
 	}, nil
 }
 
+// GetBuildOutput is responsible for parsing custom build output argument i.e `build --output` flag.
+// Takes `buildOutput` as string and returns BuildOutputOption
+func GetBuildOutput(buildOutput string) (define.BuildOutputOption, error) {
+	if len(buildOutput) == 1 && buildOutput == "-" {
+		// Feature parity with buildkit, output tar to stdout
+		// Read more here: https://docs.docker.com/engine/reference/commandline/build/#custom-build-outputs
+		return define.BuildOutputOption{Path: "",
+			IsDir:    false,
+			IsStdout: true}, nil
+	}
+	if !strings.Contains(buildOutput, ",") {
+		// expect default --output <dirname>
+		return define.BuildOutputOption{Path: buildOutput,
+			IsDir:    true,
+			IsStdout: false}, nil
+	}
+	isDir := true
+	isStdout := false
+	typeSelected := false
+	pathSelected := false
+	path := ""
+	tokens := strings.Split(buildOutput, ",")
+	for _, option := range tokens {
+		arr := strings.SplitN(option, "=", 2)
+		if len(arr) != 2 {
+			return define.BuildOutputOption{}, fmt.Errorf("invalid build output options %q, expected format key=value", buildOutput)
+		}
+		switch arr[0] {
+		case "type":
+			if typeSelected {
+				return define.BuildOutputOption{}, fmt.Errorf("Duplicate %q not supported", arr[0])
+			}
+			typeSelected = true
+			if arr[1] == "local" {
+				isDir = true
+			} else if arr[1] == "tar" {
+				isDir = false
+			} else {
+				return define.BuildOutputOption{}, fmt.Errorf("invalid type %q selected for build output options %q", arr[1], buildOutput)
+			}
+		case "dest":
+			if pathSelected {
+				return define.BuildOutputOption{}, fmt.Errorf("Duplicate %q not supported", arr[0])
+			}
+			pathSelected = true
+			path = arr[1]
+		default:
+			return define.BuildOutputOption{}, fmt.Errorf("Unrecognized key %q in build output option: %q", arr[0], buildOutput)
+		}
+	}
+
+	if !typeSelected || !pathSelected {
+		return define.BuildOutputOption{}, fmt.Errorf("invalid build output option %q, accepted keys are type and dest must be present", buildOutput)
+	}
+
+	if path == "-" {
+		if isDir {
+			return define.BuildOutputOption{}, fmt.Errorf("invalid build output option %q, type=local and dest=- is not supported", buildOutput)
+		}
+		return define.BuildOutputOption{Path: "",
+			IsDir:    false,
+			IsStdout: true}, nil
+	}
+
+	return define.BuildOutputOption{Path: path, IsDir: isDir, IsStdout: isStdout}, nil
+}
+
 // IDMappingOptions parses the build options related to user namespaces and ID mapping.
 func IDMappingOptions(c *cobra.Command, isolation define.Isolation) (usernsOptions define.NamespaceOptions, idmapOptions *define.IDMappingOptions, err error) {
 	return IDMappingOptionsFromFlagSet(c.Flags(), c.PersistentFlags(), c.Flag)

tests/bud.bats

@@ -551,6 +551,67 @@ _EOF
   expect_output "[container=buildah date=tomorrow]" "No Path should be defined"
 }
 
+@test "build with custom build output and output rootfs to directory" {
+  _prefetch alpine
+  mytmpdir=${TEST_SCRATCH_DIR}/my-dir
+  mkdir -p $mytmpdir
+  cat > $mytmpdir/Containerfile << _EOF
+FROM alpine
+RUN echo 'hello'> hello
+_EOF
+  run_buildah build --output type=local,dest=$mytmpdir/rootfs $WITH_POLICY_JSON -t test-bud -f $mytmpdir/Containerfile .
+  ls $mytmpdir/rootfs
+  # exported rootfs must contain `hello` file which we created inside the image
+  expect_output --substring 'hello'
+}
+
+@test "build with custom build output and output rootfs to tar" {
+  _prefetch alpine
+  mytmpdir=${TEST_SCRATCH_DIR}/my-dir
+  mkdir -p $mytmpdir
+  cat > $mytmpdir/Containerfile << _EOF
+FROM alpine
+RUN echo 'hello'> hello
+_EOF
+  run_buildah build --output type=tar,dest=$mytmpdir/rootfs.tar $WITH_POLICY_JSON -t test-bud -f $mytmpdir/Containerfile .
+  # explode tar
+  mkdir $mytmpdir/rootfs
+  tar -C $mytmpdir/rootfs -xvf $mytmpdir/rootfs.tar
+  ls $mytmpdir/rootfs
+  # exported rootfs must contain `hello` file which we created inside the image
+  expect_output --substring 'hello'
+}
+
+@test "build with custom build output and output rootfs to tar by pipe" {
+  _prefetch alpine
+  mytmpdir=${TEST_SCRATCH_DIR}/my-dir
+  mkdir -p $mytmpdir
+  cat > $mytmpdir/Containerfile << _EOF
+FROM alpine
+RUN echo 'hello'> hello
+_EOF
+  # Using BUILDAH_BINARY since run_buildah adds unwanted chars to tar created by pipe.
+  ${BUILDAH_BINARY} build $WITH_POLICY_JSON -o - -t test-bud -f $mytmpdir/Containerfile . > $mytmpdir/rootfs.tar
+  # explode tar
+  mkdir $mytmpdir/rootfs
+  tar -C $mytmpdir/rootfs -xvf $mytmpdir/rootfs.tar
+  ls $mytmpdir/rootfs/hello
+}
+
+@test "build with custom build output must fail for bad input" {
+  _prefetch alpine
+  mytmpdir=${TEST_SCRATCH_DIR}/my-dir
+  mkdir -p $mytmpdir
+  cat > $mytmpdir/Containerfile << _EOF
+FROM alpine
+RUN echo 'hello'> hello
+_EOF
+  run_buildah 125 build --output type=tar, $WITH_POLICY_JSON -t test-bud -f $mytmpdir/Containerfile .
+  expect_output --substring 'invalid'
+  run_buildah 125 build --output type=wrong,dest=hello --signature-policy ${TESTSDIR}/policy.json -t test-bud -f $mytmpdir/Containerfile .
+  expect_output --substring 'invalid'
+}
+
 @test "bud-from-scratch-untagged" {
   run_buildah build --iidfile ${TEST_SCRATCH_DIR}/output.iid $WITH_POLICY_JSON $BUDFILES/from-scratch
   iid=$(cat ${TEST_SCRATCH_DIR}/output.iid)