rootless, overlay: use fuse-overlayfs

when running in rootless mode, use fuse-overlayfs for mounting the
overlay file system on the host.  Then create a bind mount inside the
container.

Closes: https://github.com/containers/buildah/issues/1741

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

Closes: #1743
Approved by: rhatdan

pkg/overlay/overlay.go

@@ -4,17 +4,20 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
+	"github.com/containers/buildah/pkg/unshare"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 )
 
 // MountTemp creates a subdir of the contentDir based on the source directory
-// from the source system.  It then mounds up the source directory on to the
+// from the source system.  It then mounts up the source directory on to the
 // generated mount point and returns the mount point to the caller.
 func MountTemp(store storage.Store, containerID, source, dest string, rootUID, rootGID int) (mount specs.Mount, contentDir string, Err error) {
 
@@ -46,10 +49,55 @@ func MountTemp(store storage.Store, containerID, source, dest string, rootUID, r
 		return mount, "", errors.Wrapf(err, "failed to create the overlay %s directory", workDir)
 	}
 
+	overlayOptions := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,private", source, upperDir, workDir)
+
+	if unshare.IsRootless() {
+		mountProgram := ""
+
+		mountMap := map[string]bool{
+			".mount_program":         true,
+			"overlay.mount_program":  true,
+			"overlay2.mount_program": true,
+		}
+
+		for _, i := range store.GraphOptions() {
+			s := strings.SplitN(i, "=", 2)
+			if len(s) != 2 {
+				continue
+			}
+			k := s[0]
+			v := s[1]
+			if mountMap[k] {
+				mountProgram = v
+				break
+			}
+		}
+		if mountProgram != "" {
+			mergeDir := filepath.Join(contentDir, "merge")
+
+			if err := idtools.MkdirAllAs(mergeDir, 0700, rootUID, rootGID); err != nil {
+				return mount, "", errors.Wrapf(err, "failed to create the overlay %s directory", mergeDir)
+			}
+
+			cmd := exec.Command(mountProgram, "-o", overlayOptions, mergeDir)
+
+			if err := cmd.Run(); err != nil {
+				return mount, "", errors.Wrapf(err, "exec %s", mountProgram)
+			}
+
+			mount.Source = mergeDir
+			mount.Destination = dest
+			mount.Type = "bind"
+			mount.Options = []string{"bind", "slave"}
+			return mount, contentDir, nil
+		}
+		/* If a mount_program is not specified, fallback to try mount native overlay.  */
+	}
+
 	mount.Source = "overlay"
 	mount.Destination = dest
 	mount.Type = "overlay"
-	mount.Options = strings.Split(fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,private", source, upperDir, workDir), ",")
+	mount.Options = strings.Split(overlayOptions, ",")
 
 	return mount, contentDir, nil
 }
@@ -57,6 +105,14 @@ func MountTemp(store storage.Store, containerID, source, dest string, rootUID, r
 // RemoveTemp removes temporary mountpoint and all content from its parent
 // directory
 func RemoveTemp(contentDir string) error {
+	if unshare.IsRootless() {
+		mergeDir := filepath.Join(contentDir, "merge")
+		if err := unix.Unmount(mergeDir, 0); err != nil {
+			if !os.IsNotExist(err) {
+				return errors.Wrapf(err, "unmount overlay %s", mergeDir)
+			}
+		}
+	}
 	return os.RemoveAll(contentDir)
 }
 
@@ -64,6 +120,15 @@ func RemoveTemp(contentDir string) error {
 // directory
 func CleanupContent(containerDir string) (Err error) {
 	contentDir := filepath.Join(containerDir, "overlay")
+
+	if unshare.IsRootless() {
+		mergeDir := filepath.Join(contentDir, "merge")
+		if err := unix.Unmount(mergeDir, 0); err != nil {
+			if !os.IsNotExist(err) {
+				return errors.Wrapf(err, "unmount overlay %s", mergeDir)
+			}
+		}
+	}
 	if err := os.RemoveAll(contentDir); err != nil && !os.IsNotExist(err) {
 		return errors.Wrapf(err, "failed to cleanup overlay %s directory", contentDir)
 	}

pkg/parse/parse.go

@@ -14,7 +14,6 @@ import (
 	"unicode"
 
 	"github.com/containers/buildah"
-	"github.com/containers/buildah/pkg/unshare"
 	"github.com/containers/image/types"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/docker/go-units"
@@ -473,9 +472,6 @@ func ValidateVolumeOpts(options []string) ([]string, error) {
 			}
 			foundRWRO++
 		case "z", "Z", "O":
-			if opt == "O" && unshare.IsRootless() {
-				return nil, errors.Errorf("invalid options %q, overlay mounts not supported in rootless mode", strings.Join(options, ", "))
-			}
 			if foundLabelChange > 1 {
 				return nil, errors.Errorf("invalid options %q, can only specify 1 'z', 'Z', or 'O' option", strings.Join(options, ", "))
 			}