root
/
buildah
mirror of https://github.com/containers/buildah.git
1
0
Fork 0
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add the "copier" package 

Add new primitives for reading and writing data, represented as tar
streams, from and to possibly-chrooted directories via subprocesses.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
This commit is contained in:
Nalin Dahyabhai 2020-07-14 16:34:07 -04:00
parent b167b838b0
commit 36f4b8f7fa
10 changed files with 2640 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -35,7 +35,7 @@ LIBSECCOMP_COMMIT := release-2.3
EXTRA_LDFLAGS ?=
LDFLAGS := -ldflags '-X main.GitCommit=$(GIT_COMMIT) -X main.buildInfo=$(SOURCE_DATE_EPOCH) -X main.cniVersion=$(CNI_COMMIT) $(EXTRA_LDFLAGS)'
SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go cmd/buildah/*.go docker/*.go pkg/blobcache/*.go pkg/cli/*.go pkg/parse/*.go util/*.go
SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go cmd/buildah/*.go copier/*.go docker/*.go pkg/blobcache/*.go pkg/cli/*.go pkg/parse/*.go util/*.go
LINTFLAGS ?=

1367
copier/copier.go Normal file
View File

File diff suppressed because it is too large Load Diff

963
copier/copier_test.go Normal file
View File

@ -0,0 +1,963 @@
package copier
import (
"archive/tar"
"bufio"
"bytes"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"syscall"
"testing"
"time"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/reexec"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMain(m *testing.M) {
if reexec.Init() {
return
}
flag.Parse()
if testing.Verbose() {
logrus.SetLevel(logrus.DebugLevel)
}
os.Exit(m.Run())
}
// makeFileContents creates contents for a file of a specified size
func makeContents(length int64) io.ReadCloser {
pipeReader, pipeWriter := io.Pipe()
buffered := bufio.NewWriter(pipeWriter)
go func() {
count := int64(0)
for count < length {
if _, err := buffered.Write([]byte{"0123456789abcdef"[count%16]}); err != nil {
buffered.Flush()
pipeWriter.CloseWithError(err) // nolint:errcheck
return
}
count++
}
buffered.Flush()
pipeWriter.Close()
}()
return pipeReader
}
// makeArchiveSlice creates an archive from the set of headers and returns a byte slice.
func makeArchiveSlice(headers []tar.Header) []byte {
rc := makeArchive(headers, nil)
defer rc.Close()
buf := new(bytes.Buffer)
if _, err := io.Copy(buf, rc); err != nil {
panic("error creating in-memory archive")
}
return buf.Bytes()
}
// makeArchive creates an archive from the set of headers.
func makeArchive(headers []tar.Header, contents map[string][]byte) io.ReadCloser {
if contents == nil {
contents = make(map[string][]byte)
}
pipeReader, pipeWriter := io.Pipe()
go func() {
var err error
buffered := bufio.NewWriter(pipeWriter)
tw := tar.NewWriter(buffered)
for _, header := range headers {
var fileContent []byte
switch header.Typeflag {
case tar.TypeLink, tar.TypeSymlink:
header.Size = 0
case tar.TypeReg:
fileContent = contents[header.Name]
if len(fileContent) != 0 {
header.Size = int64(len(fileContent))
}
}
if err = tw.WriteHeader(&header); err != nil {
break
}
if header.Typeflag == tar.TypeReg && header.Size > 0 {
var fileContents io.Reader
if len(fileContent) > 0 {
fileContents = bytes.NewReader(fileContent)
} else {
rc := makeContents(header.Size)
defer rc.Close()
fileContents = rc
}
if _, err = io.Copy(tw, fileContents); err != nil {
break
}
}
}
tw.Close()
buffered.Flush()
if err != nil {
pipeWriter.CloseWithError(err) // nolint:errcheck
} else {
pipeWriter.Close()
}
}()
return pipeReader
}
// makeContextFromArchive creates a temporary directory, and a subdirectory
// inside of it, from an archive and returns its location. It can be removed
// once it's no longer needed.
func makeContextFromArchive(archive io.ReadCloser, subdir string) (string, error) {
tmp, err := ioutil.TempDir("", "copier-test-")
if err != nil {
return "", err
}
uidMap := []idtools.IDMap{{HostID: os.Getuid(), ContainerID: 0, Size: 1}}
gidMap := []idtools.IDMap{{HostID: os.Getgid(), ContainerID: 0, Size: 1}}
err = Put(tmp, path.Join(tmp, subdir), PutOptions{UIDMap: uidMap, GIDMap: gidMap}, archive)
archive.Close()
if err != nil {
os.RemoveAll(tmp)
return "", err
}
return tmp, err
}
// enumerateFiles walks a directory, returning the items it contains as a slice
// of names relative to that directory.
func enumerateFiles(directory string) ([]enumeratedFile, error) {
var results []enumeratedFile
err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
if info == nil || err != nil {
return err
}
rel, err := filepath.Rel(directory, path)
if err != nil {
return err
}
if rel != "" && rel != "." {
results = append(results, enumeratedFile{
name: rel,
date: info.ModTime().UTC().String(),
mode: info.Mode() & os.ModePerm,
})
}
return nil
})
if err != nil {
return nil, err
}
return results, nil
}
type expectedError struct {
inSubdir bool
name string
err error
}
type enumeratedFile struct {
name string
mode os.FileMode
date string
}
var (
testDate = time.Unix(1485449953, 0)
uid, gid = os.Getuid(), os.Getgid()
testArchiveSlice = makeArchiveSlice([]tar.Header{
{Name: "item-0", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 123, Mode: 0600, ModTime: testDate},
{Name: "item-1", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 456, Mode: 0600, ModTime: testDate},
{Name: "item-2", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 789, Mode: 0600, ModTime: testDate},
})
testArchives = []struct {
name string
rootOnly bool
headers []tar.Header
contents map[string][]byte
excludes []string
expectedGetErrors []expectedError
subdirContents map[string][]string
}{
{
name: "regular",
rootOnly: false,
headers: []tar.Header{
{Name: "file-0", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 123456789, Mode: 0600, ModTime: testDate},
{Name: "file-a", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 23, Mode: 0600, ModTime: testDate},
{Name: "file-b", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 23, Mode: 0600, ModTime: testDate},
{Name: "file-c", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "file-a", Mode: 0600, ModTime: testDate},
{Name: "link-0", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "../file-0", Size: 123456789, Mode: 0777, ModTime: testDate},
{Name: "link-a", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "file-a", Size: 23, Mode: 0777, ModTime: testDate},
{Name: "link-b", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "../file-a", Size: 23, Mode: 0777, ModTime: testDate},
{Name: "hlink-0", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "file-0", Size: 123456789, Mode: 0600, ModTime: testDate},
{Name: "hlink-a", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "/file-a", Size: 23, Mode: 0600, ModTime: testDate},
{Name: "hlink-b", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "../file-b", Size: 23, Mode: 0600, ModTime: testDate},
{Name: "subdir-a", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700, ModTime: testDate},
{Name: "subdir-a/file-n", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 108, Mode: 0660, ModTime: testDate},
{Name: "subdir-a/file-o", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 34, Mode: 0660, ModTime: testDate},
{Name: "subdir-a/file-a", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "../file-a", Size: 23, Mode: 0777, ModTime: testDate},
{Name: "subdir-a/file-b", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "../../file-b", Size: 23, Mode: 0777, ModTime: testDate},
{Name: "subdir-a/file-c", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "/file-c", Size: 23, Mode: 0777, ModTime: testDate},
{Name: "subdir-b", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700, ModTime: testDate},
{Name: "subdir-b/file-n", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 216, Mode: 0660, ModTime: testDate},
{Name: "subdir-b/file-o", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 45, Mode: 0660, ModTime: testDate},
{Name: "subdir-c", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700, ModTime: testDate},
{Name: "subdir-c/file-n", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 432, Mode: 0666, ModTime: testDate},
{Name: "subdir-c/file-o", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 56, Mode: 0666, ModTime: testDate},
{Name: "subdir-d", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700, ModTime: testDate},
{Name: "subdir-d/hlink-0", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "../file-0", Size: 123456789, Mode: 0600, ModTime: testDate},
{Name: "subdir-d/hlink-a", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "/file-a", Size: 23, Mode: 0600, ModTime: testDate},
{Name: "subdir-d/hlink-b", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "../../file-b", Size: 23, Mode: 0600, ModTime: testDate},
{Name: "archive-a", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 0, Mode: 0600, ModTime: testDate},
},
contents: map[string][]byte{
"archive-a": testArchiveSlice,
},
expectedGetErrors: []expectedError{
{inSubdir: false, name: "link-0", err: syscall.ENOENT},
{inSubdir: false, name: "link-b", err: syscall.ENOENT},
{inSubdir: false, name: "subdir-a/file-b", err: syscall.ENOENT},
{inSubdir: true, name: "link-0", err: syscall.ENOENT},
{inSubdir: true, name: "link-b", err: syscall.ENOENT},
{inSubdir: true, name: "subdir-a/file-b", err: syscall.ENOENT},
{inSubdir: true, name: "subdir-a/file-c", err: syscall.ENOENT},
},
},
{
name: "devices",
rootOnly: true,
headers: []tar.Header{
{Name: "char-dev", Uid: uid, Gid: gid, Typeflag: tar.TypeChar, Devmajor: 0, Devminor: 0, Mode: 0600, ModTime: testDate},
{Name: "blk-dev", Uid: uid, Gid: gid, Typeflag: tar.TypeBlock, Devmajor: 0, Devminor: 0, Mode: 0600, ModTime: testDate},
},
},
}
)
func TestPutNoChroot(t *testing.T) {
couldChroot := canChroot
canChroot = false
testPut(t)
canChroot = couldChroot
}
func TestPutChroot(t *testing.T) {
if uid != 0 {
t.Skipf("chroot() requires root privileges, skipping")
}
testPut(t)
}
func testPut(t *testing.T) {
for _, topdir := range []string{"", ".", "top"} {
for i := range testArchives {
t.Run(fmt.Sprintf("topdir=%s,archive=%s", topdir, testArchives[i].name), func(t *testing.T) {
if uid != 0 && testArchives[i].rootOnly {
t.Skipf("test archive %q can only be tested with root privileges, skipping", testArchives[i].name)
}
dir, err := makeContextFromArchive(makeArchive(testArchives[i].headers, testArchives[i].contents), topdir)
require.Nil(t, err, "error creating context from archive %q, topdir=%q", testArchives[i].name, topdir)
defer os.RemoveAll(dir)
// enumerate what we expect to have created
expected := make([]enumeratedFile, 0, len(testArchives[i].headers)+1)
if topdir != "" && topdir != "." {
info, err := os.Stat(filepath.Join(dir, topdir))
require.Nil(t, err, "error statting directory %q", filepath.Join(dir, topdir))
expected = append(expected, enumeratedFile{
name: topdir,
date: info.ModTime().UTC().String(),
mode: info.Mode() & os.ModePerm,
})
}
for _, hdr := range testArchives[i].headers {
expected = append(expected, enumeratedFile{
name: filepath.Join(topdir, hdr.Name),
date: hdr.ModTime.UTC().String(),
mode: os.FileMode(hdr.Mode) & os.ModePerm,
})
}
sort.Slice(expected, func(i, j int) bool { return strings.Compare(expected[i].name, expected[j].name) < 0 })
// enumerate what we actually created
fileList, err := enumerateFiles(dir)
require.Nil(t, err, "error walking context directory for archive %q, topdir=%q", testArchives[i].name, topdir)
sort.Slice(fileList, func(i, j int) bool { return strings.Compare(fileList[i].name, fileList[j].name) < 0 })
// make sure they're the same
require.Equal(t, expected, fileList, "list of files in context directory for archive %q under topdir %q should match the archived used to populate it", testArchives[i].name, topdir)
})
}
}
}
func isExpectedError(err error, inSubdir bool, name string, expectedErrors []expectedError) bool {
// if we couldn't read that content, check if it's one of the expected failures
for _, expectedError := range expectedErrors {
if expectedError.inSubdir != inSubdir {
continue
}
if expectedError.name != name {
continue
}
if !strings.Contains(unwrapError(err).Error(), unwrapError(expectedError.err).Error()) {
// not expecting this specific error
continue
}
// it's an expected failure
return true
}
return false
}
func TestStatNoChroot(t *testing.T) {
couldChroot := canChroot
canChroot = false
testStat(t)
canChroot = couldChroot
}
func TestStatChroot(t *testing.T) {
if uid != 0 {
t.Skipf("chroot() requires root privileges, skipping")
}
testStat(t)
}
func testStat(t *testing.T) {
for _, absolute := range []bool{false, true} {
for _, topdir := range []string{"", ".", "top"} {
for _, testArchive := range testArchives {
if uid != 0 && testArchive.rootOnly {
t.Skipf("test archive %q can only be tested with root privileges, skipping", testArchive.name)
}
dir, err := makeContextFromArchive(makeArchive(testArchive.headers, testArchive.contents), topdir)
require.Nil(t, err, "error creating context from archive %q", testArchive.name)
defer os.RemoveAll(dir)
root := dir
for _, testItem := range testArchive.headers {
name := testItem.Name
if absolute {
name = filepath.Join(root, topdir, name)
}
t.Run(fmt.Sprintf("absolute=%t,topdir=%s,archive=%s,item=%s", absolute, topdir, testArchive.name, name), func(t *testing.T) {
// read stats about this item
options := StatOptions{
CheckForArchives: false,
Excludes: testArchive.excludes,
}
stats, err := Stat(root, topdir, options, []string{name})
require.Nil(t, err, "error statting %q: %v", name, err)
for _, st := range stats {
// should not have gotten an error
require.Emptyf(t, st.Error, "expected no error from stat %q", st.Glob)
// no matching characters -> should have matched one item
require.NotEmptyf(t, st.Globbed, "expected at least one match on glob %q", st.Glob)
matches := 0
for _, glob := range st.Globbed {
matches++
require.Equal(t, st.Glob, glob, "expected entry for %q", st.Glob)
require.NotNil(t, st.Results[glob], "%q globbed %q, but there are no results for it", st.Glob, glob)
toStat := glob
if !absolute {
toStat = filepath.Join(root, topdir, name)
}
_, err = os.Lstat(toStat)
require.Nil(t, err, "got error on lstat() of returned value %q(%q(%q)): %v", toStat, glob, name, err)
result := st.Results[glob]
switch testItem.Typeflag {
case tar.TypeReg, tar.TypeRegA:
if actualContent, ok := testArchive.contents[testItem.Name]; ok {
testItem.Size = int64(len(actualContent))
}
require.Equal(t, testItem.Size, result.Size, "unexpected size difference for %q", name)
require.True(t, result.IsRegular, "expected %q.IsRegular to be true", glob)
require.False(t, result.IsDir, "expected %q.IsDir to be false", glob)
require.False(t, result.IsSymlink, "expected %q.IsSymlink to be false", glob)
case tar.TypeDir:
require.False(t, result.IsRegular, "expected %q.IsRegular to be false", glob)
require.True(t, result.IsDir, "expected %q.IsDir to be true", glob)
require.False(t, result.IsSymlink, "expected %q.IsSymlink to be false", glob)
case tar.TypeSymlink:
require.True(t, result.IsSymlink, "%q is supposed to be a symbolic link, but is not", name)
require.Equal(t, testItem.Linkname, result.ImmediateTarget, "%q is supposed to point to %q, but points to %q", glob, testItem.Linkname, result.ImmediateTarget)
case tar.TypeBlock, tar.TypeChar:
require.False(t, result.IsRegular, "%q is a regular file, but is not supposed to be", name)
require.False(t, result.IsDir, "%q is a directory, but is not supposed to be", name)
require.False(t, result.IsSymlink, "%q is not supposed to be a symbolic link, but appears to be one", name)
}
}
require.Equal(t, 1, matches, "non-glob %q matched %d items, not exactly one", name, matches)
}
})
}
}
}
}
}
func TestGetSingleNoChroot(t *testing.T) {
couldChroot := canChroot
canChroot = false
testGetSingle(t)
canChroot = couldChroot
}
func TestGetSingleChroot(t *testing.T) {
if uid != 0 {
t.Skipf("chroot() requires root privileges, skipping")
}
testGetSingle(t)
}
func testGetSingle(t *testing.T) {
for _, absolute := range []bool{false, true} {
for _, topdir := range []string{"", ".", "top"} {
for _, testArchive := range testArchives {
getOptions := GetOptions{
Excludes: testArchive.excludes,
ExpandArchives: false,
}
if uid != 0 && testArchive.rootOnly {
t.Skipf("test archive %q can only be tested with root privileges, skipping", testArchive.name)
}
dir, err := makeContextFromArchive(makeArchive(testArchive.headers, testArchive.contents), topdir)
require.Nil(t, err, "error creating context from archive %q", testArchive.name)
defer os.RemoveAll(dir)
root := dir
for _, testItem := range testArchive.headers {
name := testItem.Name
if absolute {
name = filepath.Join(root, topdir, name)
}
t.Run(fmt.Sprintf("absolute=%t,topdir=%s,archive=%s,item=%s", absolute, topdir, testArchive.name, name), func(t *testing.T) {
// check if we can get this one item
err := Get(root, topdir, getOptions, []string{name}, ioutil.Discard)
// if we couldn't read that content, check if it's one of the expected failures
if err != nil && isExpectedError(err, topdir != "" && topdir != ".", testItem.Name, testArchive.expectedGetErrors) {
return
}
require.Nil(t, err, "error getting %q under %q", name, filepath.Join(root, topdir))
// we'll check subdirectories later
if testItem.Typeflag == tar.TypeDir {
return
}
// check what we get when we get this one item
pipeReader, pipeWriter := io.Pipe()
var getErr error
var wg sync.WaitGroup
wg.Add(1)
go func() {
getErr = Get(root, topdir, getOptions, []string{name}, pipeWriter)
pipeWriter.Close()
wg.Done()
}()
tr := tar.NewReader(pipeReader)
hdr, err := tr.Next()
for err == nil {
assert.Equal(t, path.Base(name), hdr.Name, "expected item named %q, got %q", path.Base(name), hdr.Name)
if _, err = io.Copy(ioutil.Discard, tr); err != nil {
break
}
hdr, err = tr.Next()
}
assert.Equal(t, io.EOF.Error(), err.Error(), "expected EOF at end of archive, got %q", err.Error())
wg.Wait()
assert.Nil(t, getErr, "unexpected error from Get(%q): %v", name, getErr)
pipeReader.Close()
})
}
}
}
}
}
func TestGetMultipleNoChroot(t *testing.T) {
couldChroot := canChroot
canChroot = false
testGetMultiple(t)
canChroot = couldChroot
}
func TestGetMultipleChroot(t *testing.T) {
if uid != 0 {
t.Skipf("chroot() requires root privileges, skipping")
}
testGetMultiple(t)
}
func testGetMultiple(t *testing.T) {
type getTestArchiveCase struct {
name string
pattern string
exclude []string
items []string
expandArchives bool
stripSetidBits bool
stripXattrs bool
keepDirectoryNames bool
}
var getTestArchives = []struct {
name string
headers []tar.Header
contents map[string][]byte
cases []getTestArchiveCase
expectedGetErrors []expectedError
}{
{
name: "regular",
headers: []tar.Header{
{Name: "file-0", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 123456789, Mode: 0600},
{Name: "file-a", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 23, Mode: 0600},
{Name: "file-b", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 23, Mode: 0600},
{Name: "link-a", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "file-a", Size: 23, Mode: 0600},
{Name: "archive-a", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 0, Mode: 0600},
{Name: "non-archive-a", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 1199, Mode: 0600},
{Name: "hlink-0", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "file-0", Size: 123456789, Mode: 0600},
{Name: "something-a", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 34, Mode: 0600},
{Name: "subdir-a", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700},
{Name: "subdir-a/file-n", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 108, Mode: 0660},
{Name: "subdir-a/file-o", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 45, Mode: 0660},
{Name: "subdir-a/file-a", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "../file-a", Size: 23, Mode: 0600},
{Name: "subdir-a/file-b", Uid: uid, Gid: gid, Typeflag: tar.TypeSymlink, Linkname: "../../file-b", Size: 23, Mode: 0600},
{Name: "subdir-a/file-c", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 56, Mode: 0600},
{Name: "subdir-b", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700},
{Name: "subdir-b/file-n", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 216, Mode: 0660},
{Name: "subdir-b/file-o", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 67, Mode: 0660},
{Name: "subdir-c", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700},
{Name: "subdir-c/file-p", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 432, Mode: 0666},
{Name: "subdir-c/file-q", Uid: uid, Gid: gid, Typeflag: tar.TypeReg, Size: 78, Mode: 0666},
{Name: "subdir-d", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700},
{Name: "subdir-d/hlink-0", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "../file-0", Size: 123456789, Mode: 0600},
{Name: "subdir-e", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700},
{Name: "subdir-e/subdir-f", Uid: uid, Gid: gid, Typeflag: tar.TypeDir, Mode: 0700},
{Name: "subdir-e/subdir-f/hlink-b", Uid: uid, Gid: gid, Typeflag: tar.TypeLink, Linkname: "../../file-b", Size: 23, Mode: 0600},
},
contents: map[string][]byte{
"archive-a": testArchiveSlice,
},
expectedGetErrors: []expectedError{
{inSubdir: true, name: ".", err: syscall.ENOENT},
},
cases: []getTestArchiveCase{
{
name: "everything",
pattern: ".",
items: []string{
"file-0",
"file-a",
"file-b",
"link-a",
"hlink-0",
"something-a",
"archive-a",
"non-archive-a",
"subdir-a",
"subdir-a/file-n",
"subdir-a/file-o",
"subdir-a/file-a",
"subdir-a/file-b",
"subdir-a/file-c",
"subdir-b",
"subdir-b/file-n",
"subdir-b/file-o",
"subdir-c",
"subdir-c/file-p",
"subdir-c/file-q",
"subdir-d",
"subdir-d/hlink-0",
"subdir-e",
"subdir-e/subdir-f",
"subdir-e/subdir-f/hlink-b",
},
},
{
name: "wildcard",
pattern: "*",
items: []string{
"file-0",
"file-a",
"file-b",
"link-a",
"hlink-0",
"something-a",
"archive-a",
"non-archive-a",
"file-n", // from subdir-a
"file-o", // from subdir-a
"file-a", // from subdir-a
"file-b", // from subdir-a
"file-c", // from subdir-a
"file-n", // from subdir-b
"file-o", // from subdir-b
"file-p", // from subdir-c
"file-q", // from subdir-c
"hlink-0", // from subdir-d
"subdir-f", // from subdir-e
"subdir-f/hlink-b", // from subdir-e
},
},
{
name: "dot-with-wildcard-includes-and-excludes",
pattern: ".",
exclude: []string{"**/*-a", "!**/*-c"},
items: []string{
"file-0",
"file-b",
"hlink-0",
"subdir-a/file-c",
"subdir-b",
"subdir-b/file-n",
"subdir-b/file-o",
"subdir-c",
"subdir-c/file-p",
"subdir-c/file-q",
"subdir-d",
"subdir-d/hlink-0",
"subdir-e",
"subdir-e/subdir-f",
"subdir-e/subdir-f/hlink-b",
},
},
{
name: "everything-with-wildcard-includes-and-excludes",
pattern: "*",
exclude: []string{"**/*-a", "!**/*-c"},
items: []string{
"file-0",
"file-b",
"file-n",
"file-o",
"file-p",
"file-q",
"hlink-0",
"hlink-0",
"subdir-f",
"subdir-f/hlink-b",
},
},
{
name: "dot-with-dot-exclude",
pattern: ".",
exclude: []string{".", "!**/*-c"},
items: []string{
"file-0",
"file-a",
"file-b",
"link-a",
"hlink-0",
"something-a",
"archive-a",
"non-archive-a",
"subdir-a",
"subdir-a/file-a",
"subdir-a/file-b",
"subdir-a/file-c",
"subdir-a/file-n",
"subdir-a/file-o",
"subdir-b",
"subdir-b/file-n",
"subdir-b/file-o",
"subdir-c",
"subdir-c/file-p",
"subdir-c/file-q",
"subdir-d",
"subdir-d/hlink-0",
"subdir-e",
"subdir-e/subdir-f",
"subdir-e/subdir-f/hlink-b",
},
},
{
name: "everything-with-dot-exclude",
pattern: "*",
exclude: []string{".", "!**/*-c"},
items: []string{
"file-0",
"file-a",
"file-a",
"file-b",
"file-b",
"file-c",
"file-n",
"file-n",
"file-o",
"file-o",
"file-p",
"file-q",
"hlink-0",
"hlink-0",
"link-a",
"something-a",
"archive-a",
"non-archive-a",
"subdir-f",
"subdir-f/hlink-b",
},
},
{
name: "all-with-all-exclude",
pattern: "*",
exclude: []string{"*", "!**/*-c"},
items: []string{
"file-p",
"file-q",
},
},
{
name: "everything-with-all-exclude",
pattern: ".",
exclude: []string{"*", "!**/*-c"},
items: []string{
"subdir-a/file-c",
"subdir-c",
"subdir-c/file-p",
"subdir-c/file-q",
},
},
{
name: "file-wildcard",
pattern: "file-*",
items: []string{
"file-0",
"file-a",
"file-b",
},
},
{
name: "file-and-dir-wildcard",
pattern: "*-a",
items: []string{
"file-a",
"link-a",
"something-a",
"archive-a",
"non-archive-a",
"file-n", // from subdir-a
"file-o", // from subdir-a
"file-a", // from subdir-a
"file-b", // from subdir-a
"file-c", // from subdir-a
},
},
{
name: "file-and-dir-wildcard-with-exclude",
pattern: "*-a",
exclude: []string{"subdir-a", "top/subdir-a"},
items: []string{
"file-a",
"link-a",
"something-a",
"archive-a",
"non-archive-a",
},
},
{
name: "file-and-dir-wildcard-with-wildcard-exclude",
pattern: "*-a",
exclude: []string{"subdir*", "top/subdir*"},
items: []string{
"file-a",
"link-a",
"something-a",
"archive-a",
"non-archive-a",
},
},
{
name: "file-and-dir-wildcard-with-deep-exclude",
pattern: "*-a",
exclude: []string{"**/subdir-a"},
items: []string{
"file-a",
"link-a",
"something-a",
"archive-a",
"non-archive-a",
},
},
{
name: "file-and-dir-wildcard-with-wildcard-deep-exclude",
pattern: "*-a",
exclude: []string{"**/subdir*"},
items: []string{
"file-a",
"link-a",
"something-a",
"archive-a",
"non-archive-a",
},
},
{
name: "file-and-dir-wildcard-with-deep-include",
pattern: "*-a",
exclude: []string{"**/subdir-a", "!**/file-c"},
items: []string{
"file-a",
"link-a",
"something-a",
"archive-a",
"non-archive-a",
// no "file-c" from subdir-a
},
},
{
name: "file-and-dir-wildcard-with-wildcard-deep-include",
pattern: "*-a",
exclude: []string{"**/subdir*", "!**/file-c"},
items: []string{
"file-a",
"link-a",
"something-a",
"archive-a",
"non-archive-a",
// no "file-c" from subdir-a
},
},
{
name: "subdirectory",
pattern: "subdir-e",
items: []string{
"subdir-f",
"subdir-f/hlink-b",
},
},
{
name: "subdirectory-wildcard",
pattern: "*/subdir-*",
items: []string{
"hlink-b", // from subdir-e/subdir-f
},
},
{
name: "not-expanded-archive",
pattern: "*archive-a",
items: []string{
"archive-a",
"non-archive-a",
},
},
{
name: "expanded-archive",
pattern: "*archive-a",
expandArchives: true,
items: []string{
"non-archive-a",
"item-0",
"item-1",
"item-2",
},
},
{
name: "subdir-without-name",
pattern: "subdir-e",
items: []string{
"subdir-f",
"subdir-f/hlink-b",
},
},
{
name: "subdir-with-name",
pattern: "subdir-e",
keepDirectoryNames: true,
items: []string{
"subdir-e",
"subdir-e/subdir-f",
"subdir-e/subdir-f/hlink-b",
},
},
},
},
}
for _, topdir := range []string{"", ".", "top"} {
for _, testArchive := range getTestArchives {
dir, err := makeContextFromArchive(makeArchive(testArchive.headers, testArchive.contents), topdir)
require.Nil(t, err, "error creating context from archive %q", testArchive.name)
defer os.RemoveAll(dir)
root := dir
cases := make(map[string]struct{})
for _, testCase := range testArchive.cases {
if _, ok := cases[testCase.name]; ok {
t.Fatalf("duplicate case %q", testCase.name)
}
cases[testCase.name] = struct{}{}
}
for _, testCase := range testArchive.cases {
getOptions := GetOptions{
Excludes: testCase.exclude,
ExpandArchives: testCase.expandArchives,
StripSetidBits: testCase.stripSetidBits,
StripXattrs: testCase.stripXattrs,
KeepDirectoryNames: testCase.keepDirectoryNames,
}
t.Run(fmt.Sprintf("topdir=%s,archive=%s,case=%s,pattern=%s", topdir, testArchive.name, testCase.name, testCase.pattern), func(t *testing.T) {
// ensure that we can get stuff using this spec
err := Get(root, topdir, getOptions, []string{testCase.pattern}, ioutil.Discard)
if err != nil && isExpectedError(err, topdir != "" && topdir != ".", testCase.pattern, testArchive.expectedGetErrors) {
return
}
require.Nil(t, err, "error getting %q under %q", testCase.pattern, filepath.Join(root, topdir))
// see what we get when we get this pattern
pipeReader, pipeWriter := io.Pipe()
var getErr error
var wg sync.WaitGroup
wg.Add(1)
go func() {
getErr = Get(root, topdir, getOptions, []string{testCase.pattern}, pipeWriter)
pipeWriter.Close()
wg.Done()
}()
tr := tar.NewReader(pipeReader)
hdr, err := tr.Next()
actualContents := []string{}
for err == nil {
actualContents = append(actualContents, hdr.Name)
if _, err = io.Copy(ioutil.Discard, tr); err != nil {
break
}
hdr, err = tr.Next()
}
pipeReader.Close()
sort.Strings(actualContents)
// compare it to what we were supposed to get
expectedContents := append([]string{}, testCase.items...)
sort.Strings(expectedContents)
assert.Equal(t, io.EOF.Error(), err.Error(), "expected EOF at end of archive, got %q", err.Error())
wg.Wait()
assert.Nil(t, getErr, "unexpected error from Get(%q)", testCase.pattern)
assert.Equal(t, expectedContents, actualContents, "Get(%q,excludes=%v) didn't produce the right set of items", testCase.pattern, testCase.exclude)
})
}
}
}
}

66
copier/syscall_unix.go Normal file
View File

@ -0,0 +1,66 @@
// +build !windows
package copier
import (
"fmt"
"os"
"time"
"golang.org/x/sys/unix"
)
var canChroot = true
func chroot(root string) (bool, error) {
if canChroot {
if err := os.Chdir(root); err != nil {
return false, fmt.Errorf("error changing to intended-new-root directory %q: %v", root, err)
}
if err := unix.Chroot(root); err != nil {
return false, fmt.Errorf("error chrooting to directory %q: %v", root, err)
}
if err := os.Chdir(string(os.PathSeparator)); err != nil {
return false, fmt.Errorf("error changing to just-became-root directory %q: %v", root, err)
}
return true, nil
}
return false, nil
}
func getcwd() (string, error) {
return unix.Getwd()
}
func chrMode(mode os.FileMode) uint32 {
return uint32(unix.S_IFCHR | mode)
}
func blkMode(mode os.FileMode) uint32 {
return uint32(unix.S_IFBLK | mode)
}
func mkdev(major, minor uint32) uint64 {
return unix.Mkdev(major, minor)
}
func mkfifo(path string, mode uint32) error {
return unix.Mkfifo(path, mode)
}
func mknod(path string, mode uint32, dev int) error {
return unix.Mknod(path, mode, dev)
}
func lutimes(isSymlink bool, path string, atime, mtime time.Time) error {
if atime.IsZero() || mtime.IsZero() {
now := time.Now()
if atime.IsZero() {
atime = now
}
if mtime.IsZero() {
mtime = now
}
}
return unix.Lutimes(path, []unix.Timeval{unix.NsecToTimeval(atime.UnixNano()), unix.NsecToTimeval(mtime.UnixNano())})
}

57
copier/syscall_windows.go Normal file
View File

@ -0,0 +1,57 @@
// +build windows
package copier
import (
"os"
"syscall"
"time"
"golang.org/x/sys/windows"
)
var canChroot = false
func chroot(path string) (bool, error) {
return false, nil
}
func getcwd() (string, error) {
return windows.Getwd()
}
func chrMode(mode os.FileMode) uint32 {
return windows.S_IFCHR | uint32(mode)
}
func blkMode(mode os.FileMode) uint32 {
return windows.S_IFBLK | uint32(mode)
}
func mkdev(major, minor uint32) uint64 {
return 0
}
func mkfifo(path string, mode uint32) error {
return syscall.ENOSYS
}
func mknod(path string, mode uint32, dev int) error {
return syscall.ENOSYS
}
func lutimes(isSymlink bool, path string, atime, mtime time.Time) error {
if isSymlink {
return nil
}
if atime.IsZero() || mtime.IsZero() {
now := time.Now()
if atime.IsZero() {
atime = now
}
if mtime.IsZero() {
mtime = now
}
}
return windows.UtimesNano(path, []windows.Timespec{windows.NsecToTimespec(atime.UnixNano()), windows.NsecToTimespec(mtime.UnixNano())})
}

11
copier/unwrap_112.go Normal file
View File

@ -0,0 +1,11 @@
// +build !go113
package copier
import (
"github.com/pkg/errors"
)
func unwrapError(err error) error {
return errors.Cause(err)
}

18
copier/unwrap_113.go Normal file
View File

@ -0,0 +1,18 @@
// +build go113
package copier
import (
stderror "errors"
"github.com/pkg/errors"
)
func unwrapError(err error) error {
e := errors.Cause(err)
for e != nil {
err = e
e = errors.Unwrap(err)
}
return err
}

87
copier/xattrs.go Normal file
View File

@ -0,0 +1,87 @@
// +build linux netbsd freebsd darwin
package copier
import (
"path/filepath"
"strings"
"syscall"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
const (
xattrsSupported = true
)
var (
relevantAttributes = []string{"security.capability", "security.ima", "user.*"} // the attributes that we preserve - we discard others
)
// Lgetxattrs returns a map of the relevant extended attributes set on the given file.
func Lgetxattrs(path string) (map[string]string, error) {
maxSize := 64 * 1024 * 1024
listSize := 64 * 1024
var list []byte
for listSize < maxSize {
list = make([]byte, listSize)
size, err := unix.Llistxattr(path, list)
if err != nil {
if unwrapError(err) == syscall.E2BIG {
listSize *= 2
continue
}
return nil, errors.Wrapf(err, "error listing extended attributes of %q", path)
}
list = list[:size]
break
}
if listSize >= maxSize {
return nil, errors.Errorf("unable to read list of attributes for %q: size would have been too big", path)
}
m := make(map[string]string)
for _, attribute := range strings.Split(string(list), string('\000')) {
for _, relevant := range relevantAttributes {
matched, err := filepath.Match(relevant, attribute)
if err != nil || !matched {
continue
}
attributeSize := 64 * 1024
var attributeValue []byte
for attributeSize < maxSize {
attributeValue = make([]byte, attributeSize)
size, err := unix.Lgetxattr(path, attribute, attributeValue)
if err != nil {
if unwrapError(err) == syscall.E2BIG {
attributeSize *= 2
continue
}
return nil, errors.Wrapf(err, "error getting value of extended attribute %q on %q", attribute, path)
}
m[attribute] = string(attributeValue[:size])
break
}
if attributeSize >= maxSize {
return nil, errors.Errorf("unable to read attribute %q of %q: size would have been too big", attribute, path)
}
}
}
return m, nil
}
// Lsetxattrs sets the relevant members of the specified extended attributes on the given file.
func Lsetxattrs(path string, xattrs map[string]string) error {
for attribute, value := range xattrs {
for _, relevant := range relevantAttributes {
matched, err := filepath.Match(relevant, attribute)
if err != nil || !matched {
continue
}
if err = unix.Lsetxattr(path, attribute, []byte(value), 0); err != nil {
return errors.Wrapf(err, "error setting value of extended attribute %q on %q", attribute, path)
}
}
}
return nil
}

55
copier/xattrs_test.go Normal file
View File

@ -0,0 +1,55 @@
package copier
import (
"fmt"
"io/ioutil"
"os"
"syscall"
"testing"
"github.com/stretchr/testify/assert"
)
func TestXattrs(t *testing.T) {
if !xattrsSupported {
t.Skipf("xattrs are not supported on this platform, skipping")
}
testValues := map[string]string{
"user.a": "attribute value a",
"user.b": "attribute value b",
}
tmp, err := ioutil.TempDir("", "copier-xattr-test-")
if !assert.Nil(t, err, "error creating test directory: %v", err) {
t.FailNow()
}
defer os.RemoveAll(tmp)
for attribute, value := range testValues {
t.Run(fmt.Sprintf("attribute=%s", attribute), func(t *testing.T) {
f, err := ioutil.TempFile(tmp, "copier-xattr-test-")
if !assert.Nil(t, err, "error creating test file: %v", err) {
t.FailNow()
}
defer os.Remove(f.Name())
err = Lsetxattrs(f.Name(), map[string]string{attribute: value})
if unwrapError(err) == syscall.ENOTSUP {
t.Skip(fmt.Sprintf("extended attributes not supported on %q, skipping", tmp))
}
if !assert.Nil(t, err, "error setting attribute on file: %v", err) {
t.FailNow()
}
xattrs, err := Lgetxattrs(f.Name())
if !assert.Nil(t, err, "error reading attributes of file: %v", err) {
t.FailNow()
}
xvalue, ok := xattrs[attribute]
if !assert.True(t, ok, "did not read back attribute %q for file", attribute) {
t.FailNow()
}
if !assert.Equal(t, value, xvalue, "read back different value for attribute %q", attribute) {
t.FailNow()
}
})
}
}

15
copier/xattrs_unsupported.go Normal file
View File

@ -0,0 +1,15 @@
// +build !linux,!netbsd,!freebsd,!darwin
package copier
const (
xattrsSupported = false
)
func Lgetxattrs(path string) (map[string]string, error) {
return nil, nil
}
func Lsetxattrs(path string, xattrs map[string]string) error {
return nil
}