Vendor in latest container/image

Adds support for converting manifest types when using the dir transport

Signed-off-by: Urvashi Mohnani <umohnani@redhat.com>

Closes: #321
Approved by: rhatdan
This commit is contained in:
Urvashi Mohnani 2017-11-09 10:12:58 -05:00 committed by Atomic Bot
parent 4f376bbb5e
commit 5ececfad2c
35 changed files with 1290 additions and 234 deletions

View File

@ -12,8 +12,6 @@ import (
"strings"
"time"
pb "gopkg.in/cheggaaa/pb.v1"
"github.com/containers/image/image"
"github.com/containers/image/pkg/compression"
"github.com/containers/image/signature"
@ -22,6 +20,7 @@ import (
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
pb "gopkg.in/cheggaaa/pb.v1"
)
type digestingReader struct {
@ -95,6 +94,8 @@ type Options struct {
DestinationCtx *types.SystemContext
ProgressInterval time.Duration // time to wait between reports to signal the progress channel
Progress chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset.
// manifest MIME type of image set by user. "" is default and means use the autodetection to the the manifest MIME type
ForceManifestMIMEType string
}
// Image copies image from srcRef to destRef, using policyContext to validate
@ -193,7 +194,7 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe
// We compute preferredManifestMIMEType only to show it in error messages.
// Without having to add this context in an error message, we would be happy enough to know only that no conversion is needed.
preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := determineManifestConversion(&manifestUpdates, src, dest.SupportedManifestMIMETypes(), canModifyManifest)
preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := determineManifestConversion(&manifestUpdates, src, dest.SupportedManifestMIMETypes(), canModifyManifest, options.ForceManifestMIMEType)
if err != nil {
return err
}

View File

@ -41,12 +41,16 @@ func (os *orderedSet) append(s string) {
// Note that the conversion will only happen later, through src.UpdatedImage
// Returns the preferred manifest MIME type (whether we are converting to it or using it unmodified),
// and a list of other possible alternatives, in order.
func determineManifestConversion(manifestUpdates *types.ManifestUpdateOptions, src types.Image, destSupportedManifestMIMETypes []string, canModifyManifest bool) (string, []string, error) {
func determineManifestConversion(manifestUpdates *types.ManifestUpdateOptions, src types.Image, destSupportedManifestMIMETypes []string, canModifyManifest bool, forceManifestMIMEType string) (string, []string, error) {
_, srcType, err := src.Manifest()
if err != nil { // This should have been cached?!
return "", nil, errors.Wrap(err, "Error reading manifest")
}
if forceManifestMIMEType != "" {
destSupportedManifestMIMETypes = []string{forceManifestMIMEType}
}
if len(destSupportedManifestMIMETypes) == 0 {
return srcType, []string{}, nil // Anything goes; just use the original as is, do not try any conversions.
}

View File

@ -4,19 +4,77 @@ import (
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/containers/image/types"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const version = "Directory Transport Version: 1.0\n"
// ErrNotContainerImageDir indicates that the directory doesn't match the expected contents of a directory created
// using the 'dir' transport
var ErrNotContainerImageDir = errors.New("not a containers image directory, don't want to overwrite important data")
type dirImageDestination struct {
ref dirReference
ref dirReference
compress bool
}
// newImageDestination returns an ImageDestination for writing to an existing directory.
func newImageDestination(ref dirReference) types.ImageDestination {
return &dirImageDestination{ref}
// newImageDestination returns an ImageDestination for writing to a directory.
func newImageDestination(ref dirReference, compress bool) (types.ImageDestination, error) {
d := &dirImageDestination{ref: ref, compress: compress}
// If directory exists check if it is empty
// if not empty, check whether the contents match that of a container image directory and overwrite the contents
// if the contents don't match throw an error
dirExists, err := pathExists(d.ref.resolvedPath)
if err != nil {
return nil, errors.Wrapf(err, "error checking for path %q", d.ref.resolvedPath)
}
if dirExists {
isEmpty, err := isDirEmpty(d.ref.resolvedPath)
if err != nil {
return nil, err
}
if !isEmpty {
versionExists, err := pathExists(d.ref.versionPath())
if err != nil {
return nil, errors.Wrapf(err, "error checking if path exists %q", d.ref.versionPath())
}
if versionExists {
contents, err := ioutil.ReadFile(d.ref.versionPath())
if err != nil {
return nil, err
}
// check if contents of version file is what we expect it to be
if string(contents) != version {
return nil, ErrNotContainerImageDir
}
} else {
return nil, ErrNotContainerImageDir
}
// delete directory contents so that only one image is in the directory at a time
if err = removeDirContents(d.ref.resolvedPath); err != nil {
return nil, errors.Wrapf(err, "error erasing contents in %q", d.ref.resolvedPath)
}
logrus.Debugf("overwriting existing container image directory %q", d.ref.resolvedPath)
}
} else {
// create directory if it doesn't exist
if err := os.MkdirAll(d.ref.resolvedPath, 0755); err != nil {
return nil, errors.Wrapf(err, "unable to create directory %q", d.ref.resolvedPath)
}
}
// create version file
err = ioutil.WriteFile(d.ref.versionPath(), []byte(version), 0755)
if err != nil {
return nil, errors.Wrapf(err, "error creating version file %q", d.ref.versionPath())
}
return d, nil
}
// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent,
@ -42,7 +100,7 @@ func (d *dirImageDestination) SupportsSignatures() error {
// ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination.
func (d *dirImageDestination) ShouldCompressLayers() bool {
return false
return d.compress
}
// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
@ -147,3 +205,39 @@ func (d *dirImageDestination) PutSignatures(signatures [][]byte) error {
func (d *dirImageDestination) Commit() error {
return nil
}
// returns true if path exists
func pathExists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if err != nil && os.IsNotExist(err) {
return false, nil
}
return false, err
}
// returns true if directory is empty
func isDirEmpty(path string) (bool, error) {
files, err := ioutil.ReadDir(path)
if err != nil {
return false, err
}
return len(files) == 0, nil
}
// deletes the contents of a directory
func removeDirContents(path string) error {
files, err := ioutil.ReadDir(path)
if err != nil {
return err
}
for _, file := range files {
if err := os.RemoveAll(filepath.Join(path, file.Name())); err != nil {
return err
}
}
return nil
}

View File

@ -152,7 +152,11 @@ func (ref dirReference) NewImageSource(ctx *types.SystemContext) (types.ImageSou
// NewImageDestination returns a types.ImageDestination for this reference.
// The caller must call .Close() on the returned ImageDestination.
func (ref dirReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) {
return newImageDestination(ref), nil
compress := false
if ctx != nil {
compress = ctx.DirForceCompress
}
return newImageDestination(ref, compress)
}
// DeleteImage deletes the named image from the registry, if supported.
@ -175,3 +179,8 @@ func (ref dirReference) layerPath(digest digest.Digest) string {
func (ref dirReference) signaturePath(index int) string {
return filepath.Join(ref.path, fmt.Sprintf("signature-%d", index+1))
}
// versionPath returns a path for the version file within a directory using our conventions.
func (ref dirReference) versionPath() string {
return filepath.Join(ref.path, "version")
}

View File

@ -0,0 +1,69 @@
package daemon
import (
"net/http"
"path/filepath"
"github.com/containers/image/types"
dockerclient "github.com/docker/docker/client"
"github.com/docker/go-connections/tlsconfig"
)
const (
// The default API version to be used in case none is explicitly specified
defaultAPIVersion = "1.22"
)
// NewDockerClient initializes a new API client based on the passed SystemContext.
func newDockerClient(ctx *types.SystemContext) (*dockerclient.Client, error) {
host := dockerclient.DefaultDockerHost
if ctx != nil && ctx.DockerDaemonHost != "" {
host = ctx.DockerDaemonHost
}
// Sadly, unix:// sockets don't work transparently with dockerclient.NewClient.
// They work fine with a nil httpClient; with a non-nil httpClient, the transports
// TLSClientConfig must be nil (or the client will try using HTTPS over the PF_UNIX socket
// regardless of the values in the *tls.Config), and we would have to call sockets.ConfigureTransport.
//
// We don't really want to configure anything for unix:// sockets, so just pass a nil *http.Client.
proto, _, _, err := dockerclient.ParseHost(host)
if err != nil {
return nil, err
}
var httpClient *http.Client
if proto != "unix" {
hc, err := tlsConfig(ctx)
if err != nil {
return nil, err
}
httpClient = hc
}
return dockerclient.NewClient(host, defaultAPIVersion, httpClient, nil)
}
func tlsConfig(ctx *types.SystemContext) (*http.Client, error) {
options := tlsconfig.Options{}
if ctx != nil && ctx.DockerDaemonInsecureSkipTLSVerify {
options.InsecureSkipVerify = true
}
if ctx != nil && ctx.DockerDaemonCertPath != "" {
options.CAFile = filepath.Join(ctx.DockerDaemonCertPath, "ca.pem")
options.CertFile = filepath.Join(ctx.DockerDaemonCertPath, "cert.pem")
options.KeyFile = filepath.Join(ctx.DockerDaemonCertPath, "key.pem")
}
tlsc, err := tlsconfig.Client(options)
if err != nil {
return nil, err
}
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsc,
},
CheckRedirect: dockerclient.CheckRedirect,
}, nil
}

View File

@ -24,7 +24,7 @@ type daemonImageDestination struct {
}
// newImageDestination returns a types.ImageDestination for the specified image reference.
func newImageDestination(systemCtx *types.SystemContext, ref daemonReference) (types.ImageDestination, error) {
func newImageDestination(ctx *types.SystemContext, ref daemonReference) (types.ImageDestination, error) {
if ref.ref == nil {
return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport())
}
@ -33,7 +33,7 @@ func newImageDestination(systemCtx *types.SystemContext, ref daemonReference) (t
return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport())
}
c, err := client.NewClient(client.DefaultDockerHost, "1.22", nil, nil) // FIXME: overridable host
c, err := newDockerClient(ctx)
if err != nil {
return nil, errors.Wrap(err, "Error initializing docker engine client")
}
@ -42,8 +42,8 @@ func newImageDestination(systemCtx *types.SystemContext, ref daemonReference) (t
// Commit() may never be called, so we may never read from this channel; so, make this buffered to allow imageLoadGoroutine to write status and terminate even if we never read it.
statusChannel := make(chan error, 1)
ctx, goroutineCancel := context.WithCancel(context.Background())
go imageLoadGoroutine(ctx, c, reader, statusChannel)
goroutineContext, goroutineCancel := context.WithCancel(context.Background())
go imageLoadGoroutine(goroutineContext, c, reader, statusChannel)
return &daemonImageDestination{
ref: ref,

View File

@ -7,7 +7,6 @@ import (
"github.com/containers/image/docker/tarfile"
"github.com/containers/image/types"
"github.com/docker/docker/client"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
@ -35,7 +34,7 @@ type layerInfo struct {
// is the config, and that the following len(RootFS) files are the layers, but that feels
// way too brittle.)
func newImageSource(ctx *types.SystemContext, ref daemonReference) (types.ImageSource, error) {
c, err := client.NewClient(client.DefaultDockerHost, "1.22", nil, nil) // FIXME: overridable host
c, err := newDockerClient(ctx)
if err != nil {
return nil, errors.Wrap(err, "Error initializing docker engine client")
}

View File

@ -3,23 +3,20 @@ package docker
import (
"context"
"crypto/tls"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/containers/image/docker/reference"
"github.com/containers/image/pkg/docker/config"
"github.com/containers/image/pkg/tlsclientconfig"
"github.com/containers/image/types"
"github.com/containers/storage/pkg/homedir"
"github.com/docker/distribution/registry/client"
helperclient "github.com/docker/docker-credential-helpers/client"
"github.com/docker/go-connections/tlsconfig"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
@ -27,13 +24,8 @@ import (
)
const (
dockerHostname = "docker.io"
dockerRegistry = "registry-1.docker.io"
dockerAuthRegistry = "https://index.docker.io/v1/"
dockerCfg = ".docker"
dockerCfgFileName = "config.json"
dockerCfgObsolete = ".dockercfg"
dockerHostname = "docker.io"
dockerRegistry = "registry-1.docker.io"
systemPerHostCertDirPath = "/etc/docker/certs.d"
@ -51,9 +43,13 @@ const (
extensionSignatureTypeAtomic = "atomic" // extensionSignature.Type
)
// ErrV1NotSupported is returned when we're trying to talk to a
// docker V1 registry.
var ErrV1NotSupported = errors.New("can't talk to a V1 docker registry")
var (
// ErrV1NotSupported is returned when we're trying to talk to a
// docker V1 registry.
ErrV1NotSupported = errors.New("can't talk to a V1 docker registry")
// ErrUnauthorizedForCredentials is returned when the status code returned is 401
ErrUnauthorizedForCredentials = errors.New("unable to retrieve auth token: invalid username/password")
)
// extensionSignature and extensionSignatureList come from github.com/openshift/origin/pkg/dockerregistry/server/signaturedispatcher.go:
// signature represents a Docker image signature.
@ -128,52 +124,84 @@ func dockerCertDir(ctx *types.SystemContext, hostPort string) string {
return filepath.Join(hostCertDir, hostPort)
}
// newDockerClient returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry)
// newDockerClientFromRef returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry)
// “write” specifies whether the client will be used for "write" access (in particular passed to lookaside.go:toplevelFromSection)
func newDockerClient(ctx *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) {
func newDockerClientFromRef(ctx *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) {
registry := reference.Domain(ref.ref)
if registry == dockerHostname {
registry = dockerRegistry
username, password, err := config.GetAuthentication(ctx, reference.Domain(ref.ref))
if err != nil {
return nil, errors.Wrapf(err, "error getting username and password")
}
username, password, err := getAuth(ctx, reference.Domain(ref.ref))
sigBase, err := configuredSignatureStorageBase(ctx, ref, write)
if err != nil {
return nil, err
}
remoteName := reference.Path(ref.ref)
return newDockerClientWithDetails(ctx, registry, username, password, actions, sigBase, remoteName)
}
// newDockerClientWithDetails returns a new dockerClient instance for the given parameters
func newDockerClientWithDetails(ctx *types.SystemContext, registry, username, password, actions string, sigBase signatureStorageBase, remoteName string) (*dockerClient, error) {
hostName := registry
if registry == dockerHostname {
registry = dockerRegistry
}
tr := tlsclientconfig.NewTransport()
tr.TLSClientConfig = serverDefault()
// It is undefined whether the host[:port] string for dockerHostname should be dockerHostname or dockerRegistry,
// because docker/docker does not read the certs.d subdirectory at all in that case. We use the user-visible
// dockerHostname here, because it is more symmetrical to read the configuration in that case as well, and because
// generally the UI hides the existence of the different dockerRegistry. But note that this behavior is
// undocumented and may change if docker/docker changes.
certDir := dockerCertDir(ctx, reference.Domain(ref.ref))
certDir := dockerCertDir(ctx, hostName)
if err := tlsclientconfig.SetupCertificates(certDir, tr.TLSClientConfig); err != nil {
return nil, err
}
if ctx != nil && ctx.DockerInsecureSkipTLSVerify {
tr.TLSClientConfig.InsecureSkipVerify = true
}
client := &http.Client{Transport: tr}
sigBase, err := configuredSignatureStorageBase(ctx, ref, write)
if err != nil {
return nil, err
}
return &dockerClient{
ctx: ctx,
registry: registry,
username: username,
password: password,
client: client,
client: &http.Client{Transport: tr},
signatureBase: sigBase,
scope: authScope{
actions: actions,
remoteName: reference.Path(ref.ref),
remoteName: remoteName,
},
}, nil
}
// CheckAuth validates the credentials by attempting to log into the registry
// returns an error if an error occcured while making the http request or the status code received was 401
func CheckAuth(ctx context.Context, sCtx *types.SystemContext, username, password, registry string) error {
newLoginClient, err := newDockerClientWithDetails(sCtx, registry, username, password, "", nil, "")
if err != nil {
return errors.Wrapf(err, "error creating new docker client")
}
resp, err := newLoginClient.makeRequest(ctx, "GET", "/v2/", nil, nil)
if err != nil {
return err
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
return nil
case http.StatusUnauthorized:
return ErrUnauthorizedForCredentials
default:
return errors.Errorf("error occured with status code %q", resp.StatusCode)
}
}
// makeRequest creates and executes a http.Request with the specified parameters, adding authentication and TLS options for the Docker client.
// The host name and schema is taken from the client or autodetected, and the path is relative to it, i.e. the path usually starts with /v2/.
func (c *dockerClient) makeRequest(ctx context.Context, method, path string, headers map[string][]string, stream io.Reader) (*http.Response, error) {
@ -245,7 +273,10 @@ func (c *dockerClient) setupRequestAuth(req *http.Request) error {
return errors.Errorf("missing realm in bearer auth challenge")
}
service, _ := challenge.Parameters["service"] // Will be "" if not present
scope := fmt.Sprintf("repository:%s:%s", c.scope.remoteName, c.scope.actions)
var scope string
if c.scope.remoteName != "" && c.scope.actions != "" {
scope = fmt.Sprintf("repository:%s:%s", c.scope.remoteName, c.scope.actions)
}
token, err := c.getBearerToken(req.Context(), realm, service, scope)
if err != nil {
return err
@ -291,7 +322,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope
defer res.Body.Close()
switch res.StatusCode {
case http.StatusUnauthorized:
return nil, errors.Errorf("unable to retrieve auth token: 401 unauthorized")
return nil, ErrUnauthorizedForCredentials
case http.StatusOK:
break
default:
@ -315,65 +346,6 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope
return &token, nil
}
func getAuth(ctx *types.SystemContext, registry string) (string, string, error) {
if ctx != nil && ctx.DockerAuthConfig != nil {
return ctx.DockerAuthConfig.Username, ctx.DockerAuthConfig.Password, nil
}
var dockerAuth dockerConfigFile
dockerCfgPath := filepath.Join(getDefaultConfigDir(".docker"), dockerCfgFileName)
if _, err := os.Stat(dockerCfgPath); err == nil {
j, err := ioutil.ReadFile(dockerCfgPath)
if err != nil {
return "", "", err
}
if err := json.Unmarshal(j, &dockerAuth); err != nil {
return "", "", err
}
} else if os.IsNotExist(err) {
// try old config path
oldDockerCfgPath := filepath.Join(getDefaultConfigDir(dockerCfgObsolete))
if _, err := os.Stat(oldDockerCfgPath); err != nil {
if os.IsNotExist(err) {
return "", "", nil
}
return "", "", errors.Wrap(err, oldDockerCfgPath)
}
j, err := ioutil.ReadFile(oldDockerCfgPath)
if err != nil {
return "", "", err
}
if err := json.Unmarshal(j, &dockerAuth.AuthConfigs); err != nil {
return "", "", err
}
} else if err != nil {
return "", "", errors.Wrap(err, dockerCfgPath)
}
// First try cred helpers. They should always be normalized.
if ch, exists := dockerAuth.CredHelpers[registry]; exists {
return getAuthFromCredHelper(ch, registry)
}
// I'm feeling lucky.
if c, exists := dockerAuth.AuthConfigs[registry]; exists {
return decodeDockerAuth(c.Auth)
}
// bad luck; let's normalize the entries first
registry = normalizeRegistry(registry)
normalizedAuths := map[string]dockerAuthConfig{}
for k, v := range dockerAuth.AuthConfigs {
normalizedAuths[normalizeRegistry(k)] = v
}
if c, exists := normalizedAuths[registry]; exists {
return decodeDockerAuth(c.Auth)
}
return "", "", nil
}
// detectProperties detects various properties of the registry.
// See the dockerClient documentation for members which are affected by this.
func (c *dockerClient) detectProperties(ctx context.Context) error {
@ -456,67 +428,3 @@ func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerRe
}
return &parsedBody, nil
}
func getDefaultConfigDir(confPath string) string {
return filepath.Join(homedir.Get(), confPath)
}
type dockerAuthConfig struct {
Auth string `json:"auth,omitempty"`
}
type dockerConfigFile struct {
AuthConfigs map[string]dockerAuthConfig `json:"auths"`
CredHelpers map[string]string `json:"credHelpers,omitempty"`
}
func getAuthFromCredHelper(credHelper, registry string) (string, string, error) {
helperName := fmt.Sprintf("docker-credential-%s", credHelper)
p := helperclient.NewShellProgramFunc(helperName)
creds, err := helperclient.Get(p, registry)
if err != nil {
return "", "", err
}
return creds.Username, creds.Secret, nil
}
func decodeDockerAuth(s string) (string, string, error) {
decoded, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return "", "", err
}
parts := strings.SplitN(string(decoded), ":", 2)
if len(parts) != 2 {
// if it's invalid just skip, as docker does
return "", "", nil
}
user := parts[0]
password := strings.Trim(parts[1], "\x00")
return user, password, nil
}
// convertToHostname converts a registry url which has http|https prepended
// to just an hostname.
// Copied from github.com/docker/docker/registry/auth.go
func convertToHostname(url string) string {
stripped := url
if strings.HasPrefix(url, "http://") {
stripped = strings.TrimPrefix(url, "http://")
} else if strings.HasPrefix(url, "https://") {
stripped = strings.TrimPrefix(url, "https://")
}
nameParts := strings.SplitN(stripped, "/", 2)
return nameParts[0]
}
func normalizeRegistry(registry string) string {
normalized := convertToHostname(registry)
switch normalized {
case "registry-1.docker.io", "docker.io":
return "index.docker.io"
}
return normalized
}

View File

@ -34,7 +34,7 @@ type dockerImageDestination struct {
// newImageDestination creates a new ImageDestination for the specified image reference.
func newImageDestination(ctx *types.SystemContext, ref dockerReference) (types.ImageDestination, error) {
c, err := newDockerClient(ctx, ref, true, "pull,push")
c, err := newDockerClientFromRef(ctx, ref, true, "pull,push")
if err != nil {
return nil, err
}
@ -236,7 +236,7 @@ func (d *dockerImageDestination) PutManifest(m []byte) error {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusCreated {
if !successStatus(res.StatusCode) {
err = errors.Wrapf(client.HandleErrorResponse(res), "Error uploading manifest to %s", path)
if isManifestInvalidError(errors.Cause(err)) {
err = types.ManifestTypeRejectedError{Err: err}
@ -246,6 +246,12 @@ func (d *dockerImageDestination) PutManifest(m []byte) error {
return nil
}
// successStatus returns true if the argument is a successful HTTP response
// code (in the range 200 - 399 inclusive).
func successStatus(status int) bool {
return status >= 200 && status <= 399
}
// isManifestInvalidError returns true iff err from client.HandleErrorReponse is a “manifest invalid” error.
func isManifestInvalidError(err error) bool {
errors, ok := err.(errcode.Errors)

View File

@ -31,7 +31,7 @@ type dockerImageSource struct {
// newImageSource creates a new ImageSource for the specified image reference.
// The caller must call .Close() on the returned ImageSource.
func newImageSource(ctx *types.SystemContext, ref dockerReference) (*dockerImageSource, error) {
c, err := newDockerClient(ctx, ref, false, "pull")
c, err := newDockerClientFromRef(ctx, ref, false, "pull")
if err != nil {
return nil, err
}
@ -298,7 +298,7 @@ func (s *dockerImageSource) getSignaturesFromAPIExtension(ctx context.Context) (
// deleteImage deletes the named image from the registry, if supported.
func deleteImage(ctx *types.SystemContext, ref dockerReference) error {
c, err := newDockerClient(ctx, ref, true, "push")
c, err := newDockerClientFromRef(ctx, ref, true, "push")
if err != nil {
return err
}

View File

@ -161,14 +161,17 @@ func (m *manifestSchema1) imageInspectInfo() (*types.ImageInspectInfo, error) {
if err := json.Unmarshal([]byte(m.History[0].V1Compatibility), v1); err != nil {
return nil, err
}
return &types.ImageInspectInfo{
i := &types.ImageInspectInfo{
Tag: m.Tag,
DockerVersion: v1.DockerVersion,
Created: v1.Created,
Labels: v1.Config.Labels,
Architecture: v1.Architecture,
Os: v1.OS,
}, nil
}
if v1.Config != nil {
i.Labels = v1.Config.Labels
}
return i, nil
}
// UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs.

View File

@ -157,13 +157,16 @@ func (m *manifestSchema2) imageInspectInfo() (*types.ImageInspectInfo, error) {
if err := json.Unmarshal(config, v1); err != nil {
return nil, err
}
return &types.ImageInspectInfo{
i := &types.ImageInspectInfo{
DockerVersion: v1.DockerVersion,
Created: v1.Created,
Labels: v1.Config.Labels,
Architecture: v1.Architecture,
Os: v1.OS,
}, nil
}
if v1.Config != nil {
i.Labels = v1.Config.Labels
}
return i, nil
}
// UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs.

View File

@ -109,7 +109,7 @@ func (m *manifestOCI1) OCIConfig() (*imgspecv1.Image, error) {
func (m *manifestOCI1) LayerInfos() []types.BlobInfo {
blobs := []types.BlobInfo{}
for _, layer := range m.LayersDescriptors {
blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations, URLs: layer.URLs})
blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations, URLs: layer.URLs, MediaType: layer.MediaType})
}
return blobs
}
@ -130,13 +130,16 @@ func (m *manifestOCI1) imageInspectInfo() (*types.ImageInspectInfo, error) {
if err := json.Unmarshal(config, v1); err != nil {
return nil, err
}
return &types.ImageInspectInfo{
i := &types.ImageInspectInfo{
DockerVersion: v1.DockerVersion,
Created: v1.Created,
Labels: v1.Config.Labels,
Architecture: v1.Architecture,
Os: v1.OS,
}, nil
}
if v1.Config != nil {
i.Labels = v1.Config.Labels
}
return i, nil
}
// UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs.

View File

@ -106,7 +106,6 @@ func (d *ociArchiveImageDestination) Commit() error {
src := d.tempDirRef.tempDirectory
// path to save tarred up file
dst := d.ref.resolvedFile
return tarDirectory(src, dst)
}

View File

@ -18,21 +18,47 @@ import (
)
type ociImageDestination struct {
ref ociReference
index imgspecv1.Index
ref ociReference
index imgspecv1.Index
sharedBlobDir string
}
// newImageDestination returns an ImageDestination for writing to an existing directory.
func newImageDestination(ref ociReference) (types.ImageDestination, error) {
func newImageDestination(ctx *types.SystemContext, ref ociReference) (types.ImageDestination, error) {
if ref.image == "" {
return nil, errors.Errorf("cannot save image with empty image.ref.name")
}
index := imgspecv1.Index{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
var index *imgspecv1.Index
if indexExists(ref) {
var err error
index, err = ref.getIndex()
if err != nil {
return nil, err
}
} else {
index = &imgspecv1.Index{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
}
}
return &ociImageDestination{ref: ref, index: index}, nil
d := &ociImageDestination{ref: ref, index: *index}
if ctx != nil {
d.sharedBlobDir = ctx.OCISharedBlobDirPath
}
if err := ensureDirectoryExists(d.ref.dir); err != nil {
return nil, err
}
// Per the OCI image specification, layouts MUST have a "blobs" subdirectory,
// but it MAY be empty (e.g. if we never end up calling PutBlob)
// https://github.com/opencontainers/image-spec/blame/7c889fafd04a893f5c5f50b7ab9963d5d64e5242/image-layout.md#L19
if err := ensureDirectoryExists(filepath.Join(d.ref.dir, "blobs")); err != nil {
return nil, err
}
return d, nil
}
// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent,
@ -81,9 +107,6 @@ func (d *ociImageDestination) MustMatchRuntimeOS() bool {
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) {
if err := ensureDirectoryExists(d.ref.dir); err != nil {
return types.BlobInfo{}, err
}
blobFile, err := ioutil.TempFile(d.ref.dir, "oci-put-blob")
if err != nil {
return types.BlobInfo{}, err
@ -114,7 +137,7 @@ func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo
return types.BlobInfo{}, err
}
blobPath, err := d.ref.blobPath(computedDigest)
blobPath, err := d.ref.blobPath(computedDigest, d.sharedBlobDir)
if err != nil {
return types.BlobInfo{}, err
}
@ -136,7 +159,7 @@ func (d *ociImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error)
if info.Digest == "" {
return false, -1, errors.Errorf(`"Can not check for a blob with unknown digest`)
}
blobPath, err := d.ref.blobPath(info.Digest)
blobPath, err := d.ref.blobPath(info.Digest, d.sharedBlobDir)
if err != nil {
return false, -1, err
}
@ -169,7 +192,7 @@ func (d *ociImageDestination) PutManifest(m []byte) error {
desc.MediaType = imgspecv1.MediaTypeImageManifest
desc.Size = int64(len(m))
blobPath, err := d.ref.blobPath(digest)
blobPath, err := d.ref.blobPath(digest, d.sharedBlobDir)
if err != nil {
return err
}
@ -191,23 +214,20 @@ func (d *ociImageDestination) PutManifest(m []byte) error {
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
}
d.index.Manifests = append(d.index.Manifests, desc)
d.addManifest(&desc)
return nil
}
func ensureDirectoryExists(path string) error {
if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
if err := os.MkdirAll(path, 0755); err != nil {
return err
func (d *ociImageDestination) addManifest(desc *imgspecv1.Descriptor) {
for i, manifest := range d.index.Manifests {
if manifest.Annotations["org.opencontainers.image.ref.name"] == desc.Annotations["org.opencontainers.image.ref.name"] {
// TODO Should there first be a cleanup based on the descriptor we are going to replace?
d.index.Manifests[i] = *desc
return
}
}
return nil
}
// ensureParentDirectoryExists ensures the parent of the supplied path exists.
func ensureParentDirectoryExists(path string) error {
return ensureDirectoryExists(filepath.Dir(path))
d.index.Manifests = append(d.index.Manifests, *desc)
}
func (d *ociImageDestination) PutSignatures(signatures [][]byte) error {
@ -231,3 +251,30 @@ func (d *ociImageDestination) Commit() error {
}
return ioutil.WriteFile(d.ref.indexPath(), indexJSON, 0644)
}
func ensureDirectoryExists(path string) error {
if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
}
return nil
}
// ensureParentDirectoryExists ensures the parent of the supplied path exists.
func ensureParentDirectoryExists(path string) error {
return ensureDirectoryExists(filepath.Dir(path))
}
// indexExists checks whether the index location specified in the OCI reference exists.
// The implementation is opinionated, since in case of unexpected errors false is returned
func indexExists(ref ociReference) bool {
_, err := os.Stat(ref.indexPath())
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
return true
}

View File

@ -17,9 +17,10 @@ import (
)
type ociImageSource struct {
ref ociReference
descriptor imgspecv1.Descriptor
client *http.Client
ref ociReference
descriptor imgspecv1.Descriptor
client *http.Client
sharedBlobDir string
}
// newImageSource returns an ImageSource for reading from an existing directory.
@ -40,7 +41,12 @@ func newImageSource(ctx *types.SystemContext, ref ociReference) (types.ImageSour
if err != nil {
return nil, err
}
return &ociImageSource{ref: ref, descriptor: descriptor, client: client}, nil
d := &ociImageSource{ref: ref, descriptor: descriptor, client: client}
if ctx != nil {
// TODO(jonboulle): check dir existence?
d.sharedBlobDir = ctx.OCISharedBlobDirPath
}
return d, nil
}
// Reference returns the reference used to set up this source.
@ -56,7 +62,7 @@ func (s *ociImageSource) Close() error {
// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
func (s *ociImageSource) GetManifest() ([]byte, string, error) {
manifestPath, err := s.ref.blobPath(digest.Digest(s.descriptor.Digest))
manifestPath, err := s.ref.blobPath(digest.Digest(s.descriptor.Digest), s.sharedBlobDir)
if err != nil {
return nil, "", err
}
@ -69,7 +75,7 @@ func (s *ociImageSource) GetManifest() ([]byte, string, error) {
}
func (s *ociImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
manifestPath, err := s.ref.blobPath(digest)
manifestPath, err := s.ref.blobPath(digest, s.sharedBlobDir)
if err != nil {
return nil, "", err
}
@ -92,7 +98,7 @@ func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err
return s.getExternalBlob(info.URLs)
}
path, err := s.ref.blobPath(info.Digest)
path, err := s.ref.blobPath(info.Digest, s.sharedBlobDir)
if err != nil {
return nil, 0, err
}

View File

@ -189,14 +189,25 @@ func (ref ociReference) NewImage(ctx *types.SystemContext) (types.Image, error)
return image.FromSource(src)
}
func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) {
// getIndex returns a pointer to the index references by this ociReference. If an error occurs opening an index nil is returned together
// with an error.
func (ref ociReference) getIndex() (*imgspecv1.Index, error) {
indexJSON, err := os.Open(ref.indexPath())
if err != nil {
return imgspecv1.Descriptor{}, err
return nil, err
}
defer indexJSON.Close()
index := imgspecv1.Index{}
if err := json.NewDecoder(indexJSON).Decode(&index); err != nil {
index := &imgspecv1.Index{}
if err := json.NewDecoder(indexJSON).Decode(index); err != nil {
return nil, err
}
return index, nil
}
func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) {
index, err := ref.getIndex()
if err != nil {
return imgspecv1.Descriptor{}, err
}
@ -250,7 +261,7 @@ func (ref ociReference) NewImageSource(ctx *types.SystemContext) (types.ImageSou
// NewImageDestination returns a types.ImageDestination for this reference.
// The caller must call .Close() on the returned ImageDestination.
func (ref ociReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) {
return newImageDestination(ref)
return newImageDestination(ctx, ref)
}
// DeleteImage deletes the named image from the registry, if supported.
@ -269,9 +280,13 @@ func (ref ociReference) indexPath() string {
}
// blobPath returns a path for a blob within a directory using OCI image-layout conventions.
func (ref ociReference) blobPath(digest digest.Digest) (string, error) {
func (ref ociReference) blobPath(digest digest.Digest, sharedBlobDir string) (string, error) {
if err := digest.Validate(); err != nil {
return "", errors.Wrapf(err, "unexpected digest reference %s", digest)
}
return filepath.Join(ref.dir, "blobs", digest.Algorithm().String(), digest.Hex()), nil
blobDir := filepath.Join(ref.dir, "blobs")
if sharedBlobDir != "" {
blobDir = sharedBlobDir
}
return filepath.Join(blobDir, digest.Algorithm().String(), digest.Hex()), nil
}

View File

@ -0,0 +1,295 @@
package config
import (
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/containers/image/types"
helperclient "github.com/docker/docker-credential-helpers/client"
"github.com/docker/docker-credential-helpers/credentials"
"github.com/docker/docker/pkg/homedir"
"github.com/pkg/errors"
)
type dockerAuthConfig struct {
Auth string `json:"auth,omitempty"`
}
type dockerConfigFile struct {
AuthConfigs map[string]dockerAuthConfig `json:"auths"`
CredHelpers map[string]string `json:"credHelpers,omitempty"`
}
const (
defaultPath = "/run/user"
authCfg = "containers"
authCfgFileName = "auth.json"
dockerCfg = ".docker"
dockerCfgFileName = "config.json"
dockerLegacyCfg = ".dockercfg"
)
var (
// ErrNotLoggedIn is returned for users not logged into a registry
// that they are trying to logout of
ErrNotLoggedIn = errors.New("not logged in")
)
// SetAuthentication stores the username and password in the auth.json file
func SetAuthentication(ctx *types.SystemContext, registry, username, password string) error {
return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) {
if ch, exists := auths.CredHelpers[registry]; exists {
return false, setAuthToCredHelper(ch, registry, username, password)
}
creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
newCreds := dockerAuthConfig{Auth: creds}
auths.AuthConfigs[registry] = newCreds
return true, nil
})
}
// GetAuthentication returns the registry credentials stored in
// either auth.json file or .docker/config.json
// If an entry is not found empty strings are returned for the username and password
func GetAuthentication(ctx *types.SystemContext, registry string) (string, string, error) {
if ctx != nil && ctx.DockerAuthConfig != nil {
return ctx.DockerAuthConfig.Username, ctx.DockerAuthConfig.Password, nil
}
dockerLegacyPath := filepath.Join(homedir.Get(), dockerLegacyCfg)
paths := [3]string{getPathToAuth(ctx), filepath.Join(homedir.Get(), dockerCfg, dockerCfgFileName), dockerLegacyPath}
for _, path := range paths {
legacyFormat := path == dockerLegacyPath
username, password, err := findAuthentication(registry, path, legacyFormat)
if err != nil {
return "", "", err
}
if username != "" && password != "" {
return username, password, nil
}
}
return "", "", nil
}
// GetUserLoggedIn returns the username logged in to registry from either
// auth.json or XDG_RUNTIME_DIR
// Used to tell the user if someone is logged in to the registry when logging in
func GetUserLoggedIn(ctx *types.SystemContext, registry string) string {
path := getPathToAuth(ctx)
username, _, _ := findAuthentication(registry, path, false)
if username != "" {
return username
}
return ""
}
// RemoveAuthentication deletes the credentials stored in auth.json
func RemoveAuthentication(ctx *types.SystemContext, registry string) error {
return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) {
// First try cred helpers.
if ch, exists := auths.CredHelpers[registry]; exists {
return false, deleteAuthFromCredHelper(ch, registry)
}
if _, ok := auths.AuthConfigs[registry]; ok {
delete(auths.AuthConfigs, registry)
} else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok {
delete(auths.AuthConfigs, normalizeRegistry(registry))
} else {
return false, ErrNotLoggedIn
}
return true, nil
})
}
// RemoveAllAuthentication deletes all the credentials stored in auth.json
func RemoveAllAuthentication(ctx *types.SystemContext) error {
return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) {
auths.CredHelpers = make(map[string]string)
auths.AuthConfigs = make(map[string]dockerAuthConfig)
return true, nil
})
}
// getPath gets the path of the auth.json file
// The path can be overriden by the user if the overwrite-path flag is set
// If the flag is not set and XDG_RUNTIME_DIR is ser, the auth.json file is saved in XDG_RUNTIME_DIR/containers
// Otherwise, the auth.json file is stored in /run/user/UID/containers
func getPathToAuth(ctx *types.SystemContext) string {
if ctx != nil {
if ctx.AuthFilePath != "" {
return ctx.AuthFilePath
}
if ctx.RootForImplicitAbsolutePaths != "" {
return filepath.Join(ctx.RootForImplicitAbsolutePaths, defaultPath, strconv.Itoa(os.Getuid()), authCfg, authCfgFileName)
}
}
runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
if runtimeDir == "" {
runtimeDir = filepath.Join(defaultPath, strconv.Itoa(os.Getuid()))
}
return filepath.Join(runtimeDir, authCfg, authCfgFileName)
}
// readJSONFile unmarshals the authentications stored in the auth.json file and returns it
// or returns an empty dockerConfigFile data structure if auth.json does not exist
// if the file exists and is empty, readJSONFile returns an error
func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) {
var auths dockerConfigFile
raw, err := ioutil.ReadFile(path)
if os.IsNotExist(err) {
auths.AuthConfigs = map[string]dockerAuthConfig{}
return auths, nil
}
if legacyFormat {
if err = json.Unmarshal(raw, &auths.AuthConfigs); err != nil {
return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
}
return auths, nil
}
if err = json.Unmarshal(raw, &auths); err != nil {
return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
}
return auths, nil
}
// modifyJSON writes to auth.json if the dockerConfigFile has been updated
func modifyJSON(ctx *types.SystemContext, editor func(auths *dockerConfigFile) (bool, error)) error {
path := getPathToAuth(ctx)
dir := filepath.Dir(path)
if _, err := os.Stat(dir); os.IsNotExist(err) {
if err = os.Mkdir(dir, 0700); err != nil {
return errors.Wrapf(err, "error creating directory %q", dir)
}
}
auths, err := readJSONFile(path, false)
if err != nil {
return errors.Wrapf(err, "error reading JSON file %q", path)
}
updated, err := editor(&auths)
if err != nil {
return errors.Wrapf(err, "error updating %q", path)
}
if updated {
newData, err := json.MarshalIndent(auths, "", "\t")
if err != nil {
return errors.Wrapf(err, "error marshaling JSON %q", path)
}
if err = ioutil.WriteFile(path, newData, 0755); err != nil {
return errors.Wrapf(err, "error writing to file %q", path)
}
}
return nil
}
func getAuthFromCredHelper(credHelper, registry string) (string, string, error) {
helperName := fmt.Sprintf("docker-credential-%s", credHelper)
p := helperclient.NewShellProgramFunc(helperName)
creds, err := helperclient.Get(p, registry)
if err != nil {
return "", "", err
}
return creds.Username, creds.Secret, nil
}
func setAuthToCredHelper(credHelper, registry, username, password string) error {
helperName := fmt.Sprintf("docker-credential-%s", credHelper)
p := helperclient.NewShellProgramFunc(helperName)
creds := &credentials.Credentials{
ServerURL: registry,
Username: username,
Secret: password,
}
return helperclient.Store(p, creds)
}
func deleteAuthFromCredHelper(credHelper, registry string) error {
helperName := fmt.Sprintf("docker-credential-%s", credHelper)
p := helperclient.NewShellProgramFunc(helperName)
return helperclient.Erase(p, registry)
}
// findAuthentication looks for auth of registry in path
func findAuthentication(registry, path string, legacyFormat bool) (string, string, error) {
auths, err := readJSONFile(path, legacyFormat)
if err != nil {
return "", "", errors.Wrapf(err, "error reading JSON file %q", path)
}
// First try cred helpers. They should always be normalized.
if ch, exists := auths.CredHelpers[registry]; exists {
return getAuthFromCredHelper(ch, registry)
}
// I'm feeling lucky
if val, exists := auths.AuthConfigs[registry]; exists {
return decodeDockerAuth(val.Auth)
}
// bad luck; let's normalize the entries first
registry = normalizeRegistry(registry)
normalizedAuths := map[string]dockerAuthConfig{}
for k, v := range auths.AuthConfigs {
normalizedAuths[normalizeRegistry(k)] = v
}
if val, exists := normalizedAuths[registry]; exists {
return decodeDockerAuth(val.Auth)
}
return "", "", nil
}
func decodeDockerAuth(s string) (string, string, error) {
decoded, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return "", "", err
}
parts := strings.SplitN(string(decoded), ":", 2)
if len(parts) != 2 {
// if it's invalid just skip, as docker does
return "", "", nil
}
user := parts[0]
password := strings.Trim(parts[1], "\x00")
return user, password, nil
}
// convertToHostname converts a registry url which has http|https prepended
// to just an hostname.
// Copied from github.com/docker/docker/registry/auth.go
func convertToHostname(url string) string {
stripped := url
if strings.HasPrefix(url, "http://") {
stripped = strings.TrimPrefix(url, "http://")
} else if strings.HasPrefix(url, "https://") {
stripped = strings.TrimPrefix(url, "https://")
}
nameParts := strings.SplitN(stripped, "/", 2)
return nameParts[0]
}
func normalizeRegistry(registry string) string {
normalized := convertToHostname(registry)
switch normalized {
case "registry-1.docker.io", "docker.io":
return "index.docker.io"
}
return normalized
}

View File

@ -70,7 +70,11 @@ func NewPolicyFromFile(fileName string) (*Policy, error) {
if err != nil {
return nil, err
}
return NewPolicyFromBytes(contents)
policy, err := NewPolicyFromBytes(contents)
if err != nil {
return nil, errors.Wrapf(err, "invalid policy in %q", fileName)
}
return policy, nil
}
// NewPolicyFromBytes returns a policy parsed from the specified blob.

View File

@ -1,3 +1,5 @@
// +build !containers_image_storage_stub
package storage
import (

View File

@ -1,3 +1,5 @@
// +build !containers_image_storage_stub
package storage
import (

View File

@ -1,3 +1,5 @@
// +build !containers_image_storage_stub
package storage
import (

48
vendor/github.com/containers/image/tarball/doc.go generated vendored Normal file
View File

@ -0,0 +1,48 @@
// Package tarball provides a way to generate images using one or more layer
// tarballs and an optional template configuration.
//
// An example:
// package main
//
// import (
// "fmt"
//
// cp "github.com/containers/image/copy"
// "github.com/containers/image/tarball"
// "github.com/containers/image/transports/alltransports"
//
// imgspecv1 "github.com/containers/image/transports/alltransports"
// )
//
// func imageFromTarball() {
// src, err := alltransports.ParseImageName("tarball:/var/cache/mock/fedora-26-x86_64/root_cache/cache.tar.gz")
// // - or -
// // src, err := tarball.Transport.ParseReference("/var/cache/mock/fedora-26-x86_64/root_cache/cache.tar.gz")
// if err != nil {
// panic(err)
// }
// updater, ok := src.(tarball.ConfigUpdater)
// if !ok {
// panic("unexpected: a tarball reference should implement tarball.ConfigUpdater")
// }
// config := imgspecv1.Image{
// Config: imgspecv1.ImageConfig{
// Cmd: []string{"/bin/bash"},
// },
// }
// annotations := make(map[string]string)
// annotations[imgspecv1.AnnotationDescription] = "test image built from a mock root cache"
// err = updater.ConfigUpdate(config, annotations)
// if err != nil {
// panic(err)
// }
// dest, err := alltransports.ParseImageName("docker-daemon:mock:latest")
// if err != nil {
// panic(err)
// }
// err = cp.Image(nil, dest, src, nil)
// if err != nil {
// panic(err)
// }
// }
package tarball

View File

@ -0,0 +1,88 @@
package tarball
import (
"fmt"
"os"
"strings"
"github.com/containers/image/docker/reference"
"github.com/containers/image/image"
"github.com/containers/image/types"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
)
// ConfigUpdater is an interface that ImageReferences for "tarball" images also
// implement. It can be used to set values for a configuration, and to set
// image annotations which will be present in the images returned by the
// reference's NewImage() or NewImageSource() methods.
type ConfigUpdater interface {
ConfigUpdate(config imgspecv1.Image, annotations map[string]string) error
}
type tarballReference struct {
transport types.ImageTransport
config imgspecv1.Image
annotations map[string]string
filenames []string
stdin []byte
}
// ConfigUpdate updates the image's default configuration and adds annotations
// which will be visible in source images created using this reference.
func (r *tarballReference) ConfigUpdate(config imgspecv1.Image, annotations map[string]string) error {
r.config = config
if r.annotations == nil {
r.annotations = make(map[string]string)
}
for k, v := range annotations {
r.annotations[k] = v
}
return nil
}
func (r *tarballReference) Transport() types.ImageTransport {
return r.transport
}
func (r *tarballReference) StringWithinTransport() string {
return strings.Join(r.filenames, ":")
}
func (r *tarballReference) DockerReference() reference.Named {
return nil
}
func (r *tarballReference) PolicyConfigurationIdentity() string {
return ""
}
func (r *tarballReference) PolicyConfigurationNamespaces() []string {
return nil
}
func (r *tarballReference) NewImage(ctx *types.SystemContext) (types.Image, error) {
src, err := r.NewImageSource(ctx)
if err != nil {
return nil, err
}
img, err := image.FromSource(src)
if err != nil {
src.Close()
return nil, err
}
return img, nil
}
func (r *tarballReference) DeleteImage(ctx *types.SystemContext) error {
for _, filename := range r.filenames {
if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("error removing %q: %v", filename, err)
}
}
return nil
}
func (r *tarballReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) {
return nil, fmt.Errorf("destination not implemented yet")
}

View File

@ -0,0 +1,250 @@
package tarball
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"strings"
"time"
"github.com/containers/image/types"
digest "github.com/opencontainers/go-digest"
imgspecs "github.com/opencontainers/image-spec/specs-go"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
)
type tarballImageSource struct {
reference tarballReference
filenames []string
diffIDs []digest.Digest
diffSizes []int64
blobIDs []digest.Digest
blobSizes []int64
blobTypes []string
config []byte
configID digest.Digest
configSize int64
manifest []byte
}
func (r *tarballReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
// Gather up the digests, sizes, and date information for all of the files.
filenames := []string{}
diffIDs := []digest.Digest{}
diffSizes := []int64{}
blobIDs := []digest.Digest{}
blobSizes := []int64{}
blobTimes := []time.Time{}
blobTypes := []string{}
for _, filename := range r.filenames {
var file *os.File
var err error
var blobSize int64
var blobTime time.Time
var reader io.Reader
if filename == "-" {
blobSize = int64(len(r.stdin))
blobTime = time.Now()
reader = bytes.NewReader(r.stdin)
} else {
file, err = os.Open(filename)
if err != nil {
return nil, fmt.Errorf("error opening %q for reading: %v", filename, err)
}
defer file.Close()
reader = file
fileinfo, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("error reading size of %q: %v", filename, err)
}
blobSize = fileinfo.Size()
blobTime = fileinfo.ModTime()
}
// Default to assuming the layer is compressed.
layerType := imgspecv1.MediaTypeImageLayerGzip
// Set up to digest the file as it is.
blobIDdigester := digest.Canonical.Digester()
reader = io.TeeReader(reader, blobIDdigester.Hash())
// Set up to digest the file after we maybe decompress it.
diffIDdigester := digest.Canonical.Digester()
uncompressed, err := gzip.NewReader(reader)
if err == nil {
// It is compressed, so the diffID is the digest of the uncompressed version
reader = io.TeeReader(uncompressed, diffIDdigester.Hash())
} else {
// It is not compressed, so the diffID and the blobID are going to be the same
diffIDdigester = blobIDdigester
layerType = imgspecv1.MediaTypeImageLayer
uncompressed = nil
}
n, err := io.Copy(ioutil.Discard, reader)
if err != nil {
return nil, fmt.Errorf("error reading %q: %v", filename, err)
}
if uncompressed != nil {
uncompressed.Close()
}
// Grab our uncompressed and possibly-compressed digests and sizes.
filenames = append(filenames, filename)
diffIDs = append(diffIDs, diffIDdigester.Digest())
diffSizes = append(diffSizes, n)
blobIDs = append(blobIDs, blobIDdigester.Digest())
blobSizes = append(blobSizes, blobSize)
blobTimes = append(blobTimes, blobTime)
blobTypes = append(blobTypes, layerType)
}
// Build the rootfs and history for the configuration blob.
rootfs := imgspecv1.RootFS{
Type: "layers",
DiffIDs: diffIDs,
}
created := time.Time{}
history := []imgspecv1.History{}
// Pick up the layer comment from the configuration's history list, if one is set.
comment := "imported from tarball"
if len(r.config.History) > 0 && r.config.History[0].Comment != "" {
comment = r.config.History[0].Comment
}
for i := range diffIDs {
createdBy := fmt.Sprintf("/bin/sh -c #(nop) ADD file:%s in %c", diffIDs[i].Hex(), os.PathSeparator)
history = append(history, imgspecv1.History{
Created: &blobTimes[i],
CreatedBy: createdBy,
Comment: comment,
})
// Use the mtime of the most recently modified file as the image's creation time.
if created.Before(blobTimes[i]) {
created = blobTimes[i]
}
}
// Pick up other defaults from the config in the reference.
config := r.config
if config.Created == nil {
config.Created = &created
}
if config.Architecture == "" {
config.Architecture = runtime.GOARCH
}
if config.OS == "" {
config.OS = runtime.GOOS
}
config.RootFS = rootfs
config.History = history
// Encode and digest the image configuration blob.
configBytes, err := json.Marshal(&config)
if err != nil {
return nil, fmt.Errorf("error generating configuration blob for %q: %v", strings.Join(r.filenames, separator), err)
}
configID := digest.Canonical.FromBytes(configBytes)
configSize := int64(len(configBytes))
// Populate a manifest with the configuration blob and the file as the single layer.
layerDescriptors := []imgspecv1.Descriptor{}
for i := range blobIDs {
layerDescriptors = append(layerDescriptors, imgspecv1.Descriptor{
Digest: blobIDs[i],
Size: blobSizes[i],
MediaType: blobTypes[i],
})
}
annotations := make(map[string]string)
for k, v := range r.annotations {
annotations[k] = v
}
manifest := imgspecv1.Manifest{
Versioned: imgspecs.Versioned{
SchemaVersion: 2,
},
Config: imgspecv1.Descriptor{
Digest: configID,
Size: configSize,
MediaType: imgspecv1.MediaTypeImageConfig,
},
Layers: layerDescriptors,
Annotations: annotations,
}
// Encode the manifest.
manifestBytes, err := json.Marshal(&manifest)
if err != nil {
return nil, fmt.Errorf("error generating manifest for %q: %v", strings.Join(r.filenames, separator), err)
}
// Return the image.
src := &tarballImageSource{
reference: *r,
filenames: filenames,
diffIDs: diffIDs,
diffSizes: diffSizes,
blobIDs: blobIDs,
blobSizes: blobSizes,
blobTypes: blobTypes,
config: configBytes,
configID: configID,
configSize: configSize,
manifest: manifestBytes,
}
return src, nil
}
func (is *tarballImageSource) Close() error {
return nil
}
func (is *tarballImageSource) GetBlob(blobinfo types.BlobInfo) (io.ReadCloser, int64, error) {
// We should only be asked about things in the manifest. Maybe the configuration blob.
if blobinfo.Digest == is.configID {
return ioutil.NopCloser(bytes.NewBuffer(is.config)), is.configSize, nil
}
// Maybe one of the layer blobs.
for i := range is.blobIDs {
if blobinfo.Digest == is.blobIDs[i] {
// We want to read that layer: open the file or memory block and hand it back.
if is.filenames[i] == "-" {
return ioutil.NopCloser(bytes.NewBuffer(is.reference.stdin)), int64(len(is.reference.stdin)), nil
}
reader, err := os.Open(is.filenames[i])
if err != nil {
return nil, -1, fmt.Errorf("error opening %q: %v", is.filenames[i], err)
}
return reader, is.blobSizes[i], nil
}
}
return nil, -1, fmt.Errorf("no blob with digest %q found", blobinfo.Digest.String())
}
func (is *tarballImageSource) GetManifest() ([]byte, string, error) {
return is.manifest, imgspecv1.MediaTypeImageManifest, nil
}
func (*tarballImageSource) GetSignatures(context.Context) ([][]byte, error) {
return nil, nil
}
func (*tarballImageSource) GetTargetManifest(digest.Digest) ([]byte, string, error) {
return nil, "", fmt.Errorf("manifest lists are not supported by the %q transport", transportName)
}
func (is *tarballImageSource) Reference() types.ImageReference {
return &is.reference
}
// UpdatedLayerInfos() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
func (*tarballImageSource) UpdatedLayerInfos() []types.BlobInfo {
return nil
}

View File

@ -0,0 +1,66 @@
package tarball
import (
"errors"
"fmt"
"io/ioutil"
"os"
"strings"
"github.com/containers/image/transports"
"github.com/containers/image/types"
)
const (
transportName = "tarball"
separator = ":"
)
var (
// Transport implements the types.ImageTransport interface for "tarball:" images,
// which are makeshift images constructed using one or more possibly-compressed tar
// archives.
Transport = &tarballTransport{}
)
type tarballTransport struct {
}
func (t *tarballTransport) Name() string {
return transportName
}
func (t *tarballTransport) ParseReference(reference string) (types.ImageReference, error) {
var stdin []byte
var err error
filenames := strings.Split(reference, separator)
for _, filename := range filenames {
if filename == "-" {
stdin, err = ioutil.ReadAll(os.Stdin)
if err != nil {
return nil, fmt.Errorf("error buffering stdin: %v", err)
}
continue
}
f, err := os.Open(filename)
if err != nil {
return nil, fmt.Errorf("error opening %q: %v", filename, err)
}
f.Close()
}
ref := &tarballReference{
transport: t,
filenames: filenames,
stdin: stdin,
}
return ref, nil
}
func (t *tarballTransport) ValidatePolicyConfigurationScope(scope string) error {
// See the explanation in daemonReference.PolicyConfigurationIdentity.
return errors.New(`tarball: does not support any scopes except the default "" one`)
}
func init() {
transports.Register(Transport)
}

View File

@ -13,8 +13,9 @@ import (
_ "github.com/containers/image/oci/archive"
_ "github.com/containers/image/oci/layout"
_ "github.com/containers/image/openshift"
_ "github.com/containers/image/tarball"
// The ostree transport is registered by ostree*.go
_ "github.com/containers/image/storage"
// The storage transport is registered by storage*.go
"github.com/containers/image/transports"
"github.com/containers/image/types"
"github.com/pkg/errors"

View File

@ -0,0 +1,8 @@
// +build !containers_image_storage_stub
package alltransports
import (
// Register the storage transport
_ "github.com/containers/image/storage"
)

View File

@ -0,0 +1,9 @@
// +build containers_image_storage_stub
package alltransports
import "github.com/containers/image/transports"
func init() {
transports.Register(transports.NewStubTransport("containers-storage"))
}

View File

@ -96,6 +96,7 @@ type BlobInfo struct {
Size int64 // -1 if unknown
URLs []string
Annotations map[string]string
MediaType string
}
// ImageSource is a service, possibly remote (= slow), to download components of a single image.
@ -118,7 +119,7 @@ type ImageSource interface {
// out of a manifest list.
GetTargetManifest(digest digest.Digest) ([]byte, string, error)
// GetBlob returns a stream for the specified blob, and the blobs size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided; Size may be -1.
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
GetBlob(BlobInfo) (io.ReadCloser, int64, error)
// GetSignatures returns the image's signatures. It may use a remote (= slow) service.
GetSignatures(context.Context) ([][]byte, error)
@ -153,9 +154,10 @@ type ImageDestination interface {
AcceptsForeignLayerURLs() bool
// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise.
MustMatchRuntimeOS() bool
// PutBlob writes contents of stream and returns data representing the result (with all data filled in).
// PutBlob writes contents of stream and returns data representing the result.
// inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it.
// inputInfo.Size is the expected length of stream, if known.
// inputInfo.MediaType describes the blob format, if known.
// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
@ -215,7 +217,7 @@ type Image interface {
// ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object.
// Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below.
ConfigInfo() BlobInfo
// ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise.
// ConfigBlob returns the blob described by ConfigInfo, if ConfigInfo().Digest != ""; nil otherwise.
// The result is cached; it is OK to call this however often you need.
ConfigBlob() ([]byte, error)
// OCIConfig returns the image configuration as per OCI v1 image-spec. Information about
@ -223,7 +225,7 @@ type Image interface {
// old image manifests work (docker v2s1 especially).
OCIConfig() (*v1.Image, error)
// LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers).
// The Digest field is guaranteed to be provided; Size may be -1.
// The Digest field is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// WARNING: The list may contain duplicates, and they are semantically relevant.
LayerInfos() []BlobInfo
// EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref.
@ -249,7 +251,7 @@ type Image interface {
// ManifestUpdateOptions is a way to pass named optional arguments to Image.UpdatedManifest
type ManifestUpdateOptions struct {
LayerInfos []BlobInfo // Complete BlobInfos (size+digest+urls) which should replace the originals, in order (the root layer first, and then successive layered layers)
LayerInfos []BlobInfo // Complete BlobInfos (size+digest+urls+annotations) which should replace the originals, in order (the root layer first, and then successive layered layers). BlobInfos' MediaType fields are ignored.
EmbeddedDockerReference reference.Named
ManifestMIMEType string
// The values below are NOT requests to modify the image; they provide optional context which may or may not be used.
@ -283,7 +285,7 @@ type DockerAuthConfig struct {
Password string
}
// SystemContext allows parametrizing access to implicitly-accessed resources,
// SystemContext allows parameterizing access to implicitly-accessed resources,
// like configuration files in /etc and users' login state in their home directory.
// Various components can share the same field only if their semantics is exactly
// the same; if in doubt, add a new field.
@ -304,6 +306,8 @@ type SystemContext struct {
RegistriesDirPath string
// Path to the system-wide registries configuration file
SystemRegistriesConfPath string
// If not "", overrides the default path for the authentication file
AuthFilePath string
// === OCI.Transport overrides ===
// If not "", a directory containing a CA certificate (ending with ".crt"),
@ -312,6 +316,8 @@ type SystemContext struct {
OCICertPath string
// Allow downloading OCI image layers over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections.
OCIInsecureSkipTLSVerify bool
// If not "", use a shared directory for storing blobs rather than within OCI layouts
OCISharedBlobDirPath string
// === docker.Transport overrides ===
// If not "", a directory containing a CA certificate (ending with ".crt"),
@ -320,8 +326,9 @@ type SystemContext struct {
DockerCertPath string
// If not "", overrides the systems default path for a directory containing host[:port] subdirectories with the same structure as DockerCertPath above.
// Ignored if DockerCertPath is non-empty.
DockerPerHostCertDirPath string
DockerInsecureSkipTLSVerify bool // Allow contacting docker registries over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections.
DockerPerHostCertDirPath string
// Allow contacting docker registries over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections.
DockerInsecureSkipTLSVerify bool
// if nil, the library tries to parse ~/.docker/config.json to retrieve credentials
DockerAuthConfig *DockerAuthConfig
// if not "", an User-Agent header is added to each request when contacting a registry.
@ -332,6 +339,20 @@ type SystemContext struct {
DockerDisableV1Ping bool
// Directory to use for OSTree temporary files
OSTreeTmpDirPath string
// === docker/daemon.Transport overrides ===
// A directory containing a CA certificate (ending with ".crt"),
// a client certificate (ending with ".cert") and a client certificate key
// (ending with ".key") used when talking to a Docker daemon.
DockerDaemonCertPath string
// The hostname or IP to the Docker daemon. If not set (aka ""), client.DefaultDockerHost is assumed.
DockerDaemonHost string
// Used to skip TLS verification, off by default. To take effect DockerDaemonCertPath needs to be specified as well.
DockerDaemonInsecureSkipTLSVerify bool
// === dir.Transport overrides ===
// DirForceCompress compresses the image layers if set to true
DirForceCompress bool
}
// ProgressProperties is used to pass information from the copy code to a monitor which

View File

@ -22,7 +22,7 @@ github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9
github.com/pkg/errors 248dadf4e9068a0b3e79f02ed0a610d935de5302
github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2
github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987
github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721
github.com/vbatts/tar-split v0.10.2
golang.org/x/crypto 453249f01cfeb54c3d549ddb75ff152ca243f9d8
golang.org/x/net 6b27048ae5e6ad1ef927e72e437531493de612fe
golang.org/x/sys 43e60d72a8e2bd92ee98319ba9a384a0e9837c08

View File

@ -0,0 +1,23 @@
// +build linux
package homedir
import (
"os"
"github.com/docker/docker/pkg/idtools"
)
// GetStatic returns the home directory for the current user without calling
// os/user.Current(). This is useful for static-linked binary on glibc-based
// system, because a call to os/user.Current() in a static binary leads to
// segfault due to a glibc issue that won't be fixed in a short term.
// (#29344, golang/go#13470, https://sourceware.org/bugzilla/show_bug.cgi?id=19341)
func GetStatic() (string, error) {
uid := os.Getuid()
usr, err := idtools.LookupUID(uid)
if err != nil {
return "", err
}
return usr.Home, nil
}

View File

@ -0,0 +1,13 @@
// +build !linux
package homedir
import (
"errors"
)
// GetStatic is not needed for non-linux systems.
// (Precisely, it is needed only for glibc-based linux systems.)
func GetStatic() (string, error) {
return "", errors.New("homedir.GetStatic() is not supported on this system")
}

View File

@ -0,0 +1,34 @@
// +build !windows
package homedir
import (
"os"
"github.com/opencontainers/runc/libcontainer/user"
)
// Key returns the env var name for the user's home dir based on
// the platform being run on
func Key() string {
return "HOME"
}
// Get returns the home directory of the current user with the help of
// environment variables depending on the target operating system.
// Returned path should be used with "path/filepath" to form new paths.
func Get() string {
home := os.Getenv(Key())
if home == "" {
if u, err := user.CurrentUser(); err == nil {
return u.Home
}
}
return home
}
// GetShortcutString returns the string that is shortcut to user's home directory
// in the native shell of the platform running on.
func GetShortcutString() string {
return "~"
}

View File

@ -0,0 +1,24 @@
package homedir
import (
"os"
)
// Key returns the env var name for the user's home dir based on
// the platform being run on
func Key() string {
return "USERPROFILE"
}
// Get returns the home directory of the current user with the help of
// environment variables depending on the target operating system.
// Returned path should be used with "path/filepath" to form new paths.
func Get() string {
return os.Getenv(Key())
}
// GetShortcutString returns the string that is shortcut to user's home directory
// in the native shell of the platform running on.
func GetShortcutString() string {
return "%USERPROFILE%" // be careful while using in format functions
}