Update containers/image to v2.0.0

This adds v2 registries.conf format, including mirror support.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>

Closes: #1634
Approved by: rhatdan
This commit is contained in:
Miloslav Trmač 2019-06-03 15:14:32 +02:00 committed by Atomic Bot
parent dc7b50c9da
commit 765c09d6db
11 changed files with 261 additions and 198 deletions

View File

@ -106,13 +106,19 @@ func ResolveName(name string, firstRegistry string, sc *types.SystemContext, sto
// Figure out the list of registries. // Figure out the list of registries.
var registries []string var registries []string
searchRegistries, err := sysregistriesv2.FindUnqualifiedSearchRegistries(sc) searchRegistries, err := sysregistriesv2.UnqualifiedSearchRegistries(sc)
if err != nil { if err != nil {
logrus.Debugf("unable to read configured registries to complete %q: %v", name, err) logrus.Debugf("unable to read configured registries to complete %q: %v", name, err)
searchRegistries = nil
} }
for _, registry := range searchRegistries { for _, registry := range searchRegistries {
if !registry.Blocked { reg, err := sysregistriesv2.FindRegistry(sc, registry)
registries = append(registries, registry.Location) if err != nil {
logrus.Debugf("unable to read registry configuraitno for %#v: %v", registry, err)
continue
}
if reg == nil || !reg.Blocked {
registries = append(registries, registry)
} }
} }
searchRegistriesAreEmpty := len(registries) == 0 searchRegistriesAreEmpty := len(registries) == 0

View File

@ -3,7 +3,7 @@ github.com/blang/semver v3.5.0
github.com/BurntSushi/toml v0.2.0 github.com/BurntSushi/toml v0.2.0
github.com/containerd/continuity 004b46473808b3e7a4a3049c20e4376c91eb966d github.com/containerd/continuity 004b46473808b3e7a4a3049c20e4376c91eb966d
github.com/containernetworking/cni v0.7.0-rc2 github.com/containernetworking/cni v0.7.0-rc2
github.com/containers/image 9467ac9cfd92c545aa389f22f27e552de053c0f2 github.com/containers/image v2.0.0
github.com/cyphar/filepath-securejoin v0.2.1 github.com/cyphar/filepath-securejoin v0.2.1
github.com/vbauerster/mpb v3.3.4 github.com/vbauerster/mpb v3.3.4
github.com/mattn/go-isatty v0.0.4 github.com/mattn/go-isatty v0.0.4

View File

@ -23,7 +23,7 @@ import (
"github.com/containers/image/types" "github.com/containers/image/types"
"github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client"
"github.com/docker/go-connections/tlsconfig" "github.com/docker/go-connections/tlsconfig"
"github.com/opencontainers/go-digest" digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -84,27 +84,27 @@ type dockerClient struct {
sys *types.SystemContext sys *types.SystemContext
registry string registry string
// tlsClientConfig is setup by newDockerClient and will be used and updated
// by detectProperties(). Callers can edit tlsClientConfig.InsecureSkipVerify in the meantime.
tlsClientConfig *tls.Config
// The following members are not set by newDockerClient and must be set by callers if needed. // The following members are not set by newDockerClient and must be set by callers if needed.
username string username string
password string password string
signatureBase signatureStorageBase signatureBase signatureStorageBase
scope authScope scope authScope
// The following members are detected registry properties: // The following members are detected registry properties:
// They are set after a successful detectProperties(), and never change afterwards. // They are set after a successful detectProperties(), and never change afterwards.
scheme string // Empty value also used to indicate detectProperties() has not yet succeeded. client *http.Client
scheme string
challenges []challenge challenges []challenge
supportsSignatures bool supportsSignatures bool
// The tlsClientConfig is setup during the creation of the dockerClient and
// will be updated by detectPropertiesHelper(). Any HTTP request the
// dockerClient does will be done by this TLS client configuration.
tlsClientConfig *tls.Config
// Private state for setupRequestAuth (key: string, value: bearerToken) // Private state for setupRequestAuth (key: string, value: bearerToken)
tokenCache sync.Map tokenCache sync.Map
// detectPropertiesError caches the initial error. // Private state for detectProperties:
detectPropertiesError error detectPropertiesOnce sync.Once // detectPropertiesOnce is used to execute detectProperties() at most once.
// detectPropertiesOnce is used to execuute detectProperties() at most once in in makeRequest(). detectPropertiesError error // detectPropertiesError caches the initial error.
detectPropertiesOnce sync.Once
} }
type authScope struct { type authScope struct {
@ -439,18 +439,11 @@ func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method, url
} }
} }
logrus.Debugf("%s %s", method, url) logrus.Debugf("%s %s", method, url)
res, err := c.client.Do(req)
// Build the transport and do the request by using the clients tlsclientconfig if err != nil {
return c.doHTTP(req) return nil, err
} }
return res, nil
// doHttp uses the clients internal TLS configuration for doing the
// provided HTTP request. It returns the response and an error on failure.
func (c *dockerClient) doHTTP(req *http.Request) (*http.Response, error) {
tr := tlsclientconfig.NewTransport()
tr.TLSClientConfig = c.tlsClientConfig
httpClient := &http.Client{Transport: tr}
return httpClient.Do(req)
} }
// we're using the challenges from the /v2/ ping response and not the one from the destination // we're using the challenges from the /v2/ ping response and not the one from the destination
@ -558,15 +551,14 @@ func (c *dockerClient) getBearerToken(ctx context.Context, challenge challenge,
// detectPropertiesHelper performs the work of detectProperties which executes // detectPropertiesHelper performs the work of detectProperties which executes
// it at most once. // it at most once.
func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error { func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error {
if c.scheme != "" {
return nil
}
// We overwrite the TLS clients `InsecureSkipVerify` only if explicitly // We overwrite the TLS clients `InsecureSkipVerify` only if explicitly
// specified by the system context // specified by the system context
if c.sys != nil && c.sys.DockerInsecureSkipTLSVerify != types.OptionalBoolUndefined { if c.sys != nil && c.sys.DockerInsecureSkipTLSVerify != types.OptionalBoolUndefined {
c.tlsClientConfig.InsecureSkipVerify = c.sys.DockerInsecureSkipTLSVerify == types.OptionalBoolTrue c.tlsClientConfig.InsecureSkipVerify = c.sys.DockerInsecureSkipTLSVerify == types.OptionalBoolTrue
} }
tr := tlsclientconfig.NewTransport()
tr.TLSClientConfig = c.tlsClientConfig
c.client = &http.Client{Transport: tr}
ping := func(scheme string) error { ping := func(scheme string) error {
url := fmt.Sprintf(resolvedPingV2URL, scheme, c.registry) url := fmt.Sprintf(resolvedPingV2URL, scheme, c.registry)

View File

@ -16,7 +16,7 @@ import (
"github.com/containers/image/pkg/sysregistriesv2" "github.com/containers/image/pkg/sysregistriesv2"
"github.com/containers/image/types" "github.com/containers/image/types"
"github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client"
"github.com/opencontainers/go-digest" digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -29,75 +29,55 @@ type dockerImageSource struct {
cachedManifestMIMEType string // Only valid if cachedManifest != nil cachedManifestMIMEType string // Only valid if cachedManifest != nil
} }
// newImageSource creates a new `ImageSource` for the specified image reference // newImageSource creates a new ImageSource for the specified image reference.
// `ref`. // The caller must call .Close() on the returned ImageSource.
//
// The following steps will be done during the instance creation:
//
// - Lookup the registry within the configured location in
// `sys.SystemRegistriesConfPath`. If there is no configured registry available,
// we fallback to the provided docker reference `ref`.
//
// - References which contain a configured prefix will be automatically rewritten
// to the correct target reference. For example, if the configured
// `prefix = "example.com/foo"`, `location = "example.com"` and the image will be
// pulled from the ref `example.com/foo/image`, then the resulting pull will
// effectively point to `example.com/image`.
//
// - If the rewritten reference succeeds, it will be used as the `dockerRef`
// in the client. If the rewrite fails, the function immediately returns an error.
//
// - Each mirror will be used (in the configured order) to test the
// availability of the image manifest on the remote location. For example,
// if the manifest is not reachable due to connectivity issues, then the next
// mirror will be tested instead. If no mirror is configured or contains the
// target manifest, then the initial `ref` will be tested as fallback. The
// creation of the new `dockerImageSource` only succeeds if a remote
// location with the available manifest was found.
//
// A cleanup call to `.Close()` is needed if the caller is done using the returned
// `ImageSource`.
func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerReference) (*dockerImageSource, error) { func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerReference) (*dockerImageSource, error) {
registry, err := sysregistriesv2.FindRegistry(sys, ref.ref.Name()) registry, err := sysregistriesv2.FindRegistry(sys, ref.ref.Name())
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "error loading registries configuration") return nil, errors.Wrapf(err, "error loading registries configuration")
} }
if registry == nil { if registry == nil {
// No configuration was found for the provided reference, so we create // No configuration was found for the provided reference, so use the
// a fallback registry by hand to make the client creation below work // equivalent of a default configuration.
// as intended.
registry = &sysregistriesv2.Registry{ registry = &sysregistriesv2.Registry{
Endpoint: sysregistriesv2.Endpoint{ Endpoint: sysregistriesv2.Endpoint{
Location: ref.ref.String(), Location: ref.ref.String(),
}, },
Prefix: ref.ref.String(),
} }
} }
// Found the registry within the sysregistriesv2 configuration. Now we test primaryDomain := reference.Domain(ref.ref)
// all endpoints for the manifest availability. If a working image source // Check all endpoints for the manifest availability. If we find one that does
// was found, it will be used for all future pull actions. // contain the image, it will be used for all future pull actions. Always try the
var ( // non-mirror original location last; this both transparently handles the case
imageSource *dockerImageSource // of no mirrors configured, and ensures we return the error encountered when
manifestLoadErr error // acessing the upstream location if all endpoints fail.
) manifestLoadErr := errors.New("Internal error: newImageSource returned without trying any endpoint")
for _, endpoint := range append(registry.Mirrors, registry.Endpoint) { pullSources, err := registry.PullSourcesFromReference(ref.ref)
logrus.Debugf("Trying to pull %q from endpoint %q", ref.ref, endpoint.Location) if err != nil {
return nil, err
newRef, err := endpoint.RewriteReference(ref.ref, registry.Prefix) }
if err != nil { for _, pullSource := range pullSources {
return nil, err logrus.Debugf("Trying to pull %q", pullSource.Reference)
} dockerRef, err := newReference(pullSource.Reference)
dockerRef, err := newReference(newRef)
if err != nil { if err != nil {
return nil, err return nil, err
} }
client, err := newDockerClientFromRef(sys, dockerRef, false, "pull") endpointSys := sys
// sys.DockerAuthConfig does not explicitly specify a registry; we must not blindly send the credentials intended for the primary endpoint to mirrors.
if endpointSys != nil && endpointSys.DockerAuthConfig != nil && reference.Domain(dockerRef.ref) != primaryDomain {
copy := *endpointSys
copy.DockerAuthConfig = nil
endpointSys = &copy
}
client, err := newDockerClientFromRef(endpointSys, dockerRef, false, "pull")
if err != nil { if err != nil {
return nil, err return nil, err
} }
client.tlsClientConfig.InsecureSkipVerify = endpoint.Insecure client.tlsClientConfig.InsecureSkipVerify = pullSource.Endpoint.Insecure
testImageSource := &dockerImageSource{ testImageSource := &dockerImageSource{
ref: dockerRef, ref: dockerRef,
@ -106,12 +86,10 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerRef
manifestLoadErr = testImageSource.ensureManifestIsLoaded(ctx) manifestLoadErr = testImageSource.ensureManifestIsLoaded(ctx)
if manifestLoadErr == nil { if manifestLoadErr == nil {
imageSource = testImageSource return testImageSource, nil
break
} }
} }
return nil, manifestLoadErr
return imageSource, manifestLoadErr
} }
// Reference returns the reference used to set up this source, _as specified by the user_ // Reference returns the reference used to set up this source, _as specified by the user_
@ -347,7 +325,7 @@ func (s *dockerImageSource) getOneSignature(ctx context.Context, url *url.URL) (
return nil, false, err return nil, false, err
} }
req = req.WithContext(ctx) req = req.WithContext(ctx)
res, err := s.c.doHTTP(req) res, err := s.c.client.Do(req)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }

View File

@ -1,2 +1,2 @@
This is a copy of github.com/docker/distribution/reference as of commit fb0bebc4b64e3881cc52a2478d749845ed76d2a8, This is a copy of github.com/docker/distribution/reference as of commit 3226863cbcba6dbc2f6c83a37b28126c934af3f8,
except that ParseAnyReferenceWithSet has been removed to drop the dependency on github.com/docker/distribution/digestset. except that ParseAnyReferenceWithSet has been removed to drop the dependency on github.com/docker/distribution/digestset.

View File

@ -55,6 +55,35 @@ func ParseNormalizedNamed(s string) (Named, error) {
return named, nil return named, nil
} }
// ParseDockerRef normalizes the image reference following the docker convention. This is added
// mainly for backward compatibility.
// The reference returned can only be either tagged or digested. For reference contains both tag
// and digest, the function returns digested reference, e.g. docker.io/library/busybox:latest@
// sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa will be returned as
// docker.io/library/busybox@sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa.
func ParseDockerRef(ref string) (Named, error) {
named, err := ParseNormalizedNamed(ref)
if err != nil {
return nil, err
}
if _, ok := named.(NamedTagged); ok {
if canonical, ok := named.(Canonical); ok {
// The reference is both tagged and digested, only
// return digested.
newNamed, err := WithName(canonical.Name())
if err != nil {
return nil, err
}
newCanonical, err := WithDigest(newNamed, canonical.Digest())
if err != nil {
return nil, err
}
return newCanonical, nil
}
}
return TagNameOnly(named), nil
}
// splitDockerDomain splits a repository name to domain and remotename string. // splitDockerDomain splits a repository name to domain and remotename string.
// If no valid domain is found, the default domain is used. Repository name // If no valid domain is found, the default domain is used. Repository name
// needs to be already validated before. // needs to be already validated before.

View File

@ -15,7 +15,7 @@
// tag := /[\w][\w.-]{0,127}/ // tag := /[\w][\w.-]{0,127}/
// //
// digest := digest-algorithm ":" digest-hex // digest := digest-algorithm ":" digest-hex
// digest-algorithm := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ] // digest-algorithm := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ]*
// digest-algorithm-separator := /[+.-_]/ // digest-algorithm-separator := /[+.-_]/
// digest-algorithm-component := /[A-Za-z][A-Za-z0-9]*/ // digest-algorithm-component := /[A-Za-z][A-Za-z0-9]*/
// digest-hex := /[0-9a-fA-F]{32,}/ ; At least 128 bit digest value // digest-hex := /[0-9a-fA-F]{32,}/ ; At least 128 bit digest value
@ -205,7 +205,7 @@ func Parse(s string) (Reference, error) {
var repo repository var repo repository
nameMatch := anchoredNameRegexp.FindStringSubmatch(matches[1]) nameMatch := anchoredNameRegexp.FindStringSubmatch(matches[1])
if nameMatch != nil && len(nameMatch) == 3 { if len(nameMatch) == 3 {
repo.domain = nameMatch[1] repo.domain = nameMatch[1]
repo.path = nameMatch[2] repo.path = nameMatch[2]
} else { } else {

View File

@ -20,15 +20,15 @@ var (
optional(repeated(separatorRegexp, alphaNumericRegexp))) optional(repeated(separatorRegexp, alphaNumericRegexp)))
// domainComponentRegexp restricts the registry domain component of a // domainComponentRegexp restricts the registry domain component of a
// repository name to start with a component as defined by domainRegexp // repository name to start with a component as defined by DomainRegexp
// and followed by an optional port. // and followed by an optional port.
domainComponentRegexp = match(`(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])`) domainComponentRegexp = match(`(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])`)
// domainRegexp defines the structure of potential domain components // DomainRegexp defines the structure of potential domain components
// that may be part of image names. This is purposely a subset of what is // that may be part of image names. This is purposely a subset of what is
// allowed by DNS to ensure backwards compatibility with Docker image // allowed by DNS to ensure backwards compatibility with Docker image
// names. // names.
domainRegexp = expression( DomainRegexp = expression(
domainComponentRegexp, domainComponentRegexp,
optional(repeated(literal(`.`), domainComponentRegexp)), optional(repeated(literal(`.`), domainComponentRegexp)),
optional(literal(`:`), match(`[0-9]+`))) optional(literal(`:`), match(`[0-9]+`)))
@ -51,14 +51,14 @@ var (
// regexp has capturing groups for the domain and name part omitting // regexp has capturing groups for the domain and name part omitting
// the separating forward slash from either. // the separating forward slash from either.
NameRegexp = expression( NameRegexp = expression(
optional(domainRegexp, literal(`/`)), optional(DomainRegexp, literal(`/`)),
nameComponentRegexp, nameComponentRegexp,
optional(repeated(literal(`/`), nameComponentRegexp))) optional(repeated(literal(`/`), nameComponentRegexp)))
// anchoredNameRegexp is used to parse a name value, capturing the // anchoredNameRegexp is used to parse a name value, capturing the
// domain and trailing components. // domain and trailing components.
anchoredNameRegexp = anchored( anchoredNameRegexp = anchored(
optional(capture(domainRegexp), literal(`/`)), optional(capture(DomainRegexp), literal(`/`)),
capture(nameComponentRegexp, capture(nameComponentRegexp,
optional(repeated(literal(`/`), nameComponentRegexp)))) optional(repeated(literal(`/`), nameComponentRegexp))))

View File

@ -5,6 +5,7 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"strings" "strings"
"sync" "sync"
@ -35,30 +36,22 @@ type Endpoint struct {
Insecure bool `toml:"insecure"` Insecure bool `toml:"insecure"`
} }
// RewriteReference will substitute the provided reference `prefix` to the // rewriteReference will substitute the provided reference `prefix` to the
// endpoints `location` from the `ref` and creates a new named reference from it. // endpoints `location` from the `ref` and creates a new named reference from it.
// The function errors if the newly created reference is not parsable. // The function errors if the newly created reference is not parsable.
func (e *Endpoint) RewriteReference(ref reference.Named, prefix string) (reference.Named, error) { func (e *Endpoint) rewriteReference(ref reference.Named, prefix string) (reference.Named, error) {
if ref == nil {
return nil, fmt.Errorf("provided reference is nil")
}
if prefix == "" {
return ref, nil
}
refString := ref.String() refString := ref.String()
if refMatchesPrefix(refString, prefix) { if !refMatchesPrefix(refString, prefix) {
newNamedRef := strings.Replace(refString, prefix, e.Location, 1) return nil, fmt.Errorf("invalid prefix '%v' for reference '%v'", prefix, refString)
newParsedRef, err := reference.ParseNamed(newNamedRef)
if newParsedRef != nil {
logrus.Debugf("reference rewritten from '%v' to '%v'", refString, newParsedRef.String())
}
if err != nil {
return nil, errors.Wrapf(err, "error rewriting reference")
}
return newParsedRef, nil
} }
return nil, fmt.Errorf("invalid prefix '%v' for reference '%v'", prefix, refString) newNamedRef := strings.Replace(refString, prefix, e.Location, 1)
newParsedRef, err := reference.ParseNamed(newNamedRef)
if err != nil {
return nil, errors.Wrapf(err, "error rewriting reference")
}
logrus.Debugf("reference rewritten from '%v' to '%v'", refString, newParsedRef.String())
return newParsedRef, nil
} }
// Registry represents a registry. // Registry represents a registry.
@ -69,8 +62,10 @@ type Registry struct {
Mirrors []Endpoint `toml:"mirror"` Mirrors []Endpoint `toml:"mirror"`
// If true, pulling from the registry will be blocked. // If true, pulling from the registry will be blocked.
Blocked bool `toml:"blocked"` Blocked bool `toml:"blocked"`
// If true, the registry can be used when pulling an unqualified image. // If true, mirrors will only be used for digest pulls. Pulling images by
Search bool `toml:"unqualified-search"` // tag can potentially yield different images, depending on which endpoint
// we pull from. Forcing digest-pulls for mirrors avoids that issue.
MirrorByDigestOnly bool `toml:"mirror-by-digest-only"`
// Prefix is used for matching images, and to translate one namespace to // Prefix is used for matching images, and to translate one namespace to
// another. If `Prefix="example.com/bar"`, `location="example.com/foo/bar"` // another. If `Prefix="example.com/bar"`, `location="example.com/foo/bar"`
// and we pull from "example.com/bar/myimage:latest", the image will // and we pull from "example.com/bar/myimage:latest", the image will
@ -79,6 +74,41 @@ type Registry struct {
Prefix string `toml:"prefix"` Prefix string `toml:"prefix"`
} }
// PullSource consists of an Endpoint and a Reference. Note that the reference is
// rewritten according to the registries prefix and the Endpoint's location.
type PullSource struct {
Endpoint Endpoint
Reference reference.Named
}
// PullSourcesFromReference returns a slice of PullSource's based on the passed
// reference.
func (r *Registry) PullSourcesFromReference(ref reference.Named) ([]PullSource, error) {
var endpoints []Endpoint
if r.MirrorByDigestOnly {
// Only use mirrors when the reference is a digest one.
if _, isDigested := ref.(reference.Canonical); isDigested {
endpoints = append(r.Mirrors, r.Endpoint)
} else {
endpoints = []Endpoint{r.Endpoint}
}
} else {
endpoints = append(r.Mirrors, r.Endpoint)
}
sources := []PullSource{}
for _, ep := range endpoints {
rewritten, err := ep.rewriteReference(ref, r.Prefix)
if err != nil {
return nil, err
}
sources = append(sources, PullSource{Endpoint: ep, Reference: rewritten})
}
return sources, nil
}
// V1TOMLregistries is for backwards compatibility to sysregistries v1 // V1TOMLregistries is for backwards compatibility to sysregistries v1
type V1TOMLregistries struct { type V1TOMLregistries struct {
Registries []string `toml:"registries"` Registries []string `toml:"registries"`
@ -91,11 +121,35 @@ type V1TOMLConfig struct {
Block V1TOMLregistries `toml:"block"` Block V1TOMLregistries `toml:"block"`
} }
// V1RegistriesConf is the sysregistries v1 configuration format.
type V1RegistriesConf struct {
V1TOMLConfig `toml:"registries"`
}
// Nonempty returns true if config contains at least one configuration entry.
func (config *V1RegistriesConf) Nonempty() bool {
return (len(config.V1TOMLConfig.Search.Registries) != 0 ||
len(config.V1TOMLConfig.Insecure.Registries) != 0 ||
len(config.V1TOMLConfig.Block.Registries) != 0)
}
// V2RegistriesConf is the sysregistries v2 configuration format.
type V2RegistriesConf struct {
Registries []Registry `toml:"registry"`
// An array of host[:port] (not prefix!) entries to use for resolving unqualified image references
UnqualifiedSearchRegistries []string `toml:"unqualified-search-registries"`
}
// Nonempty returns true if config contains at least one configuration entry.
func (config *V2RegistriesConf) Nonempty() bool {
return (len(config.Registries) != 0 ||
len(config.UnqualifiedSearchRegistries) != 0)
}
// tomlConfig is the data type used to unmarshal the toml config. // tomlConfig is the data type used to unmarshal the toml config.
type tomlConfig struct { type tomlConfig struct {
Registries []Registry `toml:"registry"` V2RegistriesConf
// backwards compatability to sysregistries v1 V1RegistriesConf // for backwards compatibility with sysregistries v1
V1TOMLConfig `toml:"registries"`
} }
// InvalidRegistries represents an invalid registry configurations. An example // InvalidRegistries represents an invalid registry configurations. An example
@ -128,12 +182,10 @@ func parseLocation(input string) (string, error) {
return trimmed, nil return trimmed, nil
} }
// getV1Registries transforms v1 registries in the config into an array of v2 // ConvertToV2 returns a v2 config corresponding to a v1 one.
// registries of type Registry. func (config *V1RegistriesConf) ConvertToV2() (*V2RegistriesConf, error) {
func getV1Registries(config *tomlConfig) ([]Registry, error) {
regMap := make(map[string]*Registry) regMap := make(map[string]*Registry)
// We must preserve the order of config.V1Registries.Search.Registries at least. The order of the // The order of the registries is not really important, but make it deterministic (the same for the same config file)
// other registries is not really important, but make it deterministic (the same for the same config file)
// to minimize behavior inconsistency and not contribute to difficult-to-reproduce situations. // to minimize behavior inconsistency and not contribute to difficult-to-reproduce situations.
registryOrder := []string{} registryOrder := []string{}
@ -156,15 +208,6 @@ func getV1Registries(config *tomlConfig) ([]Registry, error) {
return reg, nil return reg, nil
} }
// Note: config.V1Registries.Search needs to be processed first to ensure registryOrder is populated in the right order
// if one of the search registries is also in one of the other lists.
for _, search := range config.V1TOMLConfig.Search.Registries {
reg, err := getRegistry(search)
if err != nil {
return nil, err
}
reg.Search = true
}
for _, blocked := range config.V1TOMLConfig.Block.Registries { for _, blocked := range config.V1TOMLConfig.Block.Registries {
reg, err := getRegistry(blocked) reg, err := getRegistry(blocked)
if err != nil { if err != nil {
@ -180,28 +223,31 @@ func getV1Registries(config *tomlConfig) ([]Registry, error) {
reg.Insecure = true reg.Insecure = true
} }
registries := []Registry{} res := &V2RegistriesConf{
UnqualifiedSearchRegistries: config.V1TOMLConfig.Search.Registries,
}
for _, location := range registryOrder { for _, location := range registryOrder {
reg := regMap[location] reg := regMap[location]
registries = append(registries, *reg) res.Registries = append(res.Registries, *reg)
} }
return registries, nil return res, nil
} }
// postProcessRegistries checks the consistency of all registries (e.g., set // anchoredDomainRegexp is an internal implementation detail of postProcess, defining the valid values of elements of UnqualifiedSearchRegistries.
// the Prefix to Location if not set) and applies conflict checks. It returns an var anchoredDomainRegexp = regexp.MustCompile("^" + reference.DomainRegexp.String() + "$")
// array of cleaned registries and error in case of conflicts.
func postProcessRegistries(regs []Registry) ([]Registry, error) {
var registries []Registry
regMap := make(map[string][]Registry)
for _, reg := range regs { // postProcess checks the consistency of all the configuration, looks for conflicts,
var err error // and normalizes the configuration (e.g., sets the Prefix to Location if not set).
func (config *V2RegistriesConf) postProcess() error {
regMap := make(map[string][]*Registry)
for i := range config.Registries {
reg := &config.Registries[i]
// make sure Location and Prefix are valid // make sure Location and Prefix are valid
var err error
reg.Location, err = parseLocation(reg.Location) reg.Location, err = parseLocation(reg.Location)
if err != nil { if err != nil {
return nil, err return err
} }
if reg.Prefix == "" { if reg.Prefix == "" {
@ -209,7 +255,7 @@ func postProcessRegistries(regs []Registry) ([]Registry, error) {
} else { } else {
reg.Prefix, err = parseLocation(reg.Prefix) reg.Prefix, err = parseLocation(reg.Prefix)
if err != nil { if err != nil {
return nil, err return err
} }
} }
@ -217,10 +263,9 @@ func postProcessRegistries(regs []Registry) ([]Registry, error) {
for _, mir := range reg.Mirrors { for _, mir := range reg.Mirrors {
mir.Location, err = parseLocation(mir.Location) mir.Location, err = parseLocation(mir.Location)
if err != nil { if err != nil {
return nil, err return err
} }
} }
registries = append(registries, reg)
regMap[reg.Location] = append(regMap[reg.Location], reg) regMap[reg.Location] = append(regMap[reg.Location], reg)
} }
@ -230,22 +275,32 @@ func postProcessRegistries(regs []Registry) ([]Registry, error) {
// //
// Note: we need to iterate over the registries array to ensure a // Note: we need to iterate over the registries array to ensure a
// deterministic behavior which is not guaranteed by maps. // deterministic behavior which is not guaranteed by maps.
for _, reg := range registries { for _, reg := range config.Registries {
others, _ := regMap[reg.Location] others, _ := regMap[reg.Location]
for _, other := range others { for _, other := range others {
if reg.Insecure != other.Insecure { if reg.Insecure != other.Insecure {
msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'insecure' setting", reg.Location) msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'insecure' setting", reg.Location)
return &InvalidRegistries{s: msg}
return nil, &InvalidRegistries{s: msg}
} }
if reg.Blocked != other.Blocked { if reg.Blocked != other.Blocked {
msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'blocked' setting", reg.Location) msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'blocked' setting", reg.Location)
return nil, &InvalidRegistries{s: msg} return &InvalidRegistries{s: msg}
} }
} }
} }
return registries, nil for i := range config.UnqualifiedSearchRegistries {
registry, err := parseLocation(config.UnqualifiedSearchRegistries[i])
if err != nil {
return err
}
if !anchoredDomainRegexp.MatchString(registry) {
return &InvalidRegistries{fmt.Sprintf("Invalid unqualified-search-registries entry %#v", registry)}
}
config.UnqualifiedSearchRegistries[i] = registry
}
return nil
} }
// getConfigPath returns the system-registries config path if specified. // getConfigPath returns the system-registries config path if specified.
@ -268,7 +323,7 @@ var configMutex = sync.Mutex{}
// configCache caches already loaded configs with config paths as keys and is // configCache caches already loaded configs with config paths as keys and is
// used to avoid redudantly parsing configs. Concurrent accesses to the cache // used to avoid redudantly parsing configs. Concurrent accesses to the cache
// are synchronized via configMutex. // are synchronized via configMutex.
var configCache = make(map[string][]Registry) var configCache = make(map[string]*V2RegistriesConf)
// InvalidateCache invalidates the registry cache. This function is meant to be // InvalidateCache invalidates the registry cache. This function is meant to be
// used for long-running processes that need to reload potential changes made to // used for long-running processes that need to reload potential changes made to
@ -276,20 +331,18 @@ var configCache = make(map[string][]Registry)
func InvalidateCache() { func InvalidateCache() {
configMutex.Lock() configMutex.Lock()
defer configMutex.Unlock() defer configMutex.Unlock()
configCache = make(map[string][]Registry) configCache = make(map[string]*V2RegistriesConf)
} }
// GetRegistries loads and returns the registries specified in the config. // getConfig returns the config object corresponding to ctx, loading it if it is not yet cached.
// Note the parsed content of registry config files is cached. For reloading, func getConfig(ctx *types.SystemContext) (*V2RegistriesConf, error) {
// use `InvalidateCache` and re-call `GetRegistries`.
func GetRegistries(ctx *types.SystemContext) ([]Registry, error) {
configPath := getConfigPath(ctx) configPath := getConfigPath(ctx)
configMutex.Lock() configMutex.Lock()
defer configMutex.Unlock() defer configMutex.Unlock()
// if the config has already been loaded, return the cached registries // if the config has already been loaded, return the cached registries
if registries, inCache := configCache[configPath]; inCache { if config, inCache := configCache[configPath]; inCache {
return registries, nil return config, nil
} }
// load the config // load the config
@ -300,51 +353,53 @@ func GetRegistries(ctx *types.SystemContext) ([]Registry, error) {
// isn't set. Note: if ctx.SystemRegistriesConfPath points to // isn't set. Note: if ctx.SystemRegistriesConfPath points to
// the default config, we will still return an error. // the default config, we will still return an error.
if os.IsNotExist(err) && (ctx == nil || ctx.SystemRegistriesConfPath == "") { if os.IsNotExist(err) && (ctx == nil || ctx.SystemRegistriesConfPath == "") {
return []Registry{}, nil return &V2RegistriesConf{Registries: []Registry{}}, nil
} }
return nil, err return nil, err
} }
registries := config.Registries v2Config := &config.V2RegistriesConf
// backwards compatibility for v1 configs // backwards compatibility for v1 configs
v1Registries, err := getV1Registries(config) if config.V1RegistriesConf.Nonempty() {
if err != nil { if config.V2RegistriesConf.Nonempty() {
return nil, err
}
if len(v1Registries) > 0 {
if len(registries) > 0 {
return nil, &InvalidRegistries{s: "mixing sysregistry v1/v2 is not supported"} return nil, &InvalidRegistries{s: "mixing sysregistry v1/v2 is not supported"}
} }
registries = v1Registries v2, err := config.V1RegistriesConf.ConvertToV2()
if err != nil {
return nil, err
}
v2Config = v2
} }
registries, err = postProcessRegistries(registries) if err := v2Config.postProcess(); err != nil {
if err != nil {
return nil, err return nil, err
} }
// populate the cache // populate the cache
configCache[configPath] = registries configCache[configPath] = v2Config
return v2Config, nil
return registries, err
} }
// FindUnqualifiedSearchRegistries returns all registries that are configured // GetRegistries loads and returns the registries specified in the config.
// for unqualified image search (i.e., with Registry.Search == true). // Note the parsed content of registry config files is cached. For reloading,
func FindUnqualifiedSearchRegistries(ctx *types.SystemContext) ([]Registry, error) { // use `InvalidateCache` and re-call `GetRegistries`.
registries, err := GetRegistries(ctx) func GetRegistries(ctx *types.SystemContext) ([]Registry, error) {
config, err := getConfig(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return config.Registries, nil
}
unqualified := []Registry{} // UnqualifiedSearchRegistries returns a list of host[:port] entries to try
for _, reg := range registries { // for unqualified image search, in the returned order)
if reg.Search { func UnqualifiedSearchRegistries(ctx *types.SystemContext) ([]string, error) {
unqualified = append(unqualified, reg) config, err := getConfig(ctx)
} if err != nil {
return nil, err
} }
return unqualified, nil return config.UnqualifiedSearchRegistries, nil
} }
// refMatchesPrefix returns true iff ref, // refMatchesPrefix returns true iff ref,
@ -379,14 +434,14 @@ func refMatchesPrefix(ref, prefix string) bool {
// — note that this requires the name to start with an explicit hostname!). // — note that this requires the name to start with an explicit hostname!).
// If no Registry prefixes the image, nil is returned. // If no Registry prefixes the image, nil is returned.
func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) { func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) {
registries, err := GetRegistries(ctx) config, err := getConfig(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
reg := Registry{} reg := Registry{}
prefixLen := 0 prefixLen := 0
for _, r := range registries { for _, r := range config.Registries {
if refMatchesPrefix(ref, r.Prefix) { if refMatchesPrefix(ref, r.Prefix) {
length := len(r.Prefix) length := len(r.Prefix)
if length > prefixLen { if length > prefixLen {
@ -401,21 +456,12 @@ func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) {
return nil, nil return nil, nil
} }
// Reads the global registry file from the filesystem. Returns a byte array.
func readRegistryConf(configPath string) ([]byte, error) {
configBytes, err := ioutil.ReadFile(configPath)
return configBytes, err
}
// Used in unittests to parse custom configs without a types.SystemContext.
var readConf = readRegistryConf
// Loads the registry configuration file from the filesystem and then unmarshals // Loads the registry configuration file from the filesystem and then unmarshals
// it. Returns the unmarshalled object. // it. Returns the unmarshalled object.
func loadRegistryConf(configPath string) (*tomlConfig, error) { func loadRegistryConf(configPath string) (*tomlConfig, error) {
config := &tomlConfig{} config := &tomlConfig{}
configBytes, err := readConf(configPath) configBytes, err := ioutil.ReadFile(configPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -22,6 +22,7 @@ import (
// ParseImageName converts a URL-like image name to a types.ImageReference. // ParseImageName converts a URL-like image name to a types.ImageReference.
func ParseImageName(imgName string) (types.ImageReference, error) { func ParseImageName(imgName string) (types.ImageReference, error) {
// Keep this in sync with TransportFromImageName!
parts := strings.SplitN(imgName, ":", 2) parts := strings.SplitN(imgName, ":", 2)
if len(parts) != 2 { if len(parts) != 2 {
return nil, errors.Errorf(`Invalid image name "%s", expected colon-separated transport:reference`, imgName) return nil, errors.Errorf(`Invalid image name "%s", expected colon-separated transport:reference`, imgName)
@ -32,3 +33,14 @@ func ParseImageName(imgName string) (types.ImageReference, error) {
} }
return transport.ParseReference(parts[1]) return transport.ParseReference(parts[1])
} }
// TransportFromImageName converts an URL-like name to a types.ImageTransport or nil when
// the transport is unknown or when the input is invalid.
func TransportFromImageName(imageName string) types.ImageTransport {
// Keep this in sync with ParseImageName!
parts := strings.SplitN(imageName, ":", 2)
if len(parts) == 2 {
return transports.Get(parts[0])
}
return nil
}

View File

@ -4,14 +4,14 @@ import "fmt"
const ( const (
// VersionMajor is for an API incompatible changes // VersionMajor is for an API incompatible changes
VersionMajor = 1 VersionMajor = 2
// VersionMinor is for functionality in a backwards-compatible manner // VersionMinor is for functionality in a backwards-compatible manner
VersionMinor = 7 VersionMinor = 0
// VersionPatch is for backwards-compatible bug fixes // VersionPatch is for backwards-compatible bug fixes
VersionPatch = 0 VersionPatch = 0
// VersionDev indicates development branch. Releases will be empty string. // VersionDev indicates development branch. Releases will be empty string.
VersionDev = "-dev" VersionDev = ""
) )
// Version is the specification version that the package types support. // Version is the specification version that the package types support.