root
/
helm
mirror of https://github.com/helm/helm.git
1
0
Fork 0
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Implement `helm pull` for OCI registries 

* Implement `helm dep update` for oci dependencies
* New unit tests
* Remove `helm chart pull` command
* New `helm pull` does not depend on registry cache

Signed-off-by: Peter Engelbert <pmengelbert@gmail.com>
This commit is contained in:
Peter Engelbert 2020-10-01 16:37:44 -05:00
parent cc1d2d62e9
commit 3ad08f3ea9
18 changed files with 585 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/helm/dependency.go
View File

@ -82,7 +82,7 @@ the contents of a chart.
This will produce an error if the chart cannot be loaded.
`
func newDependencyCmd(out io.Writer) *cobra.Command {
func newDependencyCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
cmd := &cobra.Command{
Use: "dependency update|build|list",
Aliases: []string{"dep", "dependencies"},
@ -92,7 +92,7 @@ func newDependencyCmd(out io.Writer) *cobra.Command {
}
cmd.AddCommand(newDependencyListCmd(out))
cmd.AddCommand(newDependencyUpdateCmd(out))
cmd.AddCommand(newDependencyUpdateCmd(cfg, out))
cmd.AddCommand(newDependencyBuildCmd(out))
return cmd

3
cmd/helm/dependency_update.go
View File

@ -43,7 +43,7 @@ in the Chart.yaml file, but (b) at the wrong version.
`
// newDependencyUpdateCmd creates a new dependency update command.
func newDependencyUpdateCmd(out io.Writer) *cobra.Command {
func newDependencyUpdateCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
client := action.NewDependency()
cmd := &cobra.Command{
@ -63,6 +63,7 @@ func newDependencyUpdateCmd(out io.Writer) *cobra.Command {
Keyring: client.Keyring,
SkipUpdate: client.SkipRefresh,
Getters: getter.All(settings),
RegistryClient: cfg.RegistryClient,
RepositoryConfig: settings.RepositoryConfig,
RepositoryCache: settings.RepositoryCache,
Debug: settings.Debug,

46
cmd/helm/dependency_update_test.go
View File

@ -40,6 +40,23 @@ func TestDependencyUpdateCmd(t *testing.T) {
defer srv.Stop()
t.Logf("Listening on directory %s", srv.Root())
ociSrv, err := repotest.NewOCIServer(t, srv.Root())
if err != nil {
t.Fatal(err)
}
ociChartName := "oci-depending-chart"
c := createTestingMetadataForOCI(ociChartName, ociSrv.RegistryURL)
if err := chartutil.SaveDir(c, ociSrv.Dir); err != nil {
t.Fatal(err)
}
ociSrv.Run(t, repotest.WithDependingChart(c))
err = os.Setenv("HELM_EXPERIMENTAL_OCI", "1")
if err != nil {
t.Fatal("failed to set environment variable enabling OCI support")
}
if err := srv.LinkIndices(); err != nil {
t.Fatal(err)
}
@ -115,6 +132,22 @@ func TestDependencyUpdateCmd(t *testing.T) {
if _, err := os.Stat(unexpected); err == nil {
t.Fatalf("Unexpected %q", unexpected)
}
// test for OCI charts
cmd := fmt.Sprintf("dependency update '%s' --repository-config %s --repository-cache %s --registry-config %s/config.json",
dir(ociChartName),
dir("repositories.yaml"),
dir(),
dir())
_, out, err = executeActionCommand(cmd)
if err != nil {
t.Logf("Output: %s", out)
t.Fatal(err)
}
expect = dir(ociChartName, "charts/oci-dependent-chart")
if _, err := os.Stat(expect); err != nil {
t.Fatal(err)
}
}
func TestDependencyUpdateCmd_DontDeleteOldChartsOnError(t *testing.T) {
@ -193,6 +226,19 @@ func createTestingMetadata(name, baseURL string) *chart.Chart {
}
}
func createTestingMetadataForOCI(name, registryURL string) *chart.Chart {
return &chart.Chart{
Metadata: &chart.Metadata{
APIVersion: chart.APIVersionV2,
Name: name,
Version: "1.2.3",
Dependencies: []*chart.Dependency{
{Name: "oci-dependent-chart", Version: "0.1.0", Repository: fmt.Sprintf("oci://%s/u/ocitestuser", registryURL)},
},
},
}
}
// createTestingChart creates a basic chart that depends on reqtest-0.1.0
//
// The baseURL can be used to point to a particular repository server.

13
cmd/helm/pull.go
View File

@ -20,6 +20,7 @@ import (
"fmt"
"io"
"log"
"strings"
"github.com/spf13/cobra"
@ -42,8 +43,8 @@ file, and MUST pass the verification process. Failure in any part of this will
result in an error, and the chart will not be saved locally.
`
func newPullCmd(out io.Writer) *cobra.Command {
client := action.NewPull()
func newPullCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
client := action.NewPull(cfg)
cmd := &cobra.Command{
Use: "pull [chart URL | repo/chartname] [...]",
@ -64,6 +65,14 @@ func newPullCmd(out io.Writer) *cobra.Command {
client.Version = ">0.0.0-0"
}
if strings.HasPrefix(args[0], "oci://") {
if !FeatureGateOCI.IsEnabled() {
return FeatureGateOCI.Error()
}
client.OCI = true
}
for i := 0; i < len(args); i++ {
output, err := client.Run(args[i])
if err != nil {

58
cmd/helm/pull_test.go
View File

@ -32,6 +32,13 @@ func TestPullCmd(t *testing.T) {
}
defer srv.Stop()
os.Setenv("HELM_EXPERIMENTAL_OCI", "1")
ociSrv, err := repotest.NewOCIServer(t, srv.Root())
if err != nil {
t.Fatal(err)
}
ociSrv.Run(t)
if err := srv.LinkIndices(); err != nil {
t.Fatal(err)
}
@ -139,23 +146,70 @@ func TestPullCmd(t *testing.T) {
failExpect: "Failed to fetch chart version",
wantError: true,
},
{
name: "Fetch OCI Chart",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0", ociSrv.RegistryURL),
expectFile: "./oci-dependent-chart-0.1.0.tgz",
},
{
name: "Fetch OCI Chart with untar",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar", ociSrv.RegistryURL),
expectFile: "./oci-dependent-chart",
expectDir: true,
},
{
name: "Fetch OCI Chart with untar and untardir",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar --untardir ocitest2", ociSrv.RegistryURL),
expectFile: "./ocitest2",
expectDir: true,
},
{
name: "OCI Fetch untar when dir with same name existed",
args: fmt.Sprintf("oci-test-chart oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar --untardir ocitest2 --untar --untardir ocitest2", ociSrv.RegistryURL),
wantError: true,
wantErrorMsg: fmt.Sprintf("failed to untar: a file or directory with the name %s already exists", filepath.Join(srv.Root(), "ocitest2")),
},
{
name: "Fail fetching non-existent OCI chart",
args: fmt.Sprintf("oci://%s/u/ocitestuser/nosuchthing --version 0.1.0", ociSrv.RegistryURL),
failExpect: "Failed to fetch",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/nosuchthing", ociSrv.RegistryURL),
wantErrorMsg: "Error: --version flag is explicitly required for OCI registries",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart:0.1.0", ociSrv.RegistryURL),
wantErrorMsg: "Error: --version flag is explicitly required for OCI registries",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart:0.1.0 --version 0.1.0", ociSrv.RegistryURL),
wantError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
outdir := srv.Root()
cmd := fmt.Sprintf("fetch %s -d '%s' --repository-config %s --repository-cache %s ",
cmd := fmt.Sprintf("fetch %s -d '%s' --repository-config %s --repository-cache %s --registry-config %s",
tt.args,
outdir,
filepath.Join(outdir, "repositories.yaml"),
outdir,
filepath.Join(outdir, "config.json"),
)
// Create file or Dir before helm pull --untar, see: https://github.com/helm/helm/issues/7182
if tt.existFile != "" {
file := filepath.Join(outdir, tt.existFile)
_, err := os.Create(file)
if err != nil {
t.Fatal("err")
t.Fatal(err)
}
}
if tt.existDir != "" {

23
cmd/helm/root.go
View File

@ -153,12 +153,22 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
flags.ParseErrorsWhitelist.UnknownFlags = true
flags.Parse(args)
registryClient, err := registry.NewClient(
registry.ClientOptDebug(settings.Debug),
registry.ClientOptWriter(out),
registry.ClientOptCredentialsFile(settings.RegistryConfig),
)
if err != nil {
return nil, err
}
actionConfig.RegistryClient = registryClient
// Add subcommands
cmd.AddCommand(
// chart commands
newCreateCmd(out),
newDependencyCmd(out),
newPullCmd(out),
newDependencyCmd(actionConfig, out),
newPullCmd(actionConfig, out),
newShowCmd(out),
newLintCmd(out),
newPackageCmd(out),
@ -188,15 +198,6 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
)
// Add *experimental* subcommands
registryClient, err := registry.NewClient(
registry.ClientOptDebug(settings.Debug),
registry.ClientOptWriter(out),
registry.ClientOptCredentialsFile(settings.RegistryConfig),
)
if err != nil {
return nil, err
}
actionConfig.RegistryClient = registryClient
cmd.AddCommand(
newRegistryCmd(actionConfig, out),
newChartCmd(actionConfig, out),

BIN
cmd/helm/testdata/testcharts/oci-dependent-chart-0.1.0.tgz vendored Normal file
View File

Binary file not shown.

61
internal/experimental/registry/client.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package registry // import "helm.sh/helm/v3/internal/experimental/registry"
import (
"bytes"
"context"
"fmt"
"io"
@ -24,14 +25,16 @@ import (
"net/http"
"sort"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/helmpath"
"github.com/deislabs/oras/pkg/content"
auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/deislabs/oras/pkg/oras"
"github.com/gosuri/uitable"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/helmpath"
)
const (
@ -144,7 +147,57 @@ func (c *Client) PushChart(ref *Reference) error {
}
// PullChart downloads a chart from a registry
func (c *Client) PullChart(ref *Reference) error {
func (c *Client) PullChart(ref *Reference) (*bytes.Buffer, error) {
buf := bytes.NewBuffer(nil)
if ref.Tag == "" {
return buf, errors.New("tag explicitly required")
}
fmt.Fprintf(c.out, "%s: Pulling from %s\n", ref.Tag, ref.Repo)
store := content.NewMemoryStore()
fullname := ref.FullName()
_ = fullname
_, layerDescriptors, err := oras.Pull(ctx(c.out, c.debug), c.resolver, ref.FullName(), store,
oras.WithPullEmptyNameAllowed(),
oras.WithAllowedMediaTypes(KnownMediaTypes()))
if err != nil {
return buf, err
}
numLayers := len(layerDescriptors)
if numLayers < 1 {
return buf, errors.New(
fmt.Sprintf("manifest does not contain at least 1 layer (total: %d)", numLayers))
}
var contentLayer *ocispec.Descriptor
for _, layer := range layerDescriptors {
layer := layer
switch layer.MediaType {
case HelmChartContentLayerMediaType:
contentLayer = &layer
}
}
if contentLayer == nil {
return buf, errors.New(
fmt.Sprintf("manifest does not contain a layer with mediatype %s",
HelmChartContentLayerMediaType))
}
_, b, ok := store.Get(*contentLayer)
if !ok {
return buf, errors.Errorf("Unable to retrieve blob with digest %s", contentLayer.Digest)
}
buf = bytes.NewBuffer(b)
return buf, nil
}
func (c *Client) PullChartToCache(ref *Reference) error {
if ref.Tag == "" {
return errors.New("tag explicitly required")
}

6
internal/experimental/registry/client_test.go
View File

@ -202,13 +202,13 @@ func (suite *RegistryClientTestSuite) Test_4_PullChart() {
// non-existent ref
ref, err := ParseReference(fmt.Sprintf("%s/testrepo/whodis:9.9.9", suite.DockerRegistryHost))
suite.Nil(err)
err = suite.RegistryClient.PullChart(ref)
_, err = suite.RegistryClient.PullChart(ref)
suite.NotNil(err)
// existing ref
ref, err = ParseReference(fmt.Sprintf("%s/testrepo/testchart:1.2.3", suite.DockerRegistryHost))
suite.Nil(err)
err = suite.RegistryClient.PullChart(ref)
_, err = suite.RegistryClient.PullChart(ref)
suite.Nil(err)
}
@ -245,7 +245,7 @@ func (suite *RegistryClientTestSuite) Test_8_ManInTheMiddle() {
suite.Nil(err)
// returns content that does not match the expected digest
err = suite.RegistryClient.PullChart(ref)
_, err = suite.RegistryClient.PullChart(ref)
suite.NotNil(err)
suite.True(errdefs.IsFailedPrecondition(err))
}

39
internal/resolver/resolver.go
View File

@ -23,16 +23,19 @@ import (
"strings"
"time"
"github.com/Masterminds/semver/v3"
"github.com/pkg/errors"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/gates"
"helm.sh/helm/v3/pkg/helmpath"
"helm.sh/helm/v3/pkg/provenance"
"helm.sh/helm/v3/pkg/repo"
"github.com/Masterminds/semver/v3"
"github.com/pkg/errors"
)
const FeatureGateOCI = gates.Gate("HELM_EXPERIMENTAL_OCI")
// Resolver resolves dependencies from semantic version ranges to a particular version.
type Resolver struct {
chartpath string
@ -88,6 +91,7 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
}
continue
}
constraint, err := semver.NewConstraint(d.Version)
if err != nil {
return nil, errors.Wrapf(err, "dependency %q has an invalid version/constraint format", d.Name)
@ -104,21 +108,34 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
continue
}
repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName)))
if err != nil {
return nil, errors.Wrapf(err, "no cached repository for %s found. (try 'helm repo update')", repoName)
}
var vs repo.ChartVersions
var version string
var ok bool
found := true
if !strings.HasPrefix(d.Repository, "oci://") {
repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName)))
if err != nil {
return nil, errors.Wrapf(err, "no cached repository for %s found. (try 'helm repo update')", repoName)
}
vs, ok := repoIndex.Entries[d.Name]
if !ok {
return nil, errors.Errorf("%s chart not found in repo %s", d.Name, d.Repository)
vs, ok = repoIndex.Entries[d.Name]
if !ok {
return nil, errors.Errorf("%s chart not found in repo %s", d.Name, d.Repository)
}
found = false
} else {
version = d.Version
if !FeatureGateOCI.IsEnabled() {
return nil, errors.Wrapf(FeatureGateOCI.Error(),
"repository %s is an OCI registry", d.Repository)
}
}
locked[i] = &chart.Dependency{
Name: d.Name,
Repository: d.Repository,
Version: version,
}
found := false
// The version are already sorted and hence the first one to satisfy the constraint is used
for _, ver := range vs {
v, err := semver.NewVersion(ver.Version)

2
pkg/action/chart_pull.go
View File

@ -40,5 +40,5 @@ func (a *ChartPull) Run(out io.Writer, ref string) error {
if err != nil {
return err
}
return a.cfg.RegistryClient.PullChart(r)
return a.cfg.RegistryClient.PullChartToCache(r)
}

17
pkg/action/pull.go
View File

@ -43,13 +43,15 @@ type Pull struct {
Devel bool
Untar bool
VerifyLater bool
OCI bool
UntarDir string
DestDir string
cfg *Configuration
}
// NewPull creates a new Pull object with the given configuration.
func NewPull() *Pull {
return &Pull{}
func NewPull(cfg *Configuration) *Pull {
return &Pull{cfg: cfg}
}
// Run executes 'helm pull' against the given release.
@ -70,6 +72,16 @@ func (p *Pull) Run(chartRef string) (string, error) {
RepositoryCache: p.Settings.RepositoryCache,
}
if p.OCI {
if p.Version == "" {
return out.String(), errors.Errorf("--version flag is explicitly required for OCI registries")
}
c.Options = append(c.Options,
getter.WithRegistryClient(p.cfg.RegistryClient),
getter.WithTagName(p.Version))
}
if p.Verify {
c.Verify = downloader.VerifyAlways
} else if p.VerifyLater {
@ -123,6 +135,7 @@ func (p *Pull) Run(chartRef string) (string, error) {
_, chartName := filepath.Split(chartRef)
udCheck = filepath.Join(udCheck, chartName)
}
if _, err := os.Stat(udCheck); err != nil {
if err := os.MkdirAll(udCheck, 0755); err != nil {
return out.String(), errors.Wrap(err, "failed to untar (mkdir)")

6
pkg/downloader/chart_downloader.go
View File

@ -25,6 +25,7 @@ import (
"github.com/pkg/errors"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/internal/fileutil"
"helm.sh/helm/v3/internal/urlutil"
"helm.sh/helm/v3/pkg/getter"
@ -68,6 +69,7 @@ type ChartDownloader struct {
Getters getter.Providers
// Options provide parameters to be passed along to the Getter being initialized.
Options []getter.Option
RegistryClient *registry.Client
RepositoryConfig string
RepositoryCache string
}
@ -100,6 +102,10 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven
}
name := filepath.Base(u.Path)
if u.Scheme == "oci" {
name = fmt.Sprintf("%s-%s.tgz", name, version)
}
destfile := filepath.Join(dest, name)
if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil {
return destfile, nil, err

60
pkg/downloader/manager.go
View File

@ -26,6 +26,7 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"strings"
"sync"
@ -33,6 +34,7 @@ import (
"github.com/pkg/errors"
"sigs.k8s.io/yaml"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/internal/resolver"
"helm.sh/helm/v3/internal/third_party/dep/fs"
"helm.sh/helm/v3/internal/urlutil"
@ -71,6 +73,7 @@ type Manager struct {
SkipUpdate bool
// Getter collection for the operation
Getters []getter.Provider
RegistryClient *registry.Client
RepositoryConfig string
RepositoryCache string
}
@ -332,11 +335,40 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error {
},
}
if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil {
untar, version := false, ""
if strings.HasPrefix(churl, "oci://") {
if !resolver.FeatureGateOCI.IsEnabled() {
return errors.Wrapf(resolver.FeatureGateOCI.Error(),
"the repository %s is an OCI registry", churl)
}
churl, version, err = parseOCIRef(churl)
if err != nil {
return errors.Wrapf(err, "could not parse OCI reference")
}
untar = true
dl.Options = append(dl.Options,
getter.WithRegistryClient(m.RegistryClient),
getter.WithTagName(version))
}
destFile, _, err := dl.DownloadTo(churl, version, destPath)
if err != nil {
saveError = errors.Wrapf(err, "could not download %s", churl)
break
}
if untar {
err = chartutil.ExpandFile(destPath, destFile)
if err != nil {
return errors.Wrapf(err, "could not open %s to untar", destFile)
}
err = os.RemoveAll(destFile)
if err != nil {
return errors.Wrapf(err, "chart was downloaded and untarred, but was unable to remove the tarball: %s", destFile)
}
}
churls[churl] = struct{}{}
}
@ -375,6 +407,18 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error {
return nil
}
func parseOCIRef(chartRef string) (string, string, error) {
refTagRegexp := regexp.MustCompile(`^(oci://[^:]+(:[0-9]{1,5})?[^:]+):(.*)$`)
caps := refTagRegexp.FindStringSubmatch(chartRef)
if len(caps) != 4 {
return "", "", errors.Errorf("improperly formatted oci chart reference: %s", chartRef)
}
chartRef = caps[1]
tag := caps[3]
return chartRef, tag, nil
}
// safeDeleteDep deletes any versions of the given dependency in the given directory.
//
// It does this by first matching the file name to an expected pattern, then loading
@ -539,6 +583,11 @@ func (m *Manager) resolveRepoNames(deps []*chart.Dependency) (map[string]string,
continue
}
if strings.HasPrefix(dd.Repository, "oci://") {
reposMap[dd.Name] = dd.Repository
continue
}
found := false
for _, repo := range repos {
@ -648,7 +697,12 @@ func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error {
//
// If it finds a URL that is "relative", it will prepend the repoURL.
func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, err error) {
if strings.HasPrefix(repoURL, "oci://") {
return fmt.Sprintf("%s/%s:%s", repoURL, name, version), "", "", nil
}
for _, cr := range repos {
if urlutil.Equal(repoURL, cr.Config.URL) {
var entry repo.ChartVersions
entry, err = findEntryByName(name, cr)
@ -671,10 +725,10 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*
}
url, err = repo.FindChartInRepoURL(repoURL, name, version, "", "", "", m.Getters)
if err == nil {
return
return url, username, password, err
}
err = errors.Errorf("chart %s not found in %s: %s", name, repoURL, err)
return
return url, username, password, err
}
// findEntryByName finds an entry in the chart repository whose name matches the given name.

29
pkg/getter/getter.go
View File

@ -22,6 +22,7 @@ import (
"github.com/pkg/errors"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/pkg/cli"
)
@ -33,10 +34,13 @@ type options struct {
certFile string
keyFile string
caFile string
unTar bool
insecureSkipVerifyTLS bool
username string
password string
userAgent string
version string
registryClient *registry.Client
timeout time.Duration
}
@ -90,6 +94,24 @@ func WithTimeout(timeout time.Duration) Option {
}
}
func WithTagName(tagname string) Option {
return func(opts *options) {
opts.version = tagname
}
}
func WithRegistryClient(client *registry.Client) Option {
return func(opts *options) {
opts.registryClient = client
}
}
func WithUntar() Option {
return func(opts *options) {
opts.unTar = true
}
}
// Getter is an interface to support GET to the specified URL.
type Getter interface {
// Get file content by url string
@ -139,11 +161,16 @@ var httpProvider = Provider{
New: NewHTTPGetter,
}
var ociProvider = Provider{
Schemes: []string{"oci"},
New: NewOCIGetter,
}
// All finds all of the registered getters as a list of Provider instances.
// Currently, the built-in getters and the discovered plugins with downloader
// notations are collected.
func All(settings *cli.EnvSettings) Providers {
result := Providers{httpProvider}
result := Providers{httpProvider, ociProvider}
pluginDownloaders, _ := collectPlugins(settings)
result = append(result, pluginDownloaders...)
return result

2
pkg/getter/getter_test.go
View File

@ -57,7 +57,7 @@ func TestAll(t *testing.T) {
env.PluginsDirectory = pluginDir
all := All(env)
if len(all) != 3 {
if len(all) != 4 {
t.Errorf("expected 3 providers (default plus two plugins), got %d", len(all))
}

69
pkg/getter/ocigetter.go Normal file
View File

@ -0,0 +1,69 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package getter
import (
"bytes"
"fmt"
"strings"
"helm.sh/helm/v3/internal/experimental/registry"
)
// OCIGetter is the default HTTP(/S) backend handler
type OCIGetter struct {
opts options
}
//Get performs a Get from repo.Getter and returns the body.
func (g *OCIGetter) Get(href string, options ...Option) (*bytes.Buffer, error) {
for _, opt := range options {
opt(&g.opts)
}
return g.get(href)
}
func (g *OCIGetter) get(href string) (*bytes.Buffer, error) {
client := g.opts.registryClient
ref := strings.TrimPrefix(href, "oci://")
if version := g.opts.version; version != "" {
ref = fmt.Sprintf("%s:%s", ref, version)
}
r, err := registry.ParseReference(ref)
if err != nil {
return nil, err
}
buf, err := client.PullChart(r)
if err != nil {
return nil, err
}
return buf, nil
}
// NewOCIGetter constructs a valid http/https client as a Getter
func NewOCIGetter(options ...Option) (Getter, error) {
var client OCIGetter
for _, opt := range options {
opt(&client.opts)
}
return &client, nil
}

192
pkg/repo/repotest/server.go
View File

@ -16,18 +16,34 @@ limitations under the License.
package repotest
import (
"context"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"helm.sh/helm/v3/internal/tlsutil"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/repo"
"sigs.k8s.io/yaml"
"helm.sh/helm/v3/pkg/repo"
auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/registry"
_ "github.com/docker/distribution/registry/auth/htpasswd" // used for docker test registry
_ "github.com/docker/distribution/registry/storage/driver/inmemory" // used for docker test registry
ociRegistry "helm.sh/helm/v3/internal/experimental/registry"
"golang.org/x/crypto/bcrypt"
)
// NewTempServerWithCleanup creates a server inside of a temp dir.
@ -43,6 +59,166 @@ func NewTempServerWithCleanup(t *testing.T, glob string) (*Server, error) {
return srv, err
}
type OCIServer struct {
*registry.Registry
RegistryURL string
Dir string
TestUsername string
TestPassword string
Client *ociRegistry.Client
}
type OCIServerRunConfig struct {
DependingChart *chart.Chart
}
type OCIServerOpt func(config *OCIServerRunConfig)
func WithDependingChart(c *chart.Chart) OCIServerOpt {
return func(config *OCIServerRunConfig) {
config.DependingChart = c
}
}
func NewOCIServer(t *testing.T, dir string) (*OCIServer, error) {
testHtpasswdFileBasename := "authtest.htpasswd"
testUsername, testPassword := "username", "password"
pwBytes, err := bcrypt.GenerateFromPassword([]byte(testPassword), bcrypt.DefaultCost)
if err != nil {
t.Fatal("error generating bcrypt password for test htpasswd file")
}
htpasswdPath := filepath.Join(dir, testHtpasswdFileBasename)
err = ioutil.WriteFile(htpasswdPath, []byte(fmt.Sprintf("%s:%s\n", testUsername, string(pwBytes))), 0644)
if err != nil {
t.Fatalf("error creating test htpasswd file")
}
// Registry config
config := &configuration.Configuration{}
port, err := getFreePort()
if err != nil {
t.Fatalf("error finding free port for test registry")
}
config.HTTP.Addr = fmt.Sprintf(":%d", port)
config.HTTP.DrainTimeout = time.Duration(10) * time.Second
config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}}
config.Auth = configuration.Auth{
"htpasswd": configuration.Parameters{
"realm": "localhost",
"path": htpasswdPath,
},
}
registryURL := fmt.Sprintf("localhost:%d", port)
r, err := registry.NewRegistry(context.Background(), config)
if err != nil {
t.Fatal(err)
}
return &OCIServer{
Registry: r,
RegistryURL: registryURL,
TestUsername: testUsername,
TestPassword: testPassword,
Dir: dir,
}, nil
}
func (srv *OCIServer) Run(t *testing.T, opts ...OCIServerOpt) {
cfg := &OCIServerRunConfig{}
for _, fn := range opts {
fn(cfg)
}
go srv.ListenAndServe()
credentialsFile := filepath.Join(srv.Dir, "config.json")
client, err := auth.NewClient(credentialsFile)
if err != nil {
t.Fatalf("error creating auth client")
}
resolver, err := client.Resolver(context.Background(), http.DefaultClient, false)
if err != nil {
t.Fatalf("error creating resolver")
}
// init test client
registryClient, err := ociRegistry.NewClient(
ociRegistry.ClientOptDebug(true),
ociRegistry.ClientOptWriter(os.Stdout),
ociRegistry.ClientOptAuthorizer(&ociRegistry.Authorizer{
Client: client,
}),
ociRegistry.ClientOptResolver(&ociRegistry.Resolver{
Resolver: resolver,
}),
)
if err != nil {
t.Fatalf("error creating registry client")
}
err = registryClient.Login(srv.RegistryURL, srv.TestUsername, srv.TestPassword, false)
if err != nil {
t.Fatalf("error logging into registry with good credentials")
}
ref, err := ociRegistry.ParseReference(fmt.Sprintf("%s/u/ocitestuser/oci-dependent-chart:0.1.0", srv.RegistryURL))
if err != nil {
t.Fatalf("")
}
err = chartutil.ExpandFile(srv.Dir, filepath.Join(srv.Dir, "oci-dependent-chart-0.1.0.tgz"))
if err != nil {
t.Fatal(err)
}
// valid chart
ch, err := loader.LoadDir(filepath.Join(srv.Dir, "oci-dependent-chart"))
if err != nil {
t.Fatal("error loading chart")
}
err = os.RemoveAll(filepath.Join(srv.Dir, "oci-dependent-chart"))
if err != nil {
t.Fatal("error removing chart before push")
}
err = registryClient.SaveChart(ch, ref)
if err != nil {
t.Fatal("error saving chart")
}
err = registryClient.PushChart(ref)
if err != nil {
t.Fatal("error pushing chart")
}
if cfg.DependingChart != nil {
c := cfg.DependingChart
dependingRef, err := ociRegistry.ParseReference(fmt.Sprintf("%s/u/ocitestuser/oci-depending-chart:1.2.3", srv.RegistryURL))
if err != nil {
t.Fatal("error parsing reference for depending chart reference")
}
err = registryClient.SaveChart(c, dependingRef)
if err != nil {
t.Fatal("error saving depending chart")
}
err = registryClient.PushChart(dependingRef)
if err != nil {
t.Fatal("error pushing depending chart")
}
}
srv.Client = registryClient
}
// NewTempServer creates a server inside of a temp dir.
//
// If the passed in string is not "", it will be treated as a shell glob, and files
@ -228,3 +404,17 @@ func setTestingRepository(url, fname string) error {
})
return r.WriteFile(fname, 0644)
}
func getFreePort() (int, error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return 0, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return 0, err
}
defer l.Close()
return l.Addr().(*net.TCPAddr).Port, nil
}