diff --git a/go.mod b/go.mod index d328ad8bb1a..49f96ea97bd 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,6 @@ require ( k8s.io/client-go v0.23.1 k8s.io/klog/v2 v2.30.0 k8s.io/kubectl v0.23.1 - oras.land/oras-go v1.1.0-rc3 + oras.land/oras-go v1.1.0 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index c56a4588583..88da64deb14 100644 --- a/go.sum +++ b/go.sum @@ -1746,8 +1746,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -oras.land/oras-go v1.1.0-rc3 h1:+HdHR0Lgm/0jmjF4SqUif8Ky2XNWhrcP4hxGVvtKIUI= -oras.land/oras-go v1.1.0-rc3/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ= +oras.land/oras-go v1.1.0 h1:tfWM1RT7PzUwWphqHU6ptPU3ZhwVnSw/9nEGf519rYg= +oras.land/oras-go v1.1.0/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/experimental/registry/client.go b/internal/experimental/registry/client.go index 3bdfac20bb9..1b686b8baed 100644 --- a/internal/experimental/registry/client.go +++ b/internal/experimental/registry/client.go @@ -17,13 +17,16 @@ limitations under the License. package registry // import "helm.sh/helm/v3/internal/experimental/registry" import ( + "context" "encoding/json" "fmt" "io" "io/ioutil" "net/http" + "sort" "strings" + "github.com/Masterminds/semver/v3" "github.com/containerd/containerd/remotes" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -31,6 +34,9 @@ import ( dockerauth "oras.land/oras-go/pkg/auth/docker" "oras.land/oras-go/pkg/content" "oras.land/oras-go/pkg/oras" + "oras.land/oras-go/pkg/registry" + registryremote "oras.land/oras-go/pkg/registry/remote" + registryauth "oras.land/oras-go/pkg/registry/remote/auth" "helm.sh/helm/v3/internal/version" "helm.sh/helm/v3/pkg/chart" @@ -49,10 +55,11 @@ type ( Client struct { debug bool // path to repository config file e.g. ~/.docker/config.json - credentialsFile string - out io.Writer - authorizer auth.Client - resolver remotes.Resolver + credentialsFile string + out io.Writer + authorizer auth.Client + registryAuthorizer *registryauth.Client + resolver remotes.Resolver } // ClientOption allows specifying various settings configurable by the user for overriding the defaults @@ -88,6 +95,32 @@ func NewClient(options ...ClientOption) (*Client, error) { } client.resolver = resolver } + if client.registryAuthorizer == nil { + client.registryAuthorizer = ®istryauth.Client{ + Header: http.Header{ + "User-Agent": {version.GetUserAgent()}, + }, + Cache: registryauth.DefaultCache, + Credential: func(ctx context.Context, reg string) (registryauth.Credential, error) { + dockerClient, ok := client.authorizer.(*dockerauth.Client) + if !ok { + return registryauth.EmptyCredential, errors.New("unable to obtain docker client") + } + + username, password, err := dockerClient.Credential(reg) + if err != nil { + return registryauth.EmptyCredential, errors.New("unable to retrieve credentials") + } + + return registryauth.Credential{ + Username: username, + Password: password, + }, nil + + }, + } + + } return client, nil } @@ -539,3 +572,55 @@ func PushOptStrictMode(strictMode bool) PushOption { operation.strictMode = strictMode } } + +// Tags provides a sorted list all semver compliant tags for a given repository +func (c *Client) Tags(ref string) ([]string, error) { + parsedReference, err := registry.ParseReference(ref) + if err != nil { + return nil, err + } + + repository := registryremote.Repository{ + Reference: parsedReference, + Client: c.registryAuthorizer, + } + + var registryTags []string + + for { + registryTags, err = registry.Tags(ctx(c.out, c.debug), &repository) + if err != nil { + // Fallback to http based request + if !repository.PlainHTTP && strings.Contains(err.Error(), "server gave HTTP response") { + repository.PlainHTTP = true + continue + } + return nil, err + } + + break + + } + + var tagVersions []*semver.Version + for _, tag := range registryTags { + // Change underscore (_) back to plus (+) for Helm + // See https://github.com/helm/helm/issues/10166 + tagVersion, err := semver.StrictNewVersion(strings.ReplaceAll(tag, "_", "+")) + if err == nil { + tagVersions = append(tagVersions, tagVersion) + } + } + + // Sort the collection + sort.Sort(sort.Reverse(semver.Collection(tagVersions))) + + tags := make([]string, len(tagVersions)) + + for iTv, tv := range tagVersions { + tags[iTv] = tv.String() + } + + return tags, nil + +} diff --git a/internal/experimental/registry/client_test.go b/internal/experimental/registry/client_test.go index 356f3eaba4c..baf9b429108 100644 --- a/internal/experimental/registry/client_test.go +++ b/internal/experimental/registry/client_test.go @@ -294,7 +294,23 @@ func (suite *RegistryClientTestSuite) Test_2_Pull() { suite.Equal(provData, result.Prov.Data) } -func (suite *RegistryClientTestSuite) Test_3_Logout() { +func (suite *RegistryClientTestSuite) Test_3_Tags() { + + // Load test chart (to build ref pushed in previous test) + chartData, err := ioutil.ReadFile("../../../pkg/downloader/testdata/local-subchart-0.1.0.tgz") + suite.Nil(err, "no error loading test chart") + meta, err := extractChartMeta(chartData) + suite.Nil(err, "no error extracting chart meta") + ref := fmt.Sprintf("%s/testrepo/%s", suite.DockerRegistryHost, meta.Name) + + // Query for tags and validate length + tags, err := suite.RegistryClient.Tags(ref) + suite.Nil(err, "no error retrieving tags") + suite.Equal(1, len(tags)) + +} + +func (suite *RegistryClientTestSuite) Test_4_Logout() { err := suite.RegistryClient.Logout("this-host-aint-real:5000") suite.NotNil(err, "error logging out of registry that has no entry") @@ -302,7 +318,7 @@ func (suite *RegistryClientTestSuite) Test_3_Logout() { suite.Nil(err, "no error logging out of registry") } -func (suite *RegistryClientTestSuite) Test_4_ManInTheMiddle() { +func (suite *RegistryClientTestSuite) Test_5_ManInTheMiddle() { ref := fmt.Sprintf("%s/testrepo/supposedlysafechart:9.9.9", suite.CompromisedRegistryHost) // returns content that does not match the expected digest diff --git a/internal/experimental/registry/util.go b/internal/experimental/registry/util.go index c421e5639e0..8c6eb19581d 100644 --- a/internal/experimental/registry/util.go +++ b/internal/experimental/registry/util.go @@ -23,6 +23,8 @@ import ( "io" "strings" + "github.com/Masterminds/semver/v3" + "github.com/pkg/errors" "github.com/sirupsen/logrus" orascontext "oras.land/oras-go/pkg/context" "oras.land/oras-go/pkg/registry" @@ -36,6 +38,53 @@ func IsOCI(url string) bool { return strings.HasPrefix(url, fmt.Sprintf("%s://", OCIScheme)) } +// ContainsTag determines whether a tag is found in a provided list of tags +func ContainsTag(tags []string, tag string) bool { + for _, t := range tags { + if tag == t { + return true + } + } + return false +} + +func GetTagMatchingVersionOrConstraint(tags []string, versionString string) (string, error) { + var constraint *semver.Constraints + if versionString == "" { + // If string is empty, set wildcard constraint + constraint, _ = semver.NewConstraint("*") + } else { + // when customer input exact version, check whether have exact match + // one first + for _, v := range tags { + if versionString == v { + return v, nil + } + } + + // Otherwise set constraint to the string given + var err error + constraint, err = semver.NewConstraint(versionString) + if err != nil { + return "", err + } + } + + // Otherwise try to find the first available version matching the string, + // in case it is a constraint + for _, v := range tags { + test, err := semver.NewVersion(v) + if err != nil { + continue + } + if constraint.Check(test) { + return v, nil + } + } + + return "", errors.Errorf("Could not locate a version matching provided version string %s", versionString) +} + // extractChartMeta is used to extract a chart metadata from a byte array func extractChartMeta(chartData []byte) (*chart.Metadata, error) { ch, err := loader.LoadArchive(bytes.NewReader(chartData)) diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 70ce6a55bfa..cf370e927a9 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -18,6 +18,7 @@ package resolver import ( "bytes" "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -39,15 +40,17 @@ const FeatureGateOCI = gates.Gate("HELM_EXPERIMENTAL_OCI") // Resolver resolves dependencies from semantic version ranges to a particular version. type Resolver struct { - chartpath string - cachepath string + chartpath string + cachepath string + registryClient *registry.Client } -// New creates a new resolver for a given chart and a given helm home. -func New(chartpath, cachepath string) *Resolver { +// New creates a new resolver for a given chart, helm home and registry client. +func New(chartpath, cachepath string, registryClient *registry.Client) *Resolver { return &Resolver{ - chartpath: chartpath, - cachepath: cachepath, + chartpath: chartpath, + cachepath: cachepath, + registryClient: registryClient, } } @@ -139,6 +142,24 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string return nil, errors.Wrapf(FeatureGateOCI.Error(), "repository %s is an OCI registry", d.Repository) } + + // Retrieve list of tags for repository + ref := fmt.Sprintf("%s/%s", strings.TrimPrefix(d.Repository, fmt.Sprintf("%s://", registry.OCIScheme)), d.Name) + tags, err := r.registryClient.Tags(ref) + if err != nil { + return nil, errors.Wrapf(err, "could not retrieve list of tags for repository %s", d.Repository) + } + + vs = make(repo.ChartVersions, len(tags)) + for ti, t := range tags { + // Mock chart version objects + version := &repo.ChartVersion{ + Metadata: &chart.Metadata{ + Version: t, + }, + } + vs[ti] = version + } } locked[i] = &chart.Dependency{ @@ -149,7 +170,8 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string // The version are already sorted and hence the first one to satisfy the constraint is used for _, ver := range vs { v, err := semver.NewVersion(ver.Version) - if err != nil || len(ver.URLs) == 0 { + // OCI does not need URLs + if err != nil || (!registry.IsOCI(d.Repository) && len(ver.URLs) == 0) { // Not a legit entry. continue } diff --git a/internal/resolver/resolver_test.go b/internal/resolver/resolver_test.go index 419f8f316f8..6fbc2e86af2 100644 --- a/internal/resolver/resolver_test.go +++ b/internal/resolver/resolver_test.go @@ -19,6 +19,7 @@ import ( "runtime" "testing" + "helm.sh/helm/v3/internal/experimental/registry" "helm.sh/helm/v3/pkg/chart" ) @@ -139,7 +140,8 @@ func TestResolve(t *testing.T) { } repoNames := map[string]string{"alpine": "kubernetes-charts", "redis": "kubernetes-charts"} - r := New("testdata/chartpath", "testdata/repository") + registryClient, _ := registry.NewClient() + r := New("testdata/chartpath", "testdata/repository", registryClient) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { l, err := r.Resolve(tt.req, repoNames) diff --git a/pkg/action/install.go b/pkg/action/install.go index 922f728b05b..250dfae95a1 100644 --- a/pkg/action/install.go +++ b/pkg/action/install.go @@ -695,10 +695,9 @@ func (c *ChartPathOptions) LocateChart(name string, settings *cli.EnvSettings) ( } if registry.IsOCI(name) { - if version == "" { - return "", errors.New("version is explicitly required for OCI registries") + if version != "" { + dl.Options = append(dl.Options, getter.WithTagName(version)) } - dl.Options = append(dl.Options, getter.WithTagName(version)) } if c.Verify { diff --git a/pkg/action/pull.go b/pkg/action/pull.go index 2f5127ea92b..55a127d4d7a 100644 --- a/pkg/action/pull.go +++ b/pkg/action/pull.go @@ -87,18 +87,14 @@ func (p *Pull) Run(chartRef string) (string, error) { getter.WithTLSClientConfig(p.CertFile, p.KeyFile, p.CaFile), getter.WithInsecureSkipVerifyTLS(p.InsecureSkipTLSverify), }, + RegistryClient: p.cfg.RegistryClient, RepositoryConfig: p.Settings.RepositoryConfig, RepositoryCache: p.Settings.RepositoryCache, } if registry.IsOCI(chartRef) { - if p.Version == "" { - return out.String(), errors.Errorf("--version flag is explicitly required for OCI registries") - } - c.Options = append(c.Options, - getter.WithRegistryClient(p.cfg.RegistryClient), - getter.WithTagName(p.Version)) + getter.WithRegistryClient(p.cfg.RegistryClient)) } if p.Verify { diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index 93afb1461cf..8ca5cacb644 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -103,7 +103,8 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven name := filepath.Base(u.Path) if u.Scheme == registry.OCIScheme { - name = fmt.Sprintf("%s-%s.tgz", name, version) + idx := strings.LastIndexByte(name, ':') + name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) } destfile := filepath.Join(dest, name) @@ -139,12 +140,37 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven return destfile, ver, nil } +func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, error) { + // Retrieve list of repository tags + tags, err := c.RegistryClient.Tags(strings.TrimPrefix(ref, fmt.Sprintf("%s://", registry.OCIScheme))) + if err != nil { + return nil, err + } + if len(tags) == 0 { + return nil, errors.Errorf("Unable to locate any tags in provided repository: %s", ref) + } + + // Determine if version provided + // If empty, try to get the highest available tag + // If exact version, try to find it + // If semver constraint string, try to find a match + tag, err := registry.GetTagMatchingVersionOrConstraint(tags, version) + if err != nil { + return nil, err + } + + u.Path = fmt.Sprintf("%s:%s", u.Path, tag) + + return u, err +} + // ResolveChartVersion resolves a chart reference to a URL. // // It returns the URL and sets the ChartDownloader's Options that can fetch // the URL using the appropriate Getter. // -// A reference may be an HTTP URL, a 'reponame/chartname' reference, or a local path. +// A reference may be an HTTP URL, an oci reference URL, a 'reponame/chartname' +// reference, or a local path. // // A version is a SemVer string (1.2.3-beta.1+f334a6789). // @@ -159,6 +185,10 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, er return nil, errors.Errorf("invalid chart URL format: %s", ref) } + if registry.IsOCI(u.String()) { + return c.getOciURI(ref, version, u) + } + rf, err := loadRepoConfig(c.RepositoryConfig) if err != nil { return u, err diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index fc6c7a432cf..cc59ae864a3 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -232,7 +232,7 @@ func (m *Manager) loadChartDir() (*chart.Chart, error) { // // This returns a lock file, which has all of the dependencies normalized to a specific version. func (m *Manager) resolve(req []*chart.Dependency, repoNames map[string]string) (*chart.Lock, error) { - res := resolver.New(m.ChartPath, m.RepositoryCache) + res := resolver.New(m.ChartPath, m.RepositoryCache, m.RegistryClient) return res.Resolve(req, repoNames) } @@ -332,6 +332,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { Keyring: m.Keyring, RepositoryConfig: m.RepositoryConfig, RepositoryCache: m.RepositoryCache, + RegistryClient: m.RegistryClient, Getters: m.Getters, Options: []getter.Option{ getter.WithBasicAuth(username, password), diff --git a/pkg/getter/ocigetter.go b/pkg/getter/ocigetter.go index 45c92749c58..dbf3821028f 100644 --- a/pkg/getter/ocigetter.go +++ b/pkg/getter/ocigetter.go @@ -28,7 +28,7 @@ type OCIGetter struct { opts options } -//Get performs a Get from repo.Getter and returns the body. +// Get performs a Get from repo.Getter and returns the body. func (g *OCIGetter) Get(href string, options ...Option) (*bytes.Buffer, error) { for _, opt := range options { opt(&g.opts) @@ -50,10 +50,6 @@ func (g *OCIGetter) get(href string) (*bytes.Buffer, error) { registry.PullOptWithProv(true)) } - if version := g.opts.version; version != "" { - ref = fmt.Sprintf("%s:%s", ref, version) - } - result, err := client.Pull(ref, pullOpts...) if err != nil { return nil, err