Merge pull request helm#10527 from scottrigby/oci-deppendency-version…

…-range-support

OCI version range support

go.mod

@@ -42,6 +42,6 @@ require (
 	k8s.io/client-go v0.23.1
 	k8s.io/klog/v2 v2.30.0
 	k8s.io/kubectl v0.23.1
-	oras.land/oras-go v1.1.0-rc3
+	oras.land/oras-go v1.1.0
 	sigs.k8s.io/yaml v1.3.0
 )

go.sum

@@ -1746,8 +1746,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-oras.land/oras-go v1.1.0-rc3 h1:+HdHR0Lgm/0jmjF4SqUif8Ky2XNWhrcP4hxGVvtKIUI=
-oras.land/oras-go v1.1.0-rc3/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ=
+oras.land/oras-go v1.1.0 h1:tfWM1RT7PzUwWphqHU6ptPU3ZhwVnSw/9nEGf519rYg=
+oras.land/oras-go v1.1.0/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

internal/experimental/registry/client.go

@@ -17,20 +17,26 @@ limitations under the License.
 package registry // import "helm.sh/helm/v3/internal/experimental/registry"
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"sort"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/containerd/containerd/remotes"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"oras.land/oras-go/pkg/auth"
 	dockerauth "oras.land/oras-go/pkg/auth/docker"
 	"oras.land/oras-go/pkg/content"
 	"oras.land/oras-go/pkg/oras"
+	"oras.land/oras-go/pkg/registry"
+	registryremote "oras.land/oras-go/pkg/registry/remote"
+	registryauth "oras.land/oras-go/pkg/registry/remote/auth"
 
 	"helm.sh/helm/v3/internal/version"
 	"helm.sh/helm/v3/pkg/chart"
@@ -49,10 +55,11 @@ type (
 	Client struct {
 		debug bool
 		// path to repository config file e.g. ~/.docker/config.json
-		credentialsFile string
-		out             io.Writer
-		authorizer      auth.Client
-		resolver        remotes.Resolver
+		credentialsFile    string
+		out                io.Writer
+		authorizer         auth.Client
+		registryAuthorizer *registryauth.Client
+		resolver           remotes.Resolver
 	}
 
 	// ClientOption allows specifying various settings configurable by the user for overriding the defaults
@@ -88,6 +95,32 @@ func NewClient(options ...ClientOption) (*Client, error) {
 		}
 		client.resolver = resolver
 	}
+	if client.registryAuthorizer == nil {
+		client.registryAuthorizer = &registryauth.Client{
+			Header: http.Header{
+				"User-Agent": {version.GetUserAgent()},
+			},
+			Cache: registryauth.DefaultCache,
+			Credential: func(ctx context.Context, reg string) (registryauth.Credential, error) {
+				dockerClient, ok := client.authorizer.(*dockerauth.Client)
+				if !ok {
+					return registryauth.EmptyCredential, errors.New("unable to obtain docker client")
+				}
+
+				username, password, err := dockerClient.Credential(reg)
+				if err != nil {
+					return registryauth.EmptyCredential, errors.New("unable to retrieve credentials")
+				}
+
+				return registryauth.Credential{
+					Username: username,
+					Password: password,
+				}, nil
+
+			},
+		}
+
+	}
 	return client, nil
 }
 
@@ -539,3 +572,55 @@ func PushOptStrictMode(strictMode bool) PushOption {
 		operation.strictMode = strictMode
 	}
 }
+
+// Tags provides a sorted list all semver compliant tags for a given repository
+func (c *Client) Tags(ref string) ([]string, error) {
+	parsedReference, err := registry.ParseReference(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	repository := registryremote.Repository{
+		Reference: parsedReference,
+		Client:    c.registryAuthorizer,
+	}
+
+	var registryTags []string
+
+	for {
+		registryTags, err = registry.Tags(ctx(c.out, c.debug), &repository)
+		if err != nil {
+			// Fallback to http based request
+			if !repository.PlainHTTP && strings.Contains(err.Error(), "server gave HTTP response") {
+				repository.PlainHTTP = true
+				continue
+			}
+			return nil, err
+		}
+
+		break
+
+	}
+
+	var tagVersions []*semver.Version
+	for _, tag := range registryTags {
+		// Change underscore (_) back to plus (+) for Helm
+		// See https://github.com/helm/helm/issues/10166
+		tagVersion, err := semver.StrictNewVersion(strings.ReplaceAll(tag, "_", "+"))
+		if err == nil {
+			tagVersions = append(tagVersions, tagVersion)
+		}
+	}
+
+	// Sort the collection
+	sort.Sort(sort.Reverse(semver.Collection(tagVersions)))
+
+	tags := make([]string, len(tagVersions))
+
+	for iTv, tv := range tagVersions {
+		tags[iTv] = tv.String()
+	}
+
+	return tags, nil
+
+}

internal/experimental/registry/client_test.go

@@ -294,15 +294,31 @@ func (suite *RegistryClientTestSuite) Test_2_Pull() {
 	suite.Equal(provData, result.Prov.Data)
 }
 
-func (suite *RegistryClientTestSuite) Test_3_Logout() {
+func (suite *RegistryClientTestSuite) Test_3_Tags() {
+
+	// Load test chart (to build ref pushed in previous test)
+	chartData, err := ioutil.ReadFile("../../../pkg/downloader/testdata/local-subchart-0.1.0.tgz")
+	suite.Nil(err, "no error loading test chart")
+	meta, err := extractChartMeta(chartData)
+	suite.Nil(err, "no error extracting chart meta")
+	ref := fmt.Sprintf("%s/testrepo/%s", suite.DockerRegistryHost, meta.Name)
+
+	// Query for tags and validate length
+	tags, err := suite.RegistryClient.Tags(ref)
+	suite.Nil(err, "no error retrieving tags")
+	suite.Equal(1, len(tags))
+
+}
+
+func (suite *RegistryClientTestSuite) Test_4_Logout() {
 	err := suite.RegistryClient.Logout("this-host-aint-real:5000")
 	suite.NotNil(err, "error logging out of registry that has no entry")
 
 	err = suite.RegistryClient.Logout(suite.DockerRegistryHost)
 	suite.Nil(err, "no error logging out of registry")
 }
 
-func (suite *RegistryClientTestSuite) Test_4_ManInTheMiddle() {
+func (suite *RegistryClientTestSuite) Test_5_ManInTheMiddle() {
 	ref := fmt.Sprintf("%s/testrepo/supposedlysafechart:9.9.9", suite.CompromisedRegistryHost)
 
 	// returns content that does not match the expected digest

internal/experimental/registry/util.go

@@ -23,6 +23,8 @@ import (
 	"io"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	orascontext "oras.land/oras-go/pkg/context"
 	"oras.land/oras-go/pkg/registry"
@@ -36,6 +38,53 @@ func IsOCI(url string) bool {
 	return strings.HasPrefix(url, fmt.Sprintf("%s://", OCIScheme))
 }
 
+// ContainsTag determines whether a tag is found in a provided list of tags
+func ContainsTag(tags []string, tag string) bool {
+	for _, t := range tags {
+		if tag == t {
+			return true
+		}
+	}
+	return false
+}
+
+func GetTagMatchingVersionOrConstraint(tags []string, versionString string) (string, error) {
+	var constraint *semver.Constraints
+	if versionString == "" {
+		// If string is empty, set wildcard constraint
+		constraint, _ = semver.NewConstraint("*")
+	} else {
+		// when customer input exact version, check whether have exact match
+		// one first
+		for _, v := range tags {
+			if versionString == v {
+				return v, nil
+			}
+		}
+
+		// Otherwise set constraint to the string given
+		var err error
+		constraint, err = semver.NewConstraint(versionString)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	// Otherwise try to find the first available version matching the string,
+	// in case it is a constraint
+	for _, v := range tags {
+		test, err := semver.NewVersion(v)
+		if err != nil {
+			continue
+		}
+		if constraint.Check(test) {
+			return v, nil
+		}
+	}
+
+	return "", errors.Errorf("Could not locate a version matching provided version string %s", versionString)
+}
+
 // extractChartMeta is used to extract a chart metadata from a byte array
 func extractChartMeta(chartData []byte) (*chart.Metadata, error) {
 	ch, err := loader.LoadArchive(bytes.NewReader(chartData))

internal/resolver/resolver.go

@@ -18,6 +18,7 @@ package resolver
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -39,15 +40,17 @@ const FeatureGateOCI = gates.Gate("HELM_EXPERIMENTAL_OCI")
 
 // Resolver resolves dependencies from semantic version ranges to a particular version.
 type Resolver struct {
-	chartpath string
-	cachepath string
+	chartpath      string
+	cachepath      string
+	registryClient *registry.Client
 }
 
-// New creates a new resolver for a given chart and a given helm home.
-func New(chartpath, cachepath string) *Resolver {
+// New creates a new resolver for a given chart, helm home and registry client.
+func New(chartpath, cachepath string, registryClient *registry.Client) *Resolver {
 	return &Resolver{
-		chartpath: chartpath,
-		cachepath: cachepath,
+		chartpath:      chartpath,
+		cachepath:      cachepath,
+		registryClient: registryClient,
 	}
 }
 
@@ -139,6 +142,24 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
 				return nil, errors.Wrapf(FeatureGateOCI.Error(),
 					"repository %s is an OCI registry", d.Repository)
 			}
+
+			// Retrieve list of tags for repository
+			ref := fmt.Sprintf("%s/%s", strings.TrimPrefix(d.Repository, fmt.Sprintf("%s://", registry.OCIScheme)), d.Name)
+			tags, err := r.registryClient.Tags(ref)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not retrieve list of tags for repository %s", d.Repository)
+			}
+
+			vs = make(repo.ChartVersions, len(tags))
+			for ti, t := range tags {
+				// Mock chart version objects
+				version := &repo.ChartVersion{
+					Metadata: &chart.Metadata{
+						Version: t,
+					},
+				}
+				vs[ti] = version
+			}
 		}
 
 		locked[i] = &chart.Dependency{
@@ -149,7 +170,8 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
 		// The version are already sorted and hence the first one to satisfy the constraint is used
 		for _, ver := range vs {
 			v, err := semver.NewVersion(ver.Version)
-			if err != nil || len(ver.URLs) == 0 {
+			// OCI does not need URLs
+			if err != nil || (!registry.IsOCI(d.Repository) && len(ver.URLs) == 0) {
 				// Not a legit entry.
 				continue
 			}

internal/resolver/resolver_test.go

@@ -19,6 +19,7 @@ import (
 	"runtime"
 	"testing"
 
+	"helm.sh/helm/v3/internal/experimental/registry"
 	"helm.sh/helm/v3/pkg/chart"
 )
 
@@ -139,7 +140,8 @@ func TestResolve(t *testing.T) {
 	}
 
 	repoNames := map[string]string{"alpine": "kubernetes-charts", "redis": "kubernetes-charts"}
-	r := New("testdata/chartpath", "testdata/repository")
+	registryClient, _ := registry.NewClient()
+	r := New("testdata/chartpath", "testdata/repository", registryClient)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			l, err := r.Resolve(tt.req, repoNames)

pkg/action/install.go

@@ -695,10 +695,9 @@ func (c *ChartPathOptions) LocateChart(name string, settings *cli.EnvSettings) (
 	}
 
 	if registry.IsOCI(name) {
-		if version == "" {
-			return "", errors.New("version is explicitly required for OCI registries")
+		if version != "" {
+			dl.Options = append(dl.Options, getter.WithTagName(version))
 		}
-		dl.Options = append(dl.Options, getter.WithTagName(version))
 	}
 
 	if c.Verify {

pkg/action/pull.go

@@ -87,18 +87,14 @@ func (p *Pull) Run(chartRef string) (string, error) {
 			getter.WithTLSClientConfig(p.CertFile, p.KeyFile, p.CaFile),
 			getter.WithInsecureSkipVerifyTLS(p.InsecureSkipTLSverify),
 		},
+		RegistryClient:   p.cfg.RegistryClient,
 		RepositoryConfig: p.Settings.RepositoryConfig,
 		RepositoryCache:  p.Settings.RepositoryCache,
 	}
 
 	if registry.IsOCI(chartRef) {
-		if p.Version == "" {
-			return out.String(), errors.Errorf("--version flag is explicitly required for OCI registries")
-		}
-
 		c.Options = append(c.Options,
-			getter.WithRegistryClient(p.cfg.RegistryClient),
-			getter.WithTagName(p.Version))
+			getter.WithRegistryClient(p.cfg.RegistryClient))
 	}
 
 	if p.Verify {