Avoid download target index.yaml several times (#35)

* Avoid unnecessaty index.yaml downloads

cmd/sync.go

@@ -55,11 +55,14 @@ func sync() error {
 	if err != nil {
 		return errors.Trace(err)
 	}
-	// Load index.yaml info into index object
 	sourceIndex, err := utils.LoadIndexFromRepo(source.Repo)
 	if err != nil {
 		return errors.Trace(fmt.Errorf("error loading index.yaml: %w", err))
 	}
+	targetIndex, err := utils.LoadIndexFromRepo(target.Repo)
+	if err != nil {
+		return errors.Trace(fmt.Errorf("error loading index.yaml: %w", err))
+	}
 	// Add target repo to helm CLI
 	helmcli.AddRepoToHelm(target.Repo.Url, target.Repo.Auth)
 	// Create client for target repo
@@ -76,15 +79,15 @@ func sync() error {
 			if publishingTime.Before(dateThreshold) {
 				continue
 			}
-			if chartExists, _ := tc.ChartExists(chartName, chartVersion, target.Repo); chartExists {
+			if chartExists, _ := tc.ChartExists(chartName, chartVersion, targetIndex); chartExists {
 				continue
 			}
 			if dryRun {
 				klog.Infof("dry-run: Chart %s-%s pending to be synced", chartName, chartVersion)
 				continue
 			}
 			klog.Infof("Syncing %s-%s", chartName, chartVersion)
-			if err := chart.Sync(chartName, chartVersion, source.Repo, target, sourceIndex, true); err != nil {
+			if err := chart.Sync(chartName, chartVersion, source.Repo, target, sourceIndex, targetIndex, true); err != nil {
 				errs = multierror.Append(errs, errors.Trace(err))
 			}
 		}

cmd/sync_chart.go

@@ -69,11 +69,14 @@ func syncChart() error {
 	source := syncConfig.Source
 	target := syncConfig.Target
 
-	// Load index.yaml info into index object
 	sourceIndex, err := utils.LoadIndexFromRepo(source.Repo)
 	if err != nil {
 		return errors.Trace(fmt.Errorf("error loading index.yaml: %w", err))
 	}
+	targetIndex, err := utils.LoadIndexFromRepo(target.Repo)
+	if err != nil {
+		return errors.Trace(fmt.Errorf("error loading index.yaml: %w", err))
+	}
 
 	// Add target repo to helm CLI
 	helmcli.AddRepoToHelm(target.Repo.Url, target.Repo.Auth)
@@ -84,7 +87,7 @@ func syncChart() error {
 	}
 
 	if syncAllVersions {
-		if err := chart.SyncAllVersions(name, source.Repo, target, syncDeps, sourceIndex, dryRun); err != nil {
+		if err := chart.SyncAllVersions(name, source.Repo, target, syncDeps, sourceIndex, targetIndex, dryRun); err != nil {
 			return errors.Trace(err)
 		}
 	} else {
@@ -95,7 +98,7 @@ func syncChart() error {
 		if !srcExists {
 			return errors.Errorf("chart %s-%s not found in source index.yaml", name, version)
 		}
-		targetExists, err := tc.ChartExists(name, version, target.Repo)
+		targetExists, err := tc.ChartExists(name, version, targetIndex)
 		if err != nil {
 			return errors.Trace(err)
 		}
@@ -107,7 +110,7 @@ func syncChart() error {
 			klog.Infof("dry-run: Chart %s-%s pending to be synced", name, version)
 			return nil
 		}
-		if err := chart.Sync(name, version, source.Repo, target, sourceIndex, syncDeps); err != nil {
+		if err := chart.Sync(name, version, source.Repo, target, sourceIndex, targetIndex, syncDeps); err != nil {
 			return errors.Trace(err)
 		}
 	}

pkg/chart/dependency.go

@@ -22,7 +22,7 @@ type dependencies struct {
 }
 
 // syncDependencies takes care of updating dependencies to correct version and sync to target repo if necesary.
-func syncDependencies(chartPath string, sourceRepo *api.Repo, target *api.TargetRepo, sourceIndex *helmRepo.IndexFile, syncDeps bool) error {
+func syncDependencies(chartPath string, sourceRepo *api.Repo, target *api.TargetRepo, sourceIndex *helmRepo.IndexFile, targetIndex *helmRepo.IndexFile, syncDeps bool) error {
 	klog.V(3).Info("Chart has dependencies...")
 	var errs error
 	var missingDependencies = false
@@ -53,7 +53,7 @@ func syncDependencies(chartPath string, sourceRepo *api.Repo, target *api.Target
 		if depRepository != sourceRepo.Url {
 			continue
 		}
-		if chartExists, _ := tc.ChartExists(depName, depVersion, target.Repo); chartExists {
+		if chartExists, _ := tc.ChartExists(depName, depVersion, targetIndex); chartExists {
 			klog.V(3).Infof("Dependency %s-%s already synced", depName, depVersion)
 			continue
 		}
@@ -63,10 +63,10 @@ func syncDependencies(chartPath string, sourceRepo *api.Repo, target *api.Target
 			continue
 		}
 		klog.Infof("Dependency %s-%s not synced yet. Syncing now", depName, depVersion)
-		if err := Sync(depName, depVersion, sourceRepo, target, sourceIndex, true); err != nil {
+		if err := Sync(depName, depVersion, sourceRepo, target, sourceIndex, targetIndex, true); err != nil {
 			return errors.Trace(err)
 		}
-		chartExists, err := tc.ChartExists(depName, depVersion, target.Repo)
+		chartExists, err := tc.ChartExists(depName, depVersion, targetIndex)
 		if err != nil {
 			return errors.Trace(err)
 		}

pkg/chart/dependency_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/bitnami-labs/charts-syncer/pkg/utils"
 	"gopkg.in/yaml.v2"
 	helmChart "helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/repo"
 )
 
 func TestSyncDependencies(t *testing.T) {
@@ -29,9 +30,10 @@ func TestSyncDependencies(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error loading index.yaml: %v", err)
 	}
+	targetIndex := repo.NewIndexFile()
 
 	chartPath := path.Join(testTmpDir, "kafka")
-	err = syncDependencies(chartPath, source.Repo, target, sourceIndex, false)
+	err = syncDependencies(chartPath, source.Repo, target, sourceIndex, targetIndex, false)
 	expectedError := "please sync zookeeper-5.14.3 dependency first"
 	if err != nil && err.Error() != expectedError {
 		t.Errorf("incorrect error, got: \n %s \n, want: \n %s \n", err.Error(), expectedError)

pkg/chart/sync.go

@@ -15,24 +15,25 @@ import (
 	"github.com/bitnami-labs/charts-syncer/pkg/repo"
 	"github.com/bitnami-labs/charts-syncer/pkg/utils"
 
+	"helm.sh/helm/v3/pkg/chart"
 	helmRepo "helm.sh/helm/v3/pkg/repo"
 )
 
 // SyncAllVersions will sync all versions of a specific chart.
-func SyncAllVersions(name string, sourceRepo *api.Repo, target *api.TargetRepo, syncDeps bool, index *helmRepo.IndexFile, dryRun bool) error {
+func SyncAllVersions(name string, sourceRepo *api.Repo, target *api.TargetRepo, syncDeps bool, sourceIndex *helmRepo.IndexFile, targetIndex *helmRepo.IndexFile, dryRun bool) error {
 	var errs error
 	// Create client for target repo
 	tc, err := repo.NewClient(target.Repo)
 	if err != nil {
 		return fmt.Errorf("could not create a client for the source repo: %w", err)
 	}
-	if index.Entries[name] != nil {
-		for i := range index.Entries[name] {
-			if chartExists, err := tc.ChartExists(name, index.Entries[name][i].Metadata.Version, target.Repo); !chartExists && err == nil {
+	if sourceIndex.Entries[name] != nil {
+		for i := range sourceIndex.Entries[name] {
+			if chartExists, err := tc.ChartExists(name, sourceIndex.Entries[name][i].Metadata.Version, targetIndex); !chartExists && err == nil {
 				if dryRun {
-					klog.Infof("dry-run: Chart %s-%s pending to be synced", name, index.Entries[name][i].Metadata.Version)
+					klog.Infof("dry-run: Chart %s-%s pending to be synced", name, sourceIndex.Entries[name][i].Metadata.Version)
 				} else {
-					if err := Sync(name, index.Entries[name][i].Metadata.Version, sourceRepo, target, index, syncDeps); err != nil {
+					if err := Sync(name, sourceIndex.Entries[name][i].Metadata.Version, sourceRepo, target, sourceIndex, targetIndex, syncDeps); err != nil {
 						errs = multierror.Append(errs, errors.Trace(err))
 					}
 				}
@@ -45,7 +46,7 @@ func SyncAllVersions(name string, sourceRepo *api.Repo, target *api.TargetRepo,
 }
 
 // Sync is the main function. It downloads, transform, package and publish a chart.
-func Sync(name string, version string, sourceRepo *api.Repo, target *api.TargetRepo, sourceIndex *helmRepo.IndexFile, syncDeps bool) error {
+func Sync(name string, version string, sourceRepo *api.Repo, target *api.TargetRepo, sourceIndex *helmRepo.IndexFile, targetIndex *helmRepo.IndexFile, syncDeps bool) error {
 	// Create temporary working directory
 	tmpDir, err := ioutil.TempDir("", "charts-syncer")
 	if err != nil {
@@ -80,7 +81,7 @@ func Sync(name string, version string, sourceRepo *api.Repo, target *api.TargetR
 	// If chart has dependencies, check that they are already in the target repo.
 	chartPath := path.Join(destDir, name)
 	if _, err := os.Stat(path.Join(chartPath, "requirements.lock")); err == nil {
-		if err := syncDependencies(chartPath, sourceRepo, target, sourceIndex, syncDeps); err != nil {
+		if err := syncDependencies(chartPath, sourceRepo, target, sourceIndex, targetIndex, syncDeps); err != nil {
 			return errors.Annotatef(err, "Error updating dependencies for chart %s-%s", name, version)
 		}
 	}
@@ -122,6 +123,9 @@ func Sync(name string, version string, sourceRepo *api.Repo, target *api.TargetR
 	if err := tc.PublishChart(packagedChartPath, target.Repo); err != nil {
 		return errors.Annotatef(err, "Error publishing chart %s-%s to target repo", name, version)
 	}
+	// Add just synced chart to our local target index so other charts that may have this as dependency
+	// know it is already synced in the target repository.
+	targetIndex.Add(&chart.Metadata{Name: name, Version: version}, "", "", "")
 	klog.Infof("Chart %s-%s published successfully", name, version)
 
 	return errors.Trace(err)

pkg/chart/sync_test.go

@@ -31,10 +31,11 @@ func TestSync(t *testing.T) {
 			if err != nil {
 				t.Fatalf("error loading index.yaml: %v", err)
 			}
+			targetIndex := helmRepo.NewIndexFile()
 
 			name := "zookeeper"
 			version := "5.11.0"
-			if err := Sync(name, version, source.Repo, target, sourceIndex, false); err != nil {
+			if err := Sync(name, version, source.Repo, target, sourceIndex, targetIndex, false); err != nil {
 				t.Fatal(err)
 			}
 
@@ -76,13 +77,17 @@ func TestSync(t *testing.T) {
 					t.Fatalf("error creating temporary: %s", testTmpDir)
 				}
 				defer os.RemoveAll(testTmpDir)
+				targetIndex, err := utils.LoadIndexFromRepo(target.Repo)
+				if err != nil {
+					t.Fatalf("error loading index.yaml: %v", err)
+				}
 				// Create client for target repo
 				tc, err := repo.NewClient(target.Repo)
 				if err != nil {
 					t.Fatal("could not create a client for the source repo", err)
 				}
 				chartPath := path.Join(testTmpDir, "zookeeper-5.11.0.tgz")
-				if err := tc.DownloadChart(chartPath, "zookeeper", "5.11.0", target.Repo, sourceIndex); err != nil {
+				if err := tc.DownloadChart(chartPath, "zookeeper", "5.11.0", target.Repo, targetIndex); err != nil {
 					t.Fatal(err)
 				}
 				if err := utils.Untar(chartPath, testTmpDir); err != nil {
@@ -122,11 +127,12 @@ func TestSyncAllVersions(t *testing.T) {
 			name := "zookeeper"
 			indexFile := "../../testdata/zookeeper-index.yaml"
 			// Load index.yaml info into index object
-			index, err := helmRepo.LoadIndexFile(indexFile)
+			sourceIndex, err := helmRepo.LoadIndexFile(indexFile)
 			if err != nil {
 				t.Fatal(err)
 			}
-			if err := SyncAllVersions(name, source.Repo, target, false, index, false); err != nil {
+			targetIndex := helmRepo.NewIndexFile()
+			if err := SyncAllVersions(name, source.Repo, target, false, sourceIndex, targetIndex, false); err != nil {
 				t.Error(err)
 			}
 		})

pkg/chartrepotest/chartmuseumreal.go

@@ -45,7 +45,7 @@ func tChartMuseumReal(t *testing.T, username, password string) (string, func())
 		id,
 	))
 
-	return "http://localhost:" + port, func() {
+	return "http://127.0.0.1:" + port, func() {
 		tDockerExec(t, "stop", id)
 		tDockerExec(t, "rm", id)
 		tDockerVolumeRm(t, chartsVolume)

pkg/repo/chartmuseum.go

@@ -1,8 +1,6 @@
 package repo
 
 import (
-	"fmt"
-
 	"github.com/bitnami-labs/charts-syncer/api"
 	"github.com/bitnami-labs/charts-syncer/pkg/utils"
 	"github.com/juju/errors"
@@ -32,8 +30,8 @@ func (c *ChartMuseumClient) PublishChart(filepath string, targetRepo *api.Repo)
 
 // DownloadChart downloads a packaged chart from ChartsMuseum repository.
 func (c *ChartMuseumClient) DownloadChart(filepath string, name string, version string, sourceRepo *api.Repo, index *helmRepo.IndexFile) error {
-	klog.V(3).Infof("Downloading %s-%s from Harbor repo", name, version)
-	apiEndpoint, err := utils.FindChartURL(name, version, index)
+	klog.V(3).Infof("Downloading %s-%s from Chartmuseum repo", name, version)
+	apiEndpoint, err := utils.FindChartURL(name, version, index, sourceRepo.Url)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -44,12 +42,8 @@ func (c *ChartMuseumClient) DownloadChart(filepath string, name string, version
 }
 
 // ChartExists checks if a chart exists in the repo.
-func (c *ChartMuseumClient) ChartExists(name string, version string, repo *api.Repo) (bool, error) {
-	klog.V(3).Infof("Checking if %s-%s chart exists in %q", name, version, repo.Url)
-	index, err := utils.LoadIndexFromRepo(repo)
-	if err != nil {
-		return false, errors.Trace(fmt.Errorf("error loading index.yaml: %w", err))
-	}
+func (c *ChartMuseumClient) ChartExists(name string, version string, index *helmRepo.IndexFile) (bool, error) {
+	klog.V(3).Infof("Checking if %s-%s chart exists", name, version)
 	chartExists, err := utils.ChartExistInIndex(name, version, index)
 	if err != nil {
 		return false, errors.Trace(err)

pkg/repo/chartmuseum_test.go

@@ -139,16 +139,14 @@ func TestDownloadFromChartmuseum(t *testing.T) {
 }
 
 func TestChartExistsInChartMuseum(t *testing.T) {
-	// Update source repo url
-	// This repo is not a chartmuseum repo but there are no differences
-	// for the ChartExists function.
-	sourceCM.Repo.Url = "https://charts.bitnami.com/bitnami"
+	sourceIndex := repo.NewIndexFile()
+	sourceIndex.Add(&chart.Metadata{Name: "grafana", Version: "1.5.2"}, "grafana-1.5.2.tgz", "https://fake-url.com/charts", "sha256:1234567890")
 	// Create client for source repo
 	sc, err := NewClient(sourceCM.Repo)
 	if err != nil {
 		t.Fatal("could not create a client for the source repo", err)
 	}
-	chartExists, err := sc.ChartExists("grafana", "1.5.2", sourceCM.Repo)
+	chartExists, err := sc.ChartExists("grafana", "1.5.2", sourceIndex)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/repo/client.go

@@ -11,7 +11,7 @@ import (
 type ChartRepoAPI interface {
 	DownloadChart(filepath string, name string, version string, sourceRepo *api.Repo, index *helmRepo.IndexFile) error
 	PublishChart(filepath string, targetRepo *api.Repo) error
-	ChartExists(name string, version string, targetRepo *api.Repo) (bool, error)
+	ChartExists(name string, version string, index *helmRepo.IndexFile) (bool, error)
 }
 
 // NewClient returns a client implementation for the given repo.

pkg/repo/harbor.go

@@ -1,7 +1,6 @@
 package repo
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/bitnami-labs/charts-syncer/api"
@@ -34,7 +33,7 @@ func (c *HarborClient) PublishChart(filepath string, targetRepo *api.Repo) error
 // DownloadChart downloads a packaged chart from Harbor repository.
 func (c *HarborClient) DownloadChart(filepath string, name string, version string, sourceRepo *api.Repo, index *helmRepo.IndexFile) error {
 	klog.V(3).Infof("Downloading %s-%s from Harbor repo", name, version)
-	apiEndpoint, err := utils.FindChartURL(name, version, index)
+	apiEndpoint, err := utils.FindChartURL(name, version, index, sourceRepo.Url)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -45,12 +44,8 @@ func (c *HarborClient) DownloadChart(filepath string, name string, version strin
 }
 
 // ChartExists checks if a chart exists in the repo.
-func (c *HarborClient) ChartExists(name string, version string, repo *api.Repo) (bool, error) {
-	klog.V(3).Infof("Checking if %s-%s chart exists in %q", name, version, repo.Url)
-	index, err := utils.LoadIndexFromRepo(repo)
-	if err != nil {
-		return false, errors.Trace(fmt.Errorf("error loading index.yaml: %w", err))
-	}
+func (c *HarborClient) ChartExists(name string, version string, index *helmRepo.IndexFile) (bool, error) {
+	klog.V(3).Infof("Checking if %s-%s chart exists", name, version)
 	chartExists, err := utils.ChartExistInIndex(name, version, index)
 	if err != nil {
 		return false, errors.Trace(err)

pkg/repo/harbor_test.go

@@ -136,16 +136,14 @@ func TestDownloadFromHarbor(t *testing.T) {
 }
 
 func TestChartExistsInHarbor(t *testing.T) {
-	// Update source repo url
-	// This repo is not a chartmuseum repo but there are no differences
-	// for the ChartExists function.
-	sourceHarbor.Repo.Url = "https://charts.bitnami.com/bitnami"
+	sourceIndex := repo.NewIndexFile()
+	sourceIndex.Add(&chart.Metadata{Name: "grafana", Version: "1.5.2"}, "grafana-1.5.2.tgz", "https://fake-url.com/charts", "sha256:1234567890")
 	// Create client for source repo
 	sc, err := NewClient(sourceHarbor.Repo)
 	if err != nil {
 		t.Fatal("could not create a client for the source repo", err)
 	}
-	chartExists, err := sc.ChartExists("grafana", "1.5.2", sourceHarbor.Repo)
+	chartExists, err := sc.ChartExists("grafana", "1.5.2", sourceIndex)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/repo/helmclassic.go

@@ -1,8 +1,6 @@
 package repo
 
 import (
-	"fmt"
-
 	"github.com/bitnami-labs/charts-syncer/api"
 	"github.com/bitnami-labs/charts-syncer/pkg/utils"
 	"github.com/juju/errors"
@@ -29,7 +27,7 @@ func (c *ClassicHelmClient) PublishChart(filepath string, targetRepo *api.Repo)
 // DownloadChart downloads a packaged chart from a classic helm repository.
 func (c *ClassicHelmClient) DownloadChart(filepath string, name string, version string, sourceRepo *api.Repo, index *helmRepo.IndexFile) error {
 	klog.V(3).Infof("Downloading %s-%s from classic helm repo", name, version)
-	downloadURL, err := utils.FindChartURL(name, version, index)
+	downloadURL, err := utils.FindChartURL(name, version, index, sourceRepo.Url)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -48,12 +46,8 @@ func (c *ClassicHelmClient) DownloadChart(filepath string, name string, version
 }
 
 // ChartExists checks if a chart exists in the repo.
-func (c *ClassicHelmClient) ChartExists(name string, version string, repo *api.Repo) (bool, error) {
-	klog.V(3).Infof("Checking if %s-%s chart exists in %q", name, version, repo.Url)
-	index, err := utils.LoadIndexFromRepo(repo)
-	if err != nil {
-		return false, errors.Trace(fmt.Errorf("error loading index.yaml: %w", err))
-	}
+func (c *ClassicHelmClient) ChartExists(name string, version string, index *helmRepo.IndexFile) (bool, error) {
+	klog.V(3).Infof("Checking if %s-%s chart exists", name, version)
 	chartExists, err := utils.ChartExistInIndex(name, version, index)
 	if err != nil {
 		return false, errors.Trace(err)