Merge pull request #23 from hypnoglow/reindex-command

Implement reindex command

README.md

@@ -110,7 +110,14 @@ The chart is deleted from the repo:
 ## Uninstall
 
     $ helm plugin remove s3
-
+
+## Documentation
+
+Additional documentation is available in the [docs](docs) directory. This currently includes:
+- estimated [usage cost calculation](docs/usage-cost.md)
+- [best practices](docs/best-practice.md)
+for organizing your repositories.
+
 ## Contributing
 
 Contributions are welcome. Please see [these instructions](.github/CONTRIBUTING.md)

cmd/helms3/delete.go

@@ -59,7 +59,7 @@ func runDelete(name, version, repoName string) error {
 	if err := storage.Delete(ctx, uri); err != nil {
 		return errors.WithMessage(err, "delete chart file from s3")
 	}
-	if _, err := storage.Upload(ctx, repoEntry.URL+"/index.yaml", idxReader); err != nil {
+	if err := storage.PutIndex(ctx, repoEntry.URL, idxReader); err != nil {
 		return errors.WithMessage(err, "upload new index to s3")
 	}
 

cmd/helms3/init.go

@@ -20,10 +20,9 @@ func runInit(uri string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
 	defer cancel()
 
-	if _, err := storage.Upload(ctx, uri+"/index.yaml", r); err != nil {
+	if err := storage.PutIndex(ctx, uri, r); err != nil {
 		return errors.WithMessage(err, "upload index to s3")
 	}
 
 	return nil
-
 }

cmd/helms3/main.go

@@ -17,6 +17,7 @@ const (
 	actionVersion = "version"
 	actionInit    = "init"
 	actionPush    = "push"
+	actionReindex = "reindex"
 	actionDelete  = "delete"
 
 	defaultTimeout = time.Second * 5
@@ -46,6 +47,11 @@ func main() {
 		Required().
 		String()
 
+	reindexCmd := cli.Command(actionReindex, "Reindex the repository.")
+	reindexTargetRepository := reindexCmd.Arg("repo", "Target repository to reindex").
+		Required().
+		String()
+
 	deleteCmd := cli.Command(actionDelete, "Delete chart from the repository.").Alias("del")
 	deleteChartName := deleteCmd.Arg("chartName", "Name of chart to delete").
 		Required().
@@ -81,6 +87,13 @@ func main() {
 		}
 		return
 
+	case actionReindex:
+		fmt.Fprint(os.Stderr, "Warning: reindex feature is in beta. If you experience any issues,\nplease provide your feedback here: https://github.com/hypnoglow/helm-s3/issues/22\n\n")
+		if err := runReindex(*reindexTargetRepository); err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("Repository %s was successfully reindexed.\n", *reindexTargetRepository)
+
 	case actionDelete:
 		if err := runDelete(*deleteChartName, *deleteChartVersion, *deleteTargetRepository); err != nil {
 			log.Fatal(err)

cmd/helms3/proxy.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"path"
+	"strings"
 
 	"github.com/pkg/errors"
 
@@ -18,6 +20,9 @@ func runProxy(uri string) error {
 
 	b, err := storage.FetchRaw(ctx, uri)
 	if err != nil {
+		if strings.HasSuffix(uri, "index.yaml") && err == awss3.ErrObjectNotFound {
+			return fmt.Errorf("The index file does not exist by the path %s. If you haven't initialized the repository yet, try running \"helm s3 init %s\"", uri, path.Dir(uri))
+		}
 		return errors.WithMessage(err, "fetch from s3")
 	}
 

cmd/helms3/push.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -52,25 +53,29 @@ func runPush(chartPath string, repoName string) error {
 		return err
 	}
 
+	hash, err := provenance.DigestFile(fname)
+	if err != nil {
+		return errors.WithMessage(err, "get chart digest")
+	}
+
 	fchart, err := os.Open(fname)
 	if err != nil {
 		return errors.Wrap(err, "open chart file")
 	}
 
-	if _, err := storage.Upload(ctx, repoEntry.URL+"/"+fname, fchart); err != nil {
+	serializedChartMeta, err := json.Marshal(chart.Metadata)
+	if err != nil {
+		return errors.Wrap(err, "encode chart metadata to json")
+	}
+
+	if _, err := storage.PutChart(ctx, repoEntry.URL+"/"+fname, fchart, string(serializedChartMeta), hash); err != nil {
 		return errors.WithMessage(err, "upload chart to s3")
 	}
 
-	// Next, update the repository index.
 	// The gap between index fetching and uploading should be as small as
 	// possible to make the best effort to avoid race conditions.
 	// See https://github.com/hypnoglow/helm-s3/issues/18 for more info.
 
-	hash, err := provenance.DigestFile(fname)
-	if err != nil {
-		return errors.WithMessage(err, "get chart digest")
-	}
-
 	// Fetch current index, update it and upload it back.
 
 	b, err := storage.FetchRaw(ctx, repoEntry.URL+"/index.yaml")
@@ -91,7 +96,7 @@ func runPush(chartPath string, repoName string) error {
 		return errors.WithMessage(err, "get index reader")
 	}
 
-	if _, err := storage.Upload(ctx, repoEntry.URL+"/index.yaml", idxReader); err != nil {
+	if err := storage.PutIndex(ctx, repoEntry.URL, idxReader); err != nil {
 		return errors.WithMessage(err, "upload index to s3")
 	}
 

cmd/helms3/reindex.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/hypnoglow/helm-s3/pkg/awss3"
+	"github.com/hypnoglow/helm-s3/pkg/helmutil"
+	"github.com/hypnoglow/helm-s3/pkg/index"
+)
+
+const (
+	reindexCommandDefaultTimeout = time.Second * 15
+)
+
+func runReindex(repoName string) error {
+	// Just one big timeout for the whole operation.
+	ctx, cancel := context.WithTimeout(context.Background(), reindexCommandDefaultTimeout)
+	defer cancel()
+
+	repoEntry, err := helmutil.LookupRepoEntry(repoName)
+	if err != nil {
+		return err
+	}
+
+	storage := awss3.New()
+
+	items, errs := storage.Traverse(ctx, repoEntry.URL)
+
+	builtIndex := make(chan *index.Index, 1)
+	go func() {
+		idx := index.New()
+		for item := range items {
+			idx.Add(item.Meta, item.Filename, repoEntry.URL, item.Hash)
+		}
+		idx.SortEntries()
+
+		builtIndex <- idx
+	}()
+
+	for err = range errs {
+		return errors.Wrap(err, "traverse the chart repository")
+	}
+
+	idx := <-builtIndex
+
+	r, err := idx.Reader()
+	if err != nil {
+		return errors.Wrap(err, "get index reader")
+	}
+
+	if err := storage.PutIndex(ctx, repoEntry.URL, r); err != nil {
+		return errors.Wrap(err, "upload index to the repository")
+	}
+
+	return nil
+}