Add multiple strategies for querying GitHub release and assets info

.github/workflows/validations.yaml

@@ -24,9 +24,6 @@ jobs:
 
       - name: Bootstrap environment
         uses: ./.github/actions/bootstrap
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          FORCE_COLOR: true
 
       - name: Run all validations
         run: make pr-validations

README.md

@@ -201,6 +201,9 @@ The `github-release` version method uses the GitHub Releases API to determine th
 The `version.want` option allows a special entry:
 - `latest`: don't pin to a version, use the latest available
 
+Note: this approach will might require a GitHub API token to be set in the `GITHUB_TOKEN` environment variable if there
+is a version constraint used.
+
 #### `go-proxy`
 
 The `go-proxy` version method reaches out to `proxy.golang.org` to determine the latest version of a Go module. It requires the following configuration options:

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/gabriel-vasile/mimetype v1.4.2
 	github.com/gkampitakis/go-snaps v0.4.10
 	github.com/go-git/go-git/v5 v5.9.0
+	github.com/google/go-cmp v0.5.9
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/google/yamlfmt v0.9.1-0.20230607021126-908b19015fc4
 	github.com/hashicorp/go-multierror v1.1.1
@@ -31,6 +32,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651
 	github.com/wagoodman/go-progress v0.0.0-20230911172108-cf810b7e365c
+	golang.org/x/net v0.15.0
 	golang.org/x/oauth2 v0.8.0
 	golang.org/x/sync v0.3.0
 	golang.org/x/term v0.12.0
@@ -70,7 +72,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
 	github.com/google/uuid v1.3.1 // indirect
 	github.com/gookit/color v1.5.4 // indirect
@@ -132,7 +133,6 @@ require (
 	go.mongodb.org/mongo-driver v1.11.3 // indirect
 	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/tools v0.13.0 // indirect

internal/download_file.go

@@ -1,47 +1,101 @@
 package internal
 
 import (
+	"crypto/md5"  // nolint:gosec // MD5 is used for legacy compatibility
+	"crypto/sha1" // nolint:gosec // SHA1 is used for legacy compatibility
 	"crypto/sha256"
+	"crypto/sha512"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"strings"
 
-	"github.com/anchore/binny/internal/log"
+	"github.com/go-git/go-git/v5/plumbing/hash"
+
+	"github.com/anchore/go-logger"
 )
 
-func DownloadFile(url string, filepath string, checksum string) (err error) {
-	out, err := os.Create(filepath)
+func DownloadFile(lgr logger.Logger, url string, filepath string, checksum string) (err error) {
+	reader, err := DownloadURL(lgr, url)
 	if err != nil {
 		return err
 	}
-	defer out.Close()
+	defer reader.Close()
 
-	resp, err := http.Get(url) // nolint:gosec
+	out, err := os.Create(filepath)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("bad status: %s", resp.Status)
-	}
+	defer out.Close()
 
-	// take sha256 of file and compare with checksum while copying to disk
-	h := sha256.New()
-	tee := io.TeeReader(resp.Body, h)
+	// hash the file and compare with checksum while copying to disk
+	h := getHasher(checksum)
+	tee := io.TeeReader(reader, h)
 
 	if _, err := io.Copy(out, tee); err != nil {
 		return err
 	}
 
 	if checksum != "" {
-		if checksum != fmt.Sprintf("%x", h.Sum(nil)) {
+		expectedChecksum := cleanChecksum(checksum)
+		actualChecksum := fmt.Sprintf("%x", h.Sum(nil))
+
+		if expectedChecksum != actualChecksum {
+			lgr.WithFields("url", url, "expected", expectedChecksum, "actual", actualChecksum).Warn("checksum mismatch")
 			return fmt.Errorf("checksum mismatch for %q", filepath)
 		}
 
-		log.WithFields("checksum", checksum, "asset", filepath, "url", url).Trace("checksum verified")
+		lgr.WithFields("checksum", expectedChecksum, "asset", filepath, "url", url).Trace("checksum verified")
 	}
 
 	return nil
 }
+
+func DownloadURL(lgr logger.Logger, url string) (io.ReadCloser, error) {
+	resp, err := http.Get(url) // nolint: gosec  // we must be able to get arbitrary URLs
+	if err != nil {
+		return nil, fmt.Errorf("unable to download %q: %w", url, err)
+	}
+
+	lgr.WithFields("http-status", resp.StatusCode).Tracef("http get %q", url)
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, nil
+	}
+	return resp.Body, nil
+}
+
+func cleanChecksum(checksum string) string {
+	parts := strings.SplitN(checksum, ":", 2)
+	if len(parts) < 2 {
+		return checksum
+	}
+
+	return parts[1]
+}
+
+func getHasher(checksum string) hash.Hash {
+	// Default to SHA-256 if no prefix or unsupported prefix
+	defaultHash := sha256.New()
+
+	parts := strings.SplitN(checksum, ":", 2)
+	if len(parts) < 2 {
+		return defaultHash
+	}
+
+	algorithm := strings.ToLower(parts[0])
+
+	switch algorithm {
+	case "sha256":
+		return sha256.New()
+	case "sha1":
+		return sha1.New() // nolint:gosec // SHA1 is used for legacy compatibility
+	case "sha512":
+		return sha512.New()
+	case "md5":
+		return md5.New() // nolint:gosec // MD5 is used for legacy compatibility
+	default:
+		return defaultHash
+	}
+}

internal/download_file_test.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/go-logger/adapter/discard"
 )
 
 func Test_DownloadFile(t *testing.T) {
@@ -49,7 +51,7 @@ func Test_DownloadFile(t *testing.T) {
 			dir := t.TempDir()
 			dlPath := filepath.Join(dir, "the-file-path.txt")
 
-			tt.wantErr(t, DownloadFile(s.URL, dlPath, tt.checksum))
+			tt.wantErr(t, DownloadFile(discard.New(), s.URL, dlPath, tt.checksum))
 
 			gotContents, err := os.ReadFile(dlPath)
 			require.NoError(t, err)

store.go

@@ -257,7 +257,7 @@ func (s Store) saveState() error {
 func (e *StoreEntry) Verify(useXxh64, useSha256 bool) error {
 	// at least the file must exist
 	if _, err := os.Stat(e.Path()); err != nil {
-		return fmt.Errorf("failed to validate tool %q: %w", e.Name, err)
+		return err
 	}
 
 	if useXxh64 {

tool/githubrelease/gh_release.go

@@ -0,0 +1,48 @@
+package githubrelease
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+type ghRelease struct {
+	Tag      string
+	Date     *time.Time
+	IsLatest *bool
+	IsDraft  *bool
+	Assets   []ghAsset
+}
+
+type ghAsset struct {
+	Name        string
+	ContentType string
+	URL         string
+	Checksum    string
+}
+
+func (a *ghAsset) addChecksum(value string) {
+	if strings.Contains(value, ":") {
+		a.Checksum = value
+		return
+	}
+
+	// note: assume this is a hex digest
+	var method string
+	switch len(value) {
+	case 32:
+		method = "md5"
+	case 40:
+		method = "sha1"
+	case 64:
+		method = "sha256"
+	case 128:
+		method = "sha512"
+	default:
+		// dunno, just capture the value
+		a.Checksum = value
+		return
+	}
+
+	a.Checksum = fmt.Sprintf("%s:%s", method, value)
+}