Adding caching/mirroring

.github/workflows/verify.yml

@@ -17,10 +17,10 @@ jobs:
         with:
           go-version-file: 'go.mod'
       - name: Check license headers
-        run: go run github.com/opentofu/tofudl/tools/license-headers -check-only
+        run: go run github.com/opentofu/tofudl/internal/tools/license-headers -check-only
       - name: Check GPG key
         working-directory: branding
-        run: go run github.com/opentofu/tofudl/tools/bundle-gpg-key -check-only
+        run: go run github.com/opentofu/tofudl/internal/tools/bundle-gpg-key -check-only
   lint:
     name: Lint
     runs-on: ubuntu-latest
@@ -32,7 +32,7 @@ jobs:
         with:
           go-version-file: 'go.mod'
       - name: Lint
-        run: go run github.com/opentofu/tofudl/tools/lint
+        run: go run github.com/opentofu/tofudl/internal/tools/lint
   tests:
     name: Tests
     strategy:

README.md

@@ -12,45 +12,117 @@ The downloader will work without any extra configuration out of the box:
 package main
 
 import (
-	"context"
-	"os"
-	"os/exec"
-	"runtime"
+    "context"
+    "os"
+    "os/exec"
+    "runtime"
 
-	"github.com/opentofu/tofudl"
+    "github.com/opentofu/tofudl"
 )
 
 func main() {
-	// Initialize the downloader:
-	dl, err := tofudl.New()
-	if err != nil {
-		panic(err)
-	}
-
-	// Download the latest stable version
-	// for the current architecture and platform:
-	binary, err := dl.Download(context.TODO())
-	if err != nil {
-		panic(err)
-	}
-
-	// Write out the tofu binary to the disk:
-	file := "tofu"
-	if runtime.GOOS == "windows" {
-		file += ".exe"
-	}
-	if err := os.WriteFile(file, binary, 0755); err != nil {
-		panic(err)
-	}
-
-	// Run tofu:
-	cmd := exec.Command("./"+file, "init")
-	if err := cmd.Run(); err != nil {
-		panic(err)
-	}
+    // Initialize the downloader:
+    dl, err := tofudl.New()
+    if err != nil {
+        panic(err)
+    }
+
+    // Download the latest stable version
+    // for the current architecture and platform:
+    binary, err := dl.Download(context.TODO())
+    if err != nil {
+        panic(err)
+    }
+
+    // Write out the tofu binary to the disk:
+    file := "tofu"
+    if runtime.GOOS == "windows" {
+        file += ".exe"
+    }
+    if err := os.WriteFile(file, binary, 0755); err != nil {
+        panic(err)
+    }
+
+    // Run tofu:
+    cmd := exec.Command("./"+file, "init")
+    if err := cmd.Run(); err != nil {
+        panic(err)
+    }
 }
 ```
 
+## Caching
+
+This library also supports caching using the mirror tool:
+
+```go
+package main
+
+import (
+    "context"
+    "os"
+    "os/exec"
+    "runtime"
+    "time"
+
+    "github.com/opentofu/tofudl"
+)
+
+func main() {
+    // Initialize the downloader:
+    dl, err := tofudl.New()
+    if err != nil {
+        panic(err)
+    }
+
+    // Set up the caching layer:
+    storage, err := tofudl.NewFilesystemStorage("/tmp")
+    if err != nil {
+        panic(err)
+    }
+    mirror, err := tofudl.NewMirror(
+        tofudl.MirrorConfig{
+            AllowStale: false,
+            APICacheTimeout: time.Minute * 10,
+            ArtifactCacheTimeout: time.Hour * 24,
+        },
+        storage,
+        dl,
+    )
+    if err != nil {
+        panic(err)
+    }
+
+    // Download the latest stable version
+    // for the current architecture and platform:
+    binary, err := mirror.Download(context.TODO())
+    if err != nil {
+        panic(err)
+    }
+
+    // Write out the tofu binary to the disk:
+    file := "tofu"
+    if runtime.GOOS == "windows" {
+        file += ".exe"
+    }
+    if err := os.WriteFile(file, binary, 0755); err != nil {
+        panic(err)
+    }
+
+    // Run tofu:
+    cmd := exec.Command("./"+file, "init")
+    if err := cmd.Run(); err != nil {
+        panic(err)
+    }
+}
+```
+
+You can also use the `mirror` variable as an `http.Handler`. Additionally, you can also call `PreWarm` on the caching layer in order to pre-warm your local caches. (Be careful, this may take a long time!)
+
+## Standalone mirror
+
+The example above showed a cache/mirror that acts as a pull-through cache to upstream. You can alternatively also use the mirror as a stand-alone mirror and publish your own binaries. The mirror has functions to facilitate uploading basic artifacts, but you can also use the `ReleaseBuilder` to make building releases easier. (Note: the `ReleaseBuilder` only builds artifacts needed for TofuDL, not all artifacts OpenTofu typically publishes.)
+
 ## Advanced usage
 
 Both `New()` and `Download()` accept a number of options. You can find the detailed documentation [here](https://pkg.go.dev/github.com/opentofu/tofudl).

branding/branding.go

@@ -16,6 +16,9 @@ const DefaultMirrorURLTemplate = "https://github.com/opentofu/opentofu/releases/
 // BinaryName holds the name of the binary in the artifact. This may be suffixed .exe on Windows.
 const BinaryName = "tofu"
 
+// ArtifactPrefix is the prefix for the artifact names.
+const ArtifactPrefix = "tofu_"
+
 // GPGKeyURL describes the URL to download the bundled GPG key from. The GPG key bundler uses this to download the
 // GPG key for verification.
 const GPGKeyURL = "https://get.opentofu.org/opentofu.asc"

branding/branding_nonwindows.go

@@ -0,0 +1,9 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+
+//go:build !windows
+
+package branding
+
+// PlatformBinaryName is the platform-dependent binary name for the current platform.
+const PlatformBinaryName = BinaryName

branding/branding_windows.go

@@ -0,0 +1,9 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+
+//go:build windows
+
+package branding
+
+// PlatformBinaryName is the platform-dependent binary name for the current platform.
+const PlatformBinaryName = BinaryName + ".exe"

cli/options.go

@@ -143,11 +143,11 @@ var optionStability = option{
 	description:        "Minimum stability to download for. Possible values are: " + getStabilityValues() + "",
 	defaultDescription: "stable",
 	applyDownloadOption: func(value string) (tofudl.DownloadOpt, error) {
-		platform := tofudl.Platform(value)
-		if err := platform.Validate(); err != nil {
+		stability := tofudl.Stability(value)
+		if err := stability.Validate(); err != nil {
 			return nil, err
 		}
-		return tofudl.DownloadOptPlatform(platform), nil
+		return tofudl.DownloadOptMinimumStability(stability), nil
 	},
 }
 

downloader.go

@@ -15,6 +15,12 @@ type Downloader interface {
 	// ListVersions lists all versions matching the filter options in descending order.
 	ListVersions(ctx context.Context, opts ...ListVersionOpt) ([]VersionWithArtifacts, error)
 
+	// DownloadArtifact downloads an artifact for a version.
+	DownloadArtifact(ctx context.Context, version VersionWithArtifacts, artifactName string) ([]byte, error)
+
+	// VerifyArtifact verifies a named artifact against a checksum file with SHA256 hashes and the checksum file against a GPG signature file.
+	VerifyArtifact(artifactName string, artifactContents []byte, sumsFileContents []byte, signatureFileContent []byte) error
+
 	// DownloadVersion downloads the OpenTofu binary from a specific artifact obtained from ListVersions.
 	DownloadVersion(ctx context.Context, version VersionWithArtifacts, platform Platform, architecture Architecture) ([]byte, error)