Add a new freeBytes gc policy

api/services/control/control.proto

@@ -31,7 +31,10 @@ message PruneRequest {
 	repeated string filter = 1;
 	bool all = 2;
 	int64 keepDuration = 3 [(gogoproto.nullable) = true];
-	int64 keepBytes = 4 [(gogoproto.nullable) = true];
+
+	int64 minStorage = 5 [(gogoproto.nullable) = true];
+	int64 maxStorage = 4 [(gogoproto.nullable) = true];
+	int64 free = 6 [(gogoproto.nullable) = true];
 }
 
 message DiskUsageRequest {

api/types/worker.proto

@@ -20,8 +20,11 @@ message WorkerRecord {
 message GCPolicy {
 	bool all = 1;
 	int64 keepDuration = 2;
-	int64 keepBytes = 3;
 	repeated string filters = 4;
+
+	int64 minStorage = 5 [(gogoproto.nullable) = true];
+	int64 maxStorage = 3 [(gogoproto.nullable) = true];
+	int64 free = 6 [(gogoproto.nullable) = true];
 }
 
 message BuildkitVersion {

cache/manager.go

@@ -22,6 +22,7 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/util/bklog"
+	"github.com/moby/buildkit/util/disk"
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/moby/buildkit/util/progress"
 	digest "github.com/opencontainers/go-digest"
@@ -1040,7 +1041,7 @@ func (cm *cacheManager) pruneOnce(ctx context.Context, ch chan client.UsageInfo,
 	}
 
 	totalSize := int64(0)
-	if opt.KeepBytes != 0 {
+	if opt.MaxStorage != 0 {
 		du, err := cm.DiskUsage(ctx, client.DiskUsageInfo{})
 		if err != nil {
 			return err
@@ -1053,27 +1054,57 @@ func (cm *cacheManager) pruneOnce(ctx context.Context, ch chan client.UsageInfo,
 		}
 	}
 
+	// TODO: pick a better path here
+	dstat, err := disk.GetDiskStat(cm.mountPool.tmpdirRoot)
+	if err != nil {
+		return err
+	}
+
 	return cm.prune(ctx, ch, pruneOpt{
 		filter:       filter,
 		all:          opt.All,
 		checkShared:  check,
 		keepDuration: opt.KeepDuration,
-		keepBytes:    opt.KeepBytes,
+		keepBytes:    calculateKeepBytes(totalSize, dstat, opt),
 		totalSize:    totalSize,
 	})
 }
 
-func (cm *cacheManager) prune(ctx context.Context, ch chan client.UsageInfo, opt pruneOpt) (err error) {
-	var toDelete []*deleteRecord
+func calculateKeepBytes(totalSize int64, dstat disk.DiskStat, opt client.PruneInfo) int64 {
+	// 0 values are special, and means we have no keep cap
+	if opt.MaxStorage == 0 && opt.MinStorage == 0 && opt.Free == 0 {
+		return 0
+	}
+
+	// try and keep as many bytes as we can
+	keepBytes := opt.MaxStorage
 
+	// if we need to free up space, then decrease to that
+	if excess := opt.Free - dstat.Free; excess > 0 {
+		if keepBytes == 0 {
+			keepBytes = totalSize - excess
+		} else {
+			keepBytes = min(keepBytes, totalSize-excess)
+		}
+	}
+
+	// but make sure we don't take the total below the minimum
+	keepBytes = max(keepBytes, opt.MinStorage)
+
+	return keepBytes
+}
+
+func (cm *cacheManager) prune(ctx context.Context, ch chan client.UsageInfo, opt pruneOpt) (err error) {
 	if opt.keepBytes != 0 && opt.totalSize < opt.keepBytes {
 		return nil
 	}
 
+	var toDelete []*deleteRecord
+
 	cm.mu.Lock()
 
-	gcMode := opt.keepBytes != 0
 	cutOff := time.Now().Add(-opt.keepDuration)
+	gcMode := opt.keepBytes != 0
 
 	locked := map[*sync.Mutex]struct{}{}
 
@@ -1610,8 +1641,9 @@ type pruneOpt struct {
 	all          bool
 	checkShared  ExternalRefChecker
 	keepDuration time.Duration
-	keepBytes    int64
-	totalSize    int64
+
+	keepBytes int64
+	totalSize int64
 }
 
 type deleteRecord struct {