cli,storage: add emergency ballast

build/teamcity-support.sh

@@ -102,6 +102,12 @@ function run_json_test() {
     rm -f "${fullfile}"
   fi
   rm -f "${tmpfile}" artifacts/stripped.txt
+
+  # Some unit tests test automatic ballast creation. These ballasts can be
+  # larger than the maximum artifact size. Remove any artifacts with the
+  # EMERGENCY_BALLAST filename.
+  find artifacts -name "EMERGENCY_BALLAST" -delete
+
   tc_end_block "artifacts"
 
   # Make it easier to figure out whether we're exiting because of a test failure

pkg/base/config.go

@@ -510,6 +510,9 @@ type StorageConfig struct {
 	// MaxSize is used for calculating free space and making rebalancing
 	// decisions. Zero indicates that there is no maximum size.
 	MaxSize int64
+	// BallastSize is the amount reserved by a ballast file for manual
+	// out-of-disk recovery.
+	BallastSize int64
 	// Settings instance for cluster-wide knobs.
 	Settings *cluster.Settings
 	// UseFileRegistry is true if the file registry is needed (eg: encryption-at-rest).

pkg/base/store_spec.go

@@ -29,6 +29,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/errors/oserror"
 	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/redact"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/spf13/pflag"
 )
@@ -79,7 +80,7 @@ type floatInterval struct {
 // NewSizeSpec parses the string passed into a --size flag and returns a
 // SizeSpec if it is correctly parsed.
 func NewSizeSpec(
-	value string, bytesRange *intInterval, percentRange *floatInterval,
+	field redact.SafeString, value string, bytesRange *intInterval, percentRange *floatInterval,
 ) (SizeSpec, error) {
 	var size SizeSpec
 	if fractionRegex.MatchString(value) {
@@ -93,13 +94,14 @@ func NewSizeSpec(
 		size.Percent, err = strconv.ParseFloat(factorValue, 64)
 		size.Percent *= percentFactor
 		if err != nil {
-			return SizeSpec{}, fmt.Errorf("could not parse store size (%s) %s", value, err)
+			return SizeSpec{}, errors.Newf("could not parse %s size (%s) %s", field, value, err)
 		}
 		if percentRange != nil {
 			if (percentRange.min != nil && size.Percent < *percentRange.min) ||
 				(percentRange.max != nil && size.Percent > *percentRange.max) {
-				return SizeSpec{}, fmt.Errorf(
-					"store size (%s) must be between %f%% and %f%%",
+				return SizeSpec{}, errors.Newf(
+					"%s size (%s) must be between %f%% and %f%%",
+					field,
 					value,
 					*percentRange.min,
 					*percentRange.max,
@@ -110,16 +112,16 @@ func NewSizeSpec(
 		var err error
 		size.InBytes, err = humanizeutil.ParseBytes(value)
 		if err != nil {
-			return SizeSpec{}, fmt.Errorf("could not parse store size (%s) %s", value, err)
+			return SizeSpec{}, errors.Newf("could not parse %s size (%s) %s", field, value, err)
 		}
 		if bytesRange != nil {
 			if bytesRange.min != nil && size.InBytes < *bytesRange.min {
-				return SizeSpec{}, fmt.Errorf("store size (%s) must be larger than %s", value,
-					humanizeutil.IBytes(*bytesRange.min))
+				return SizeSpec{}, errors.Newf("%s size (%s) must be larger than %s",
+					field, value, humanizeutil.IBytes(*bytesRange.min))
 			}
 			if bytesRange.max != nil && size.InBytes > *bytesRange.max {
-				return SizeSpec{}, fmt.Errorf("store size (%s) must be smaller than %s", value,
-					humanizeutil.IBytes(*bytesRange.max))
+				return SizeSpec{}, errors.Newf("%s size (%s) must be smaller than %s",
+					field, value, humanizeutil.IBytes(*bytesRange.max))
 			}
 		}
 	}
@@ -150,7 +152,7 @@ var _ pflag.Value = &SizeSpec{}
 // Set adds a new value to the StoreSpecValue. It is the important part of
 // pflag's value interface.
 func (ss *SizeSpec) Set(value string) error {
-	spec, err := NewSizeSpec(value, nil, nil)
+	spec, err := NewSizeSpec("specified", value, nil, nil)
 	if err != nil {
 		return err
 	}
@@ -162,10 +164,11 @@ func (ss *SizeSpec) Set(value string) error {
 // StoreSpec contains the details that can be specified in the cli pertaining
 // to the --store flag.
 type StoreSpec struct {
-	Path       string
-	Size       SizeSpec
-	InMemory   bool
-	Attributes roachpb.Attributes
+	Path        string
+	Size        SizeSpec
+	BallastSize *SizeSpec
+	InMemory    bool
+	Attributes  roachpb.Attributes
 	// StickyInMemoryEngineID is a unique identifier associated with a given
 	// store which will remain in memory even after the default Engine close
 	// until it has been explicitly cleaned up by CleanupStickyInMemEngine[s]
@@ -190,6 +193,7 @@ type StoreSpec struct {
 
 // String returns a fully parsable version of the store spec.
 func (ss StoreSpec) String() string {
+	// TODO(jackson): Implement redact.SafeFormatter
 	var buffer bytes.Buffer
 	if len(ss.Path) != 0 {
 		fmt.Fprintf(&buffer, "path=%s,", ss.Path)
@@ -203,6 +207,14 @@ func (ss StoreSpec) String() string {
 	if ss.Size.Percent > 0 {
 		fmt.Fprintf(&buffer, "size=%s%%,", humanize.Ftoa(ss.Size.Percent))
 	}
+	if ss.BallastSize != nil {
+		if ss.BallastSize.InBytes > 0 {
+			fmt.Fprintf(&buffer, "ballast-size=%s,", humanizeutil.IBytes(ss.BallastSize.InBytes))
+		}
+		if ss.BallastSize.Percent > 0 {
+			fmt.Fprintf(&buffer, "ballast-size=%s%%,", humanize.Ftoa(ss.BallastSize.Percent))
+		}
+	}
 	if len(ss.Attributes.Attrs) > 0 {
 		fmt.Fprint(&buffer, "attrs=")
 		for i, attr := range ss.Attributes.Attrs {
@@ -308,13 +320,28 @@ func NewStoreSpec(value string) (StoreSpec, error) {
 			var minPercent float64 = 1
 			var maxPercent float64 = 100
 			ss.Size, err = NewSizeSpec(
+				"store",
 				value,
 				&intInterval{min: &minBytesAllowed},
 				&floatInterval{min: &minPercent, max: &maxPercent},
 			)
 			if err != nil {
 				return StoreSpec{}, err
 			}
+		case "ballast-size":
+			var minBytesAllowed int64
+			var minPercent float64 = 0
+			var maxPercent float64 = 50
+			ballastSize, err := NewSizeSpec(
+				"ballast",
+				value,
+				&intInterval{min: &minBytesAllowed},
+				&floatInterval{min: &minPercent, max: &maxPercent},
+			)
+			if err != nil {
+				return StoreSpec{}, err
+			}
+			ss.BallastSize = &ballastSize
 		case "attrs":
 			// Check to make sure there are no duplicate attributes.
 			attrMap := make(map[string]struct{})
@@ -384,6 +411,9 @@ func NewStoreSpec(value string) (StoreSpec, error) {
 		if ss.Size.Percent == 0 && ss.Size.InBytes == 0 {
 			return StoreSpec{}, fmt.Errorf("size must be specified for an in memory store")
 		}
+		if ss.BallastSize != nil {
+			return StoreSpec{}, fmt.Errorf("ballast-size specified for in memory store")
+		}
 	} else if ss.Path == "" {
 		return StoreSpec{}, fmt.Errorf("no path specified")
 	}
@@ -417,6 +447,14 @@ func (ssl StoreSpecList) String() string {
 // root directory. It must not be changed without a proper migration.
 const AuxiliaryDir = "auxiliary"
 
+// EmergencyBallastFile returns the path (relative to a data directory) used
+// for an emergency ballast file. The returned path must be stable across
+// releases (eg, we cannot change these constants), otherwise we may duplicate
+// ballasts.
+func EmergencyBallastFile(pathJoin func(...string) string, dataDir string) string {
+	return pathJoin(dataDir, AuxiliaryDir, "EMERGENCY_BALLAST")
+}
+
 // PreventedStartupFile is the filename (relative to 'dir') used for files that
 // can block server startup.
 func PreventedStartupFile(dir string) string {

pkg/base/store_spec_test.go

@@ -128,6 +128,13 @@ target_file_size=2097152`
 		{"size=20GiB,path=/mnt/hda1,size=20GiB", "size field was used twice in store definition", StoreSpec{}},
 		{"size=123TB", "no path specified", StoreSpec{}},
 
+		// ballast size
+		{"path=/mnt/hda1,ballast-size=671088640", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{InBytes: 671088640}}},
+		{"path=/mnt/hda1,ballast-size=20GB", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{InBytes: 20000000000}}},
+		{"path=/mnt/hda1,ballast-size=1%", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{Percent: 1}}},
+		{"path=/mnt/hda1,ballast-size=100.000%", "ballast size (100.000%) must be between 0.000000% and 50.000000%", StoreSpec{}},
+		{"ballast-size=20GiB,path=/mnt/hda1,ballast-size=20GiB", "ballast-size field was used twice in store definition", StoreSpec{}},
+
 		// type
 		{"type=mem,size=20GiB", "", StoreSpec{Size: SizeSpec{InBytes: 21474836480}, InMemory: true}},
 		{"size=20GiB,type=mem", "", StoreSpec{Size: SizeSpec{InBytes: 21474836480}, InMemory: true}},

pkg/cli/exit/codes.go

@@ -58,6 +58,10 @@ func TimeoutAfterFatalError() Code { return Code{8} }
 // during a logging operation to a network collector.
 func LoggingNetCollectorUnavailable() Code { return Code{9} }
 
+// DiskFull (10) indicates an emergency shutdown in response to a
+// store's full disk.
+func DiskFull() Code { return Code{10} }
+
 // Codes that are specific to client commands follow. It's possible
 // for codes to be reused across separate client or server commands.
 // Command-specific exit codes should be allocated down from 125.

pkg/cli/start.go

@@ -55,6 +55,8 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/cockroach/pkg/util/tracing"
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/redact"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
@@ -347,6 +349,16 @@ func runStart(cmd *cobra.Command, args []string, startSingleNode bool) (returnEr
 		}()
 	}
 
+	// Check for stores with full disks and exit with an informative exit
+	// code. This needs to happen early during start, before we perform any
+	// writes to the filesystem including log rotation. We need to guarantee
+	// that the process continues to exit with the Disk Full exit code. A
+	// flapping exit code can affect alerting, including the alerting
+	// performed within CockroachCloud.
+	if err := exitIfDiskFull(serverCfg.Stores.Specs); err != nil {
+		return err
+	}
+
 	// Set up a cancellable context for the entire start command.
 	// The context will be canceled at the end.
 	ctx, cancel := context.WithCancel(context.Background())
@@ -1015,6 +1027,46 @@ func maybeWarnMemorySizes(ctx context.Context) {
 	}
 }
 
+func exitIfDiskFull(specs []base.StoreSpec) error {
+	var cause error
+	var ballastPaths []string
+	var ballastMissing bool
+	for _, spec := range specs {
+		isDiskFull, err := storage.IsDiskFull(vfs.Default, spec)
+		if err != nil {
+			return err
+		}
+		if !isDiskFull {
+			continue
+		}
+		path := base.EmergencyBallastFile(vfs.Default.PathJoin, spec.Path)
+		ballastPaths = append(ballastPaths, path)
+		if _, err := vfs.Default.Stat(path); oserror.IsNotExist(err) {
+			ballastMissing = true
+		}
+		cause = errors.CombineErrors(cause, errors.Newf(`store %s: out of disk space`, spec.Path))
+	}
+	if cause == nil {
+		return nil
+	}
+
+	// TODO(jackson): Link to documentation surrounding the ballast.
+
+	err := clierror.NewError(cause, exit.DiskFull())
+	if ballastMissing {
+		return errors.WithHint(err, `At least one ballast file is missing.
+You may need to replace this node because there is
+insufficient disk space to start.`)
+	}
+
+	ballastPathsStr := strings.Join(ballastPaths, "\n")
+	err = errors.WithHintf(err, `Deleting or truncating the ballast file(s) at
+%s
+may reclaim enough space to start. Proceed with caution. Complete
+disk space exhaustion may result in node loss.`, ballastPathsStr)
+	return err
+}
+
 // setupAndInitializeLoggingAndProfiling does what it says on the label.
 // Prior to this however it determines suitable defaults for the
 // logging output directory and the verbosity level of stderr logging.

pkg/cli/start_test.go

@@ -104,6 +104,13 @@ func TestStartArgChecking(t *testing.T) {
 		{[]string{`--store=size=-1231MB`}, `store size \(-1231MB\) must be larger than`},
 		{[]string{`--store=size=1231B`}, `store size \(1231B\) must be larger than`},
 		{[]string{`--store=size=1231BLA`}, `unhandled size name: bla`},
+		{[]string{`--store=ballast-size=60.0`}, `ballast size \(60.0\) must be between 0.000000% and 50.000000%`},
+		{[]string{`--store=ballast-size=1231BLA`}, `unhandled size name: bla`},
+		{[]string{`--store=ballast-size=0.5%,path=.`}, ``},
+		{[]string{`--store=ballast-size=.5,path=.`}, ``},
+		{[]string{`--store=ballast-size=50.%,path=.`}, ``},
+		{[]string{`--store=ballast-size=50%,path=.`}, ``},
+		{[]string{`--store=ballast-size=2GiB,path=.`}, ``},
 		{[]string{`--store=attrs=bli:bli`}, `duplicate attribute`},
 		{[]string{`--store=type=bli`}, `bli is not a valid store type`},
 		{[]string{`--store=bla=bli`}, `bla is not a valid store field`},

pkg/server/config.go

@@ -529,11 +529,14 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) {
 				engines = append(engines, e)
 			}
 		} else {
+			if err := vfs.Default.MkdirAll(spec.Path, 0755); err != nil {
+				return Engines{}, errors.Wrap(err, "creating store directory")
+			}
+			du, err := vfs.Default.GetDiskUsage(spec.Path)
+			if err != nil {
+				return Engines{}, errors.Wrap(err, "retrieving disk usage")
+			}
 			if spec.Size.Percent > 0 {
-				du, err := vfs.Default.GetDiskUsage(spec.Path)
-				if err != nil {
-					return Engines{}, err
-				}
 				sizeInBytes = int64(float64(du.TotalBytes) * spec.Size.Percent / 100)
 			}
 			if sizeInBytes != 0 && !skipSizeCheck && sizeInBytes < base.MinimumStoreSize {
@@ -548,6 +551,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) {
 				Attrs:                   spec.Attributes,
 				Dir:                     spec.Path,
 				MaxSize:                 sizeInBytes,
+				BallastSize:             storage.BallastSizeBytes(spec, du),
 				Settings:                cfg.Settings,
 				UseFileRegistry:         spec.UseFileRegistry,
 				DisableSeparatedIntents: disableSeparatedIntents,

pkg/storage/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "array_32bit.go",
         "array_64bit.go",
+        "ballast.go",
         "batch.go",
         "disk_map.go",
         "doc.go",
@@ -41,6 +42,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/base",
+        "//pkg/cli/exit",
         "//pkg/clusterversion",
         "//pkg/keys",
         "//pkg/kv/kvserver/concurrency/lock",
@@ -63,6 +65,7 @@ go_library(
         "//pkg/util/protoutil",
         "//pkg/util/stop",
         "//pkg/util/syncutil",
+        "//pkg/util/sysutil",
         "//pkg/util/timeutil",
         "//pkg/util/tracing",
         "//pkg/util/uuid",
@@ -84,6 +87,7 @@ go_test(
     name = "storage_test",
     size = "medium",
     srcs = [
+        "ballast_test.go",
         "batch_test.go",
         "bench_pebble_test.go",
         "bench_test.go",
@@ -137,6 +141,7 @@ go_test(
         "//pkg/util/randutil",
         "//pkg/util/shuffle",
         "//pkg/util/stop",
+        "//pkg/util/sysutil",
         "//pkg/util/timeutil",
         "//pkg/util/uint128",
         "//pkg/util/uuid",