{cmd,pkg}: add satellite startup check to linksharing and gateway

Closes #329

Change-Id: Id3bb6a057c38a490fa0e6c536741fac323fe85e3

cmd/gateway-mt/config.yaml.lock

@@ -175,6 +175,16 @@ cert-magic.staging: false
 # time to delay server shutdown while returning 503s on the health endpoint
 # shutdown-delay: 45s
 
+# whether to check for satellite connectivity before starting
+startup-check.enabled: true
+
+# list of satellite NodeURLs
+startup-check.satellites:
+- https://www.storj.io/dcs-satellites
+
+# maximum time to spend on checks
+startup-check.timeout: 30s
+
 # address for jaeger agent
 # tracing.agent-addr: agent.tracing.datasci.storj.io:5775
 

cmd/gateway-mt/main.go

@@ -120,7 +120,9 @@ func cmdRun(cmd *cobra.Command, _ []string) (err error) {
 		return err
 	}
 
-	var g errgroup.Group
+	// if peer.Run() fails, we want to ensure the context is canceled so we
+	// don't hang on ctx.Done before closing the peer.
+	g, ctx := errgroup.WithContext(ctx)
 
 	g.Go(func() error {
 		<-ctx.Done()

cmd/linksharing/config.yaml.lock

@@ -151,6 +151,16 @@ standard-renders-content: false
 # serve HTML as text/html instead of text/plain for standard (non-hosting) requests
 standard-views-html: false
 
+# whether to check for satellite connectivity before starting
+startup-check.enabled: true
+
+# list of satellite NodeURLs
+startup-check.satellites:
+- https://www.storj.io/dcs-satellites
+
+# maximum time to spend on checks
+startup-check.timeout: 30s
+
 # the path to where web assets are located
 static-sources-path: ./pkg/linksharing/web/static
 

cmd/linksharing/main.go

@@ -53,6 +53,7 @@ type LinkSharing struct {
 	ConnectionPool         connectionPoolConfig
 	CertMagic              certMagic
 	ShutdownDelay          time.Duration `user:"true" help:"time to delay server shutdown while returning 503s on the health endpoint" devDefault:"1s" releaseDefault:"45s"`
+	StartupCheck           startupCheck
 }
 
 // connectionPoolConfig is a config struct for configuring RPC connection pool options.
@@ -75,6 +76,12 @@ type certMagic struct {
 	SkipPaidTierAllowlist []string      `user:"true" help:"comma separated list of domain names which bypass paid tier queries. Set to * to disable tier check entirely"`
 }
 
+type startupCheck struct {
+	Enabled    bool          `user:"true" help:"whether to check for satellite connectivity before starting" default:"true"`
+	Satellites []string      `user:"true" help:"list of satellite NodeURLs" default:"https://www.storj.io/dcs-satellites"`
+	Timeout    time.Duration `user:"true" help:"maximum time to spend on checks" default:"30s"`
+}
+
 var (
 	rootCmd = &cobra.Command{
 		Use:   "link sharing service",
@@ -142,12 +149,13 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
 
 	peer, err := linksharing.New(log, linksharing.Config{
 		Server: httpserver.Config{
-			Name:            "Link Sharing",
-			Address:         runCfg.Address,
-			AddressTLS:      runCfg.AddressTLS,
-			TrafficLogging:  true,
-			TLSConfig:       tlsConfig,
-			ShutdownTimeout: -1,
+			Name:               "Link Sharing",
+			Address:            runCfg.Address,
+			AddressTLS:         runCfg.AddressTLS,
+			TrafficLogging:     true,
+			TLSConfig:          tlsConfig,
+			ShutdownTimeout:    -1,
+			StartupCheckConfig: httpserver.StartupCheckConfig(runCfg.StartupCheck),
 		},
 		Handler: sharing.Config{
 			URLBases:               publicURLs,
@@ -175,7 +183,9 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
-	var g errgroup.Group
+	// if peer.Run() fails, we want to ensure the context is canceled so we
+	// don't hang on ctx.Done before closing the peer.
+	g, ctx := errgroup.WithContext(ctx)
 
 	g.Go(func() error {
 		<-ctx.Done()

pkg/httpserver/server.go

@@ -27,6 +27,7 @@ import (
 	"storj.io/gateway-mt/pkg/certstorage"
 	"storj.io/gateway-mt/pkg/gpublicca"
 	"storj.io/gateway-mt/pkg/middleware"
+	"storj.io/gateway-mt/pkg/startupcheck"
 )
 
 var mon = monkit.Package()
@@ -59,6 +60,10 @@ type Config struct {
 	// 10 seconds if unset. If set to a negative value, the server will be
 	// closed immediately.
 	ShutdownTimeout time.Duration
+
+	// StartupCheckConfig configures a startup check that must pass in order for
+	// servers to start listening.
+	StartupCheckConfig StartupCheckConfig
 }
 
 // TestIssuerConfig is configuration to a test ACME server, which if defined will
@@ -150,6 +155,13 @@ type TLSConfig struct {
 	Ctx context.Context
 }
 
+// StartupCheckConfig provides startup check configuration.
+type StartupCheckConfig struct {
+	Enabled    bool
+	Satellites []string
+	Timeout    time.Duration
+}
+
 // Server is the HTTP server.
 //
 // architecture: Endpoint
@@ -163,6 +175,7 @@ type Server struct {
 	server          *http.Server
 	serverTLS       *http.Server
 	shutdownTimeout time.Duration
+	startupCheck    *startupcheck.NodeURLCheck
 }
 
 // CertMagicOnDemandDecisionFunc is a concrete type for
@@ -217,6 +230,18 @@ func New(log *zap.Logger, handler http.Handler, decisionFunc CertMagicOnDemandDe
 		config.ShutdownTimeout = DefaultShutdownTimeout
 	}
 
+	var startupCheck *startupcheck.NodeURLCheck
+	if config.StartupCheckConfig.Enabled {
+		startupCheck, err = startupcheck.NewNodeURLCheck(startupcheck.NodeURLCheckConfig{
+			NodeURLs: config.StartupCheckConfig.Satellites,
+			Logger:   log.Sugar(),
+			Timeout:  config.StartupCheckConfig.Timeout,
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if config.Name != "" {
 		log = log.With(zap.String("server", config.Name))
 	}
@@ -229,6 +254,7 @@ func New(log *zap.Logger, handler http.Handler, decisionFunc CertMagicOnDemandDe
 		server:          server,
 		serverTLS:       serverTLS,
 		shutdownTimeout: config.ShutdownTimeout,
+		startupCheck:    startupCheck,
 		handler:         handler,
 	}, nil
 }
@@ -237,6 +263,12 @@ func New(log *zap.Logger, handler http.Handler, decisionFunc CertMagicOnDemandDe
 func (server *Server) Run(ctx context.Context) (err error) {
 	defer mon.Task()(&ctx)(&err)
 
+	if server.startupCheck != nil {
+		if err = server.startupCheck.Check(ctx); err != nil {
+			return err
+		}
+	}
+
 	var group errgroup.Group
 
 	group.Go(func() (err error) {

pkg/server/config.go

@@ -37,6 +37,7 @@ type Config struct {
 	SatelliteConnectionpool SatelliteConnectionPoolConfig
 	ConnectionPool          ConnectionPoolConfig
 	CertMagic               certMagic
+	StartupCheck            startupCheck
 }
 
 type certMagic struct {
@@ -49,6 +50,12 @@ type certMagic struct {
 	Bucket                  string `user:"true" help:"bucket to use for certificate storage"`
 }
 
+type startupCheck struct {
+	Enabled    bool          `user:"true" help:"whether to check for satellite connectivity before starting" default:"true"`
+	Satellites []string      `user:"true" help:"list of satellite NodeURLs" default:"https://www.storj.io/dcs-satellites"`
+	Timeout    time.Duration `user:"true" help:"maximum time to spend on checks" default:"30s"`
+}
+
 // ConnectionPoolConfig is a config struct for configuring RPC connection pool
 // options.
 type ConnectionPoolConfig struct {

pkg/server/server.go

@@ -147,10 +147,11 @@ func New(config Config, log *zap.Logger, trustedIPs trustedip.List, corsAllowedO
 	}
 
 	server, err := httpserver.New(log, handler, nil, httpserver.Config{
-		Address:        config.Server.Address,
-		AddressTLS:     config.Server.AddressTLS,
-		TLSConfig:      tlsConfig,
-		TrafficLogging: false, // gateway-mt has its own logging middleware for this
+		Address:            config.Server.Address,
+		AddressTLS:         config.Server.AddressTLS,
+		TLSConfig:          tlsConfig,
+		TrafficLogging:     false, // gateway-mt has its own logging middleware for this
+		StartupCheckConfig: httpserver.StartupCheckConfig(config.StartupCheck),
 	})
 	if err != nil {
 		return nil, err