Merge pull request kubernetes-csi#158 from darkowlzz/separate-ctrl-no…

…de-addresses

Add support for separate controller and node addresses

cmd/csi-sanity/sanity_test.go

@@ -36,6 +36,7 @@ var (
 
 func init() {
 	flag.StringVar(&config.Address, prefix+"endpoint", "", "CSI endpoint")
+	flag.StringVar(&config.ControllerAddress, prefix+"controllerendpoint", "", "CSI controller endpoint")
 	flag.BoolVar(&version, prefix+"version", false, "Version of this program")
 	flag.StringVar(&config.TargetPath, prefix+"mountdir", os.TempDir()+"/csi", "Mount point for NodePublish")
 	flag.StringVar(&config.StagingPath, prefix+"stagingdir", os.TempDir()+"/csi", "Mount point for NodeStage if staging is supported")

cmd/mock-driver/main.go

@@ -48,51 +48,136 @@ func main() {
 		os.Exit(1)
 	}
 
+	controllerEndpoint := os.Getenv("CSI_CONTROLLER_ENDPOINT")
+	if len(controllerEndpoint) == 0 {
+		// If empty, set to the common endpoint.
+		controllerEndpoint = endpoint
+	}
+	if strings.Contains(controllerEndpoint, ":") {
+		fmt.Println("CSI_CONTROLLER_ENDPOINT must be a unix path")
+		os.Exit(1)
+	}
+
 	// Create mock driver
 	s := service.New(config)
-	servers := &driver.CSIDriverServers{
-		Controller: s,
-		Identity:   s,
-		Node:       s,
-	}
-	d := driver.NewCSIDriver(servers)
 
-	// If creds is enabled, set the default creds.
-	setCreds := os.Getenv("CSI_ENABLE_CREDS")
-	if len(setCreds) > 0 && setCreds == "true" {
-		d.SetDefaultCreds()
-	}
+	if endpoint == controllerEndpoint {
+		servers := &driver.CSIDriverServers{
+			Controller: s,
+			Identity:   s,
+			Node:       s,
+		}
+		d := driver.NewCSIDriver(servers)
 
-	// Listen
-	os.Remove(endpoint)
-	l, err := net.Listen("unix", endpoint)
-	if err != nil {
-		fmt.Printf("Error: Unable to listen on %s socket: %v\n",
-			endpoint,
-			err)
-		os.Exit(1)
-	}
-	defer os.Remove(endpoint)
+		// If creds is enabled, set the default creds.
+		setCreds := os.Getenv("CSI_ENABLE_CREDS")
+		if len(setCreds) > 0 && setCreds == "true" {
+			d.SetDefaultCreds()
+		}
 
-	// Start server
-	if err := d.Start(l); err != nil {
-		fmt.Printf("Error: Unable to start mock CSI server: %v\n",
-			err)
-		os.Exit(1)
-	}
-	fmt.Println("mock driver started")
-
-	// Wait for signal
-	sigc := make(chan os.Signal, 1)
-	sigs := []os.Signal{
-		syscall.SIGTERM,
-		syscall.SIGHUP,
-		syscall.SIGINT,
-		syscall.SIGQUIT,
-	}
-	signal.Notify(sigc, sigs...)
+		// Listen
+		os.Remove(endpoint)
+		os.Remove(controllerEndpoint)
+		l, err := net.Listen("unix", endpoint)
+		if err != nil {
+			fmt.Printf("Error: Unable to listen on %s socket: %v\n",
+				endpoint,
+				err)
+			os.Exit(1)
+		}
+		defer os.Remove(endpoint)
 
-	<-sigc
-	d.Stop()
-	fmt.Println("mock driver stopped")
+		// Start server
+		if err := d.Start(l); err != nil {
+			fmt.Printf("Error: Unable to start mock CSI server: %v\n",
+				err)
+			os.Exit(1)
+		}
+		fmt.Println("mock driver started")
+
+		// Wait for signal
+		sigc := make(chan os.Signal, 1)
+		sigs := []os.Signal{
+			syscall.SIGTERM,
+			syscall.SIGHUP,
+			syscall.SIGINT,
+			syscall.SIGQUIT,
+		}
+		signal.Notify(sigc, sigs...)
+
+		<-sigc
+		d.Stop()
+		fmt.Println("mock driver stopped")
+	} else {
+		controllerServer := &driver.CSIDriverControllerServer{
+			Controller: s,
+			Identity:   s,
+		}
+		dc := driver.NewCSIDriverController(controllerServer)
+
+		nodeServer := &driver.CSIDriverNodeServer{
+			Node:     s,
+			Identity: s,
+		}
+		dn := driver.NewCSIDriverNode(nodeServer)
+
+		setCreds := os.Getenv("CSI_ENABLE_CREDS")
+		if len(setCreds) > 0 && setCreds == "true" {
+			dc.SetDefaultCreds()
+			dn.SetDefaultCreds()
+		}
+
+		// Listen controller.
+		os.Remove(controllerEndpoint)
+		l, err := net.Listen("unix", controllerEndpoint)
+		if err != nil {
+			fmt.Printf("Error: Unable to listen on %s socket: %v\n",
+				controllerEndpoint,
+				err)
+			os.Exit(1)
+		}
+		defer os.Remove(controllerEndpoint)
+
+		// Start controller server.
+		if err = dc.Start(l); err != nil {
+			fmt.Printf("Error: Unable to start mock CSI controller server: %v\n",
+				err)
+			os.Exit(1)
+		}
+		fmt.Println("mock controller driver started")
+
+		// Listen node.
+		os.Remove(endpoint)
+		l, err = net.Listen("unix", endpoint)
+		if err != nil {
+			fmt.Printf("Error: Unable to listen on %s socket: %v\n",
+				endpoint,
+				err)
+			os.Exit(1)
+		}
+		defer os.Remove(endpoint)
+
+		// Start node server.
+		if err = dn.Start(l); err != nil {
+			fmt.Printf("Error: Unable to start mock CSI node server: %v\n",
+				err)
+			os.Exit(1)
+		}
+		fmt.Println("mock node driver started")
+
+		// Wait for signal
+		sigc := make(chan os.Signal, 1)
+		sigs := []os.Signal{
+			syscall.SIGTERM,
+			syscall.SIGHUP,
+			syscall.SIGINT,
+			syscall.SIGQUIT,
+		}
+		signal.Notify(sigc, sigs...)
+
+		<-sigc
+		dc.Stop()
+		dn.Stop()
+		fmt.Println("mock drivers stopped")
+	}
 }

driver/driver-controller.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2019 Kubernetes Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package driver
+
+import (
+	"context"
+	"net"
+	"sync"
+
+	"google.golang.org/grpc/reflection"
+
+	csi "github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc"
+)
+
+// CSIDriverControllerServer is the Controller service component of the driver.
+type CSIDriverControllerServer struct {
+	Controller csi.ControllerServer
+	Identity   csi.IdentityServer
+}
+
+// CSIDriverController is the CSI Driver Controller backend.
+type CSIDriverController struct {
+	listener         net.Listener
+	server           *grpc.Server
+	controllerServer *CSIDriverControllerServer
+	wg               sync.WaitGroup
+	running          bool
+	lock             sync.Mutex
+	creds            *CSICreds
+}
+
+func NewCSIDriverController(controllerServer *CSIDriverControllerServer) *CSIDriverController {
+	return &CSIDriverController{
+		controllerServer: controllerServer,
+	}
+}
+
+func (c *CSIDriverController) goServe(started chan<- bool) {
+	goServe(c.server, &c.wg, c.listener, started)
+}
+
+func (c *CSIDriverController) Address() string {
+	return c.listener.Addr().String()
+}
+
+func (c *CSIDriverController) Start(l net.Listener) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	// Set listener.
+	c.listener = l
+
+	// Create a new grpc server.
+	c.server = grpc.NewServer(
+		grpc.UnaryInterceptor(c.callInterceptor),
+	)
+
+	if c.controllerServer.Controller != nil {
+		csi.RegisterControllerServer(c.server, c.controllerServer.Controller)
+	}
+	if c.controllerServer.Identity != nil {
+		csi.RegisterIdentityServer(c.server, c.controllerServer.Identity)
+	}
+
+	reflection.Register(c.server)
+
+	waitForServer := make(chan bool)
+	c.goServe(waitForServer)
+	<-waitForServer
+	c.running = true
+	return nil
+}
+
+func (c *CSIDriverController) Stop() {
+	stop(&c.lock, &c.wg, c.server, c.running)
+}
+
+func (c *CSIDriverController) Close() {
+	c.server.Stop()
+}
+
+func (c *CSIDriverController) IsRunning() bool {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	return c.running
+}
+
+func (c *CSIDriverController) SetDefaultCreds() {
+	setDefaultCreds(c.creds)
+}
+
+func (c *CSIDriverController) callInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+	return callInterceptor(ctx, c.creds, req, info, handler)
+}