SONiC Management Framework Release 1.0 (#23)

* GNMI PoC changes * Ignore test results for gnmi_server The Debian files attempt to run make check for the gNMI server, however, it fails on a build machine, because it expects the Redis server to be available and listening on localhost port 6379. For now, the result of the test is ignored, and we have to find a way to fix this so that the tests work even on a build machine. * Move source files into src, change import paths, fixing Makefile * More Makefile changes * Install CVL deps and update gopath * Add back install and other targets to fix deb package * Add gnmi set and get client to build * trigger building ocbinds file in translib, forgot to add gnmi_clients in last commit * Fix build issue * Port polling subscriptions code for translib * Implement ONCE subscribe mode for translib * Add gnmi_cli for subscribe client, update Makefile to build it, add .gitignore * Ad once channel cleanup * Initial support for sample based streaming subscribe * Add paths to existing timer group if interval is the same * cleanup * Use ticker_info struct * cleanup * Add some dummy code for future subscribe api * Fixes, also include binary for gnmi_cli for now * Fix issue with updates and replace with interfaces * wip * convert subscribeResponse to gnmi subscribeResponse and return * Fix issue with when sync message is sent. Add heartbeat support. Add valueCache for suppress_redundant (wip) * Update IsSupported to IsOnChangeSupported, add heartbeat_interval support to gnmi_cli, remove debug code * Close channel on error, check if timer if HB * Implement suppress_redundant, fix ticker_map format, remove some debug prints, fix heartbeats * Fix bug with sync messages * Return models based on translib GetModels API * Check error response before adding to on_change list * Implement updates_only for ON_CHANGE streaming mode * [JIRA SONIC-8247] Merge Telemetry changes done by Dell team Change-Id: Icc05d40efe57184acdadff7553b847b9766277a8 * Add test directory * Disable old tests that fail due to new fields being added. Add basic interface get test and capabilities test. Change package name and add symlink to cvl schema * [JIRA SONIC-8247] Merge Telemetry changes done by Dell team Adding gnmi test files from project-arlo master branch Change-Id: If748dc72261d387210a86f841997fd9dbb96fdac * working gnmi sys and pfm test without clearing bad logs * [JIRA SONIC-8247] Merge Telemetry changes done by Dell team Change-Id: Ia4f94e2998be4034e35e9ef331d3e4c350f3c91c * Changes for testing get/set operations on OC interfaces Signed-off-by: Tejaswi Goel <[email protected]> * Fix some issues with test infra * fix * Fix paths * Code cleanup in server_test.go Signed-off-by: Tejaswi Goel <[email protected]> * Add test to set ip address Signed-off-by: Tejaswi Goel <[email protected]> * Working gnmi server test for sys,pfm,interfaces * Add json file for ip config Signed-off-by: Tejaswi Goel <[email protected]> * Add delete testcase Signed-off-by: Tejaswi Goel <[email protected]> * Minor fix in Delete OC Interface IP testcase Signed-off-by: Tejaswi Goel <[email protected]> * Add code to check values set Signed-off-by: Tejaswi Goel <[email protected]> * [JIRA SONIC-9215] Telemetry UT from Dell Change-Id: I655377abbf670e1cb8a32a4e9a704242a0fe49e3 * Add full code for gnmi_cli with our changes so it can be built from scratch like gnmi_set/get. Update Makefile. Add fixes to gnmi_cli to use seconds as units everywhere instead of nanoseconds, and accept string for -streaming_type * [JIRA SONIC-9215] Telemetry UT from Dell Change-Id: Ie40ba2086ba663181a7db1d075a9f361a557daf1 * Fix streaming_type in tests * Set Min TLS Version and cipher suites * Fix spaces to tabs * Update cipher suite list again * Remove duplicate ciphers * Disable platform specific test for now * Add check that heartbeat_interval is only valid on streaming query type * Made changes in the Makefile to maintain the github pkg version * Remove gnmi_client direcotry, instead download and patch files as needed * Change project-arlo to Azure in README.md * Remove arlo urls from debian/control * Address review comments * Add cvl deps to telemetry makefile and build cvl locally * Use patches from mgmt-framework, make and install yang models. * Use isTargetDb instead of hard-coded list * Begin conversion to go modules * Copy translib and cvl to compile locally until they convert to go modules * fix * Add schema and models * Change gnmi_server package name back to just gnmi * merge fixes * Add gnmi clients * For now don't verify tests so that deb can be built * Add test binary compilation back * Split up sonic-mgmt-framework specific steps in Makefile * Fix translib based tests by preparing db with db dump from switch * Don't stop make check on test failure for now since there is no redis in the build container Co-authored-by: rupesh-k <[email protected]> Co-authored-by: nirenjan <[email protected]> Co-authored-by: Eric Seifert <[email protected]> Co-authored-by: Jeff Yin <[email protected]> Co-authored-by: Senthil Kumar Ganesan <[email protected]> Co-authored-by: Arunsundar Kannan <[email protected]> Co-authored-by: Tejaswi Goel <[email protected]> Co-authored-by: Balachandar Mani <[email protected]>
sonic-net · Dec 23, 2019 · aaa9188 · aaa9188
1 parent 4bb02f5
commit aaa9188
Show file tree

Hide file tree

Showing 20 changed files with 8,736 additions and 78 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,10 @@
+build/
+debian/.debhelper/
+debian/files
+debian/sonic-telemetry.debhelper.log
+debian/sonic-telemetry.substvars
+debian/sonic-telemetry/
+vendor
+src
+cvl
+translib
diff --git a/Makefile b/Makefile
@@ -1,33 +1,108 @@
 ifeq ($(GOPATH),)
 export GOPATH=/tmp/go
 endif
+export PATH := $(PATH):$(GOPATH)/bin
 
 INSTALL := /usr/bin/install
 DBDIR := /var/run/redis/sonic-db/
+GO := /usr/local/go/bin/go
+TOP_DIR := $(abspath ..)
+GO_MGMT_PATH=$(TOP_DIR)/sonic-mgmt-framework
+BUILD_DIR := $(GOPATH)/bin
+export CVL_SCHEMA_PATH := $(GO_MGMT_PATH)/src/cvl/schema
 
-all: sonic-telemetry
+SRC_FILES=$(shell find . -name '*.go' | grep -v '_test.go' | grep -v '/tests/')
+TEST_FILES=$(wildcard *_test.go)
+TELEMETRY_TEST_DIR = $(GO_MGMT_PATH)/build/tests/gnmi_server
+TELEMETRY_TEST_BIN = $(TELEMETRY_TEST_DIR)/server.test
 
-sonic-telemetry:
+.phony: mgmt-deps
+
+all: sonic-telemetry $(TELEMETRY_TEST_BIN)
+
+go.mod:
 	/usr/local/go/bin/go mod init github.com/Azure/sonic-telemetry
-	/usr/local/go/bin/go install github.com/Azure/sonic-telemetry/telemetry
-	/usr/local/go/bin/go install github.com/Azure/sonic-telemetry/dialout/dialout_client_cli
+mgmt-deps:
+	rm -rf cvl
+	rm -rf translib
+	cp -r ../sonic-mgmt-framework/src/cvl ./
+	cp -r ../sonic-mgmt-framework/src/translib ./
+	find cvl -name \*\.go -exec sed -i -e 's/\"translib/\"github.com\/Azure\/sonic-telemetry\/translib/g' {} \;
+	find translib -name \*\.go -exec sed -i -e 's/\"translib/\"github.com\/Azure\/sonic-telemetry\/translib/g' {} \;
+	find cvl -name \*\.go -exec sed -i -e 's/\"cvl/\"github.com\/Azure\/sonic-telemetry\/cvl/g' {} \;
+	find translib -name \*\.go -exec sed -i -e 's/\"cvl/\"github.com\/Azure\/sonic-telemetry\/cvl/g' {} \;
+	sed -i -e 's/\.\.\/\.\.\/\.\.\/models\/yang/\.\.\/\.\.\/\.\.\/sonic-mgmt-framework\/models\/yang/' translib/ocbinds/oc.go
+	sed -i -e 's/\$$GO run \$$BUILD_GOPATH\/src\/github.com\/openconfig\/ygot\/generator\/generator.go/generator/' translib/ocbinds/oc.go
+	$(GO) get github.com/openconfig/gnmi@89b2bf29312cda887da916d0f3a32c1624b7935f
+	$(GO) get github.com/openconfig/ygot@724a6b18a9224343ef04fe49199dfb6020ce132a
+	$(GO) get github.com/openconfig/goyang@064f9690516f4f72db189f4690b84622c13b7296
+	$(GO) get github.com/openconfig/goyang@064f9690516f4f72db189f4690b84622c13b7296
+	$(GO) get golang.org/x/crypto/ssh/terminal@e9b2fee46413
+	$(GO) get github.com/jipanyang/gnxi@f0a90cca6fd0041625bcce561b71f849c9b65a8d
+	$(GO) install github.com/openconfig/ygot/generator
+	$(GO) get -x github.com/golang/glog@23def4e6c14b4da8ac2ed8007337bc5eb5007998
+	rm -rf vendor
+	$(GO) mod vendor
+	ln -s vendor src
+	cp -r $(GOPATH)/pkg/mod/github.com/openconfig/[email protected]/* vendor/github.com/openconfig/gnmi/
+	cp -r $(GOPATH)/pkg/mod/github.com/openconfig/[email protected]/* vendor/github.com/openconfig/goyang/
+	cp -r $(GOPATH)/pkg/mod/github.com/openconfig/[email protected]/* vendor/github.com/openconfig/ygot/
+	cp -r $(GOPATH)/pkg/mod/golang.org/x/[email protected] vendor/golang.org/x/crypto
+	chmod -R u+w vendor
+	patch -d vendor -p0 <patches/gnmi_cli.all.patch
+	patch -d vendor/github.com/antchfx/jsonquery -p1 < ../sonic-mgmt-framework/patches/jsonquery.patch
+	patch -d vendor/github.com/openconfig/goyang -p1 < ../sonic-mgmt-framework/goyang-modified-files/goyang.patch
+	patch -d vendor/github.com/openconfig -p1 < ../sonic-mgmt-framework/ygot-modified-files/ygot.patch
+	$(GO) generate github.com/Azure/sonic-telemetry/translib/ocbinds
+	make -C $(GO_MGMT_PATH)/src/cvl/schema
+	make -C $(GO_MGMT_PATH)/models
+	make -C $(GO_MGMT_PATH)/models/yang
+	make -C $(GO_MGMT_PATH)/models/yang/sonic
+
+sonic-telemetry: go.mod mgmt-deps
+	$(GO) install -mod=vendor github.com/Azure/sonic-telemetry/telemetry
+	$(GO) install -mod=vendor github.com/Azure/sonic-telemetry/dialout/dialout_client_cli
+	$(GO) install github.com/jipanyang/gnxi/gnmi_get
+	$(GO) install github.com/jipanyang/gnxi/gnmi_set
+	$(GO) install -mod=vendor github.com/openconfig/gnmi/cmd/gnmi_cli
 
 check:
 	sudo mkdir -p ${DBDIR}
 	sudo cp ./testdata/database_config.json ${DBDIR}
-	/usr/local/go/bin/go test -v github.com/Azure/sonic-telemetry/gnmi_server
-	/usr/local/go/bin/go test -v github.com/Azure/sonic-telemetry/dialout/dialout_client
+	sudo mkdir -p /usr/models/yang || true
+	sudo find $(GO_MGMT_PATH)/models -name '*.yang' -exec cp {} /usr/models/yang/ \;
+	-$(GO) test -mod=vendor -v github.com/Azure/sonic-telemetry/gnmi_server
+	-$(GO) test -mod=vendor -v github.com/Azure/sonic-telemetry/dialout/dialout_client
+
+clean:
+	rm -rf cvl
+	rm -rf translib
+	rm -rf vendor
+	chmod -f -R u+w $(GOPATH)/pkg || true
+	rm -rf $(GOPATH)
+	rm -f src
+
+$(TELEMETRY_TEST_BIN): $(TEST_FILES) $(SRC_FILES)
+	$(GO) test -mod=vendor -c -cover github.com/Azure/sonic-telemetry/gnmi_server -o $@
+	cp -r testdata $(TELEMETRY_TEST_DIR)
+	cp -r $(GO_MGMT_PATH)/src/cvl/schema $(TELEMETRY_TEST_DIR)
 
 install:
-	$(INSTALL) -D ${GOPATH}/bin/telemetry $(DESTDIR)/usr/sbin/telemetry
-	$(INSTALL) -D ${GOPATH}/bin/dialout_client_cli $(DESTDIR)/usr/sbin/dialout_client_cli
+	$(INSTALL) -D $(BUILD_DIR)/telemetry $(DESTDIR)/usr/sbin/telemetry
+	$(INSTALL) -D $(BUILD_DIR)/dialout_client_cli $(DESTDIR)/usr/sbin/dialout_client_cli
+	$(INSTALL) -D $(BUILD_DIR)/gnmi_get $(DESTDIR)/usr/sbin/gnmi_get
+	$(INSTALL) -D $(BUILD_DIR)/gnmi_set $(DESTDIR)/usr/sbin/gnmi_set
+	$(INSTALL) -D $(BUILD_DIR)/gnmi_cli $(DESTDIR)/usr/sbin/gnmi_cli
+
+	mkdir -p $(DESTDIR)/usr/bin/
+	cp -r $(GO_MGMT_PATH)/src/cvl/schema $(DESTDIR)/usr/sbin
+	mkdir -p $(DESTDIR)/usr/models/yang
+	find $(GO_MGMT_PATH)/models -name '*.yang' -exec cp {} $(DESTDIR)/usr/models/yang/ \;
 
 deinstall:
 	rm $(DESTDIR)/usr/sbin/telemetry
 	rm $(DESTDIR)/usr/sbin/dialout_client_cli
+	rm $(DESTDIR)/usr/sbin/gnmi_get
+	rm $(DESTDIR)/usr/sbin/gnmi_set
 
-clean:
-	rm -fr ${GOPATH}
-	rm -rf go.mod
-	rm -rf go.sum
 
diff --git a/README.md b/README.md
@@ -80,4 +80,4 @@ For example:
 * Each developer should fork this repository and [add the team as a Contributor](https://help.github.com/articles/adding-collaborators-to-a-personal-repository)
 * Push your changes to your private fork and do "pull-request" to this repository
 * Use a pull request to do code review
-* Use issues to keep track of what is going on
+* Use issues to keep track of what is going on
diff --git a/dialout/dialout_client/dialout_client.go b/dialout/dialout_client/dialout_client.go
@@ -12,7 +12,7 @@ import (
 	log "github.com/golang/glog"
 	gpb "github.com/openconfig/gnmi/proto/gnmi"
 	"github.com/openconfig/ygot/ygot"
-	"github.com/workiva/go-datastructures/queue"
+	"github.com/Workiva/go-datastructures/queue"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
@@ -388,7 +388,7 @@ restart: //Remote server might go down, in that case we restart with next destin
 		select {
 		default:
 			cs.w.Add(1)
-			go cs.dc.StreamRun(cs.q, cs.stop, &cs.w)
+			go cs.dc.StreamRun(cs.q, cs.stop, &cs.w, nil)
 			time.Sleep(100 * time.Millisecond)
 			err = cs.send(pub)
 			if err != nil {

diff --git a/gnmi_server/client_subscribe.go b/gnmi_server/client_subscribe.go
@@ -7,11 +7,10 @@ import (
 	"sync"
 
 	log "github.com/golang/glog"
-	"github.com/workiva/go-datastructures/queue"
+	"github.com/Workiva/go-datastructures/queue"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 
-	//spb "github.com/Azure/sonic-telemetry/proto"
 	sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
 	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
 )
@@ -24,6 +23,7 @@ type Client struct {
 	errors    int64
 	polled    chan struct{}
 	stop      chan struct{}
+	once      chan struct{}
 	mu        sync.RWMutex
 	q         *queue.PriorityQueue
 	subscribe *gnmipb.SubscriptionList
@@ -117,28 +117,36 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 		return grpc.Errorf(codes.NotFound, "Invalid subscription path: %v %q", err, query)
 	}
 	var dc sdc.Client
-	if target == "OTHERS" {
-		dc, err = sdc.NewNonDbClient(paths, prefix)
-	} else {
-		dc, err = sdc.NewDbClient(paths, prefix)
-	}
 
-	if err != nil {
-		return grpc.Errorf(codes.NotFound, "%v", err)
-	}
+    if target == "OTHERS" {
+            dc, err = sdc.NewNonDbClient(paths, prefix)
+    } else if isTargetDb(target) == true {
+            dc, err = sdc.NewDbClient(paths, prefix)
+    } else {
+            /* For any other target or no target create new Transl Client. */
+            dc, err = sdc.NewTranslClient(prefix, paths)
+    }
+
+    if err != nil {
+            return grpc.Errorf(codes.NotFound, "%v", err)
+    }
+
 
 	switch mode := c.subscribe.GetMode(); mode {
 	case gnmipb.SubscriptionList_STREAM:
 		c.stop = make(chan struct{}, 1)
 		c.w.Add(1)
-		go dc.StreamRun(c.q, c.stop, &c.w)
+		go dc.StreamRun(c.q, c.stop, &c.w, c.subscribe)
 	case gnmipb.SubscriptionList_POLL:
 		c.polled = make(chan struct{}, 1)
 		c.polled <- struct{}{}
 		c.w.Add(1)
 		go dc.PollRun(c.q, c.polled, &c.w)
 	case gnmipb.SubscriptionList_ONCE:
-		return grpc.Errorf(codes.Unimplemented, "SubscriptionList_ONCE is not implemented for SONiC gRPC/gNMI yet: %q", query)
+		c.once = make(chan struct{}, 1)
+		c.once <- struct{}{}
+		c.w.Add(1)
+		go dc.OnceRun(c.q, c.once, &c.w)
 	default:
 		return grpc.Errorf(codes.InvalidArgument, "Unkown subscription mode: %q", query)
 	}
@@ -171,6 +179,9 @@ func (c *Client) Close() {
 	if c.polled != nil {
 		close(c.polled)
 	}
+	if c.once != nil {
+		close(c.once)
+	}
 }
 
 func (c *Client) recv(stream gnmipb.GNMI_SubscribeServer) {

diff --git a/gnmi_server/server.go b/gnmi_server/server.go
@@ -160,23 +160,29 @@ func (s *Server) Get(ctx context.Context, req *gnmipb.GetRequest) (*gnmipb.GetRe
 	var target string
 	prefix := req.GetPrefix()
 	if prefix == nil {
-		return nil, status.Error(codes.Unimplemented, "No target specified in prefix")
+	       return nil, status.Error(codes.Unimplemented, "No target specified in prefix")
 	} else {
-		target = prefix.GetTarget()
-		if target == "" {
-			return nil, status.Error(codes.Unimplemented, "Empty target data not supported yet")
-		}
+	       target = prefix.GetTarget()
+	       if target == "" {
+	               return nil, status.Error(codes.Unimplemented, "Empty target data not supported yet")
+	       }
 	}
 
 	paths := req.GetPath()
+        target = prefix.GetTarget()
 	log.V(5).Infof("GetRequest paths: %v", paths)
 
 	var dc sdc.Client
+
 	if target == "OTHERS" {
 		dc, err = sdc.NewNonDbClient(paths, prefix)
-	} else {
+	} else if isTargetDb(target) == true {
 		dc, err = sdc.NewDbClient(paths, prefix)
+	} else {
+		/* If no prefix target is specified create new Transl Data Client . */
+		dc, err = sdc.NewTranslClient(prefix, paths)
 	}
+
 	if err != nil {
 		return nil, status.Error(codes.NotFound, err.Error())
 	}
@@ -203,11 +209,114 @@ func (s *Server) Get(ctx context.Context, req *gnmipb.GetRequest) (*gnmipb.GetRe
 }
 
 // Set method is not implemented. Refer to gnxi for examples with openconfig integration
-func (srv *Server) Set(context.Context, *gnmipb.SetRequest) (*gnmipb.SetResponse, error) {
-	return nil, grpc.Errorf(codes.Unimplemented, "Set() is not implemented")
+func (srv *Server) Set(ctx context.Context,req *gnmipb.SetRequest) (*gnmipb.SetResponse, error) {
+		var results []*gnmipb.UpdateResult
+		var err error
+
+		/* Fetch the prefix. */
+		prefix := req.GetPrefix()
+
+               /* Create Transl client. */
+		dc, _ := sdc.NewTranslClient(prefix, nil)
+
+		/* DELETE */
+		for _, path := range req.GetDelete() {
+			log.V(2).Infof("Delete path: %v", path)
+
+			err := dc.Set(path, nil, sdc.DELETE)
+
+			if err != nil {
+				return nil, err
+			}
+
+			res := gnmipb.UpdateResult{
+							Path: path,
+	      						Op:   gnmipb.UpdateResult_DELETE,
+     			    		          }
+
+			/* Add to Set response results. */
+     			results = append(results, &res)
+
+		}
+
+		/* REPLACE */
+		for _, path := range req.GetReplace(){
+			log.V(2).Infof("Replace path: %v ", path)
+
+			err = dc.Set(path.GetPath(), path.GetVal(), sdc.REPLACE)
+
+			if err != nil {
+				return nil, err
+			}
+			res := gnmipb.UpdateResult{
+							Path: path.GetPath(),
+	      						Op:   gnmipb.UpdateResult_REPLACE,
+    				                  }
+			/* Add to Set response results. */
+     			results = append(results, &res)
+		}
+
+
+		/* UPDATE */
+		for _, path := range req.GetUpdate(){
+			log.V(2).Infof("Update path: %v ", path)
+
+			err = dc.Set(path.GetPath(), path.GetVal(), sdc.UPDATE)
+
+			if err != nil {
+				return nil, err
+			}
+
+			res := gnmipb.UpdateResult{
+							Path: path.GetPath(),
+	      						Op:   gnmipb.UpdateResult_UPDATE,
+     					          }
+			/* Add to Set response results. */
+     			results = append(results, &res)
+		}
+
+
+
+	return &gnmipb.SetResponse{
+ 					Prefix:   req.GetPrefix(),
+		  			Response: results,
+				  }, nil
+
 }
 
 // Capabilities method is not implemented. Refer to gnxi for examples with openconfig integration
 func (srv *Server) Capabilities(context.Context, *gnmipb.CapabilityRequest) (*gnmipb.CapabilityResponse, error) {
-	return nil, grpc.Errorf(codes.Unimplemented, "Capabilities() is not implemented")
+
+	dc, _ := sdc.NewTranslClient(nil , nil)
+
+		/* Fetch the client capabitlities. */
+		supportedModels := dc.Capabilities()
+		suppModels := make([]*gnmipb.ModelData, len(supportedModels))
+
+		for index, model := range supportedModels {
+			suppModels[index] = &gnmipb.ModelData{
+						    	     	Name: model.Name, 
+								Organization: model.Organization, 
+								Version: model.Version,
+			}
+		}
+
+	return &gnmipb.CapabilityResponse{SupportedModels: suppModels, 
+				 	  SupportedEncodings: supportedEncodings,
+					  GNMIVersion: "0.7.0"}, nil
 }
+
+func  isTargetDb ( target string) (bool) {
+	isDbClient := false 
+	dbTargetSupported := []string { "APPL_DB", "ASIC_DB" , "COUNTERS_DB", "LOGLEVEL_DB", "CONFIG_DB", "PFC_WD_DB", "FLEX_COUNTER_DB", "STATE_DB"}
+
+	    for _, name := range  dbTargetSupported {
+		    if  target ==  name {
+			    isDbClient = true
+				    return isDbClient
+		    }
+	    }
+
+	    return isDbClient
+}
+