feat: initial grpc support

.circleci/config.yml

@@ -109,9 +109,14 @@ commands:
     steps:
       - run: echo "${DOCKER_PASS}" | docker login --username ${DOCKER_USER} --password-stdin
 
+  install-protoc:
+    steps:
+      - run: curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.12.1/protoc-3.12.1-linux-x86_64.zip
+      - run: unzip protoc-3.12.1-linux-x86_64.zip -d $HOME/.local
 
   go-get-deps:
     steps:
+      - install-protoc
       - restore_cache:
           keys:
             - det-go-deps-v1dev6-{{ checksum  "master/go.sum" }}-{{ checksum  "agent/go.sum" }}

CONTRIBUTING.md

@@ -46,6 +46,7 @@ git clone [email protected]:determined-ai/determined.git
 - Node (>= 12)
 - Yarn (>= 1.22.0)
 - Docker (>= 19.03)
+- Protoc (>= 3.0)
 
 ### Building Determined
 

agent/go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e
+	golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f
 	golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770
 	gotest.tools v2.2.0+incompatible // indirect
 )

master/.gitignore

@@ -19,3 +19,7 @@ vendor/
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+# Proto generated files
+pkg/proto
+static/swagger-spec/

master/.goreleaser.yml

@@ -3,6 +3,7 @@ project_name: determined-master
 before:
   hooks:
     - make pre-package
+    - make proto
 
 snapshot:
   name_template: "{{ .Tag }}"

master/Makefile

@@ -5,31 +5,51 @@ export GO111MODULE := on
 clean:
 	rm -rf dist
 	rm -rf build
+	rm -rf pkg/proto
+	rm -rf static/swagger-spec/
 
 .PHONY: get-deps
 get-deps:
 	go mod download
 	go install github.com/golangci/golangci-lint/cmd/golangci-lint
 	go install golang.org/x/tools/cmd/goimports
 	go install github.com/goreleaser/goreleaser
+	go install github.com/bufbuild/buf/cmd/buf
+	go install github.com/golang/protobuf/protoc-gen-go
+	go install google.golang.org/grpc
+	go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
+	go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
+
+%.pb.go: %.proto
+	@mkdir -p build/proto
+	protoc -I proto $< --go_out=plugins=grpc:build/proto
+
+.PHONY: proto
+proto: $(patsubst %.proto, %.pb.go, $(shell find proto -type f -name '*.proto'))
+	rm -rf pkg/proto
+	protoc -I proto proto/api/v1/api.proto --grpc-gateway_out=logtostderr=true:build/proto
+	cp -r build/proto/github.com/determined-ai/determined/master/pkg/proto/ ./pkg/proto
+	@mkdir -p static/swagger-spec/
+	protoc -I proto proto/api/v1/api.proto --swagger_out=logtostderr=true:static/swagger-spec/
 
 .PHONY: build
-build:
+build: proto
 	go build \
 		-ldflags "-X github.com/determined-ai/determined/master/version.Version=$(VERSION)" \
 		-o build/determined-master \
 		./cmd/determined-master
 
 .PHONY: check
-check:
+check: proto
+	buf check lint
 	golangci-lint run
 
 .PHONY: fmt
 fmt:
 	goimports -l -local github.com/determined-ai -w .
 
 .PHONY: test
-test:
+test: proto
 	go test -v -short -coverprofile=coverage.out -covermode count -cover ./...
 
 .PHONY: pre-package

master/buf.yaml

@@ -0,0 +1,13 @@
+build:
+  roots:
+    - proto
+
+lint:
+  use:
+    - DEFAULT
+    - COMMENTS
+  except:
+    - SERVICE_SUFFIX
+  ignore:
+    - google
+    - protoc-gen-swagger

master/cmd/determined-master/init.go

@@ -106,6 +106,8 @@ func registerConfig() {
 	registerString(flags, name("security", "tls", "key"),
 		defaults.Security.TLS.Key, "TLS key file")
 
+	registerInt(flags, name("grpc-port"),
+		defaults.GRPCPort, "GRPC server port")
 	registerInt(flags, name("http-port"),
 		defaults.HTTPPort, "HTTP server port")
 	registerInt(flags, name("https-port"),

master/go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Microsoft/go-winio v0.4.9 // indirect
 	github.com/aws/aws-sdk-go v1.25.11
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
+	github.com/bufbuild/buf v0.12.1 // indirect
 	github.com/containerd/containerd v1.3.2 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1
@@ -15,12 +16,15 @@ require (
 	github.com/emirpasic/gods v1.12.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/golang-migrate/migrate v3.5.4+incompatible
+	github.com/golang/protobuf v1.4.2
 	github.com/golangci/golangci-lint v1.27.0
-	github.com/google/go-cmp v0.3.1
+	github.com/google/go-cmp v0.4.0
 	github.com/google/uuid v1.1.1
 	github.com/goreleaser/goreleaser v0.133.0
 	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/gorilla/websocket v1.4.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
+	github.com/grpc-ecosystem/grpc-gateway v1.9.2
 	github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5
 	github.com/labstack/echo v3.3.5+incompatible
 	github.com/labstack/gommon v0.0.0-20180613044413-d6898124de91
@@ -37,10 +41,13 @@ require (
 	github.com/spf13/viper v1.6.1
 	github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 // indirect
 	github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
-	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
-	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b
+	golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37
+	golang.org/x/net v0.0.0-20200506145744-7e3656a0809f
 	golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770
 	google.golang.org/api v0.9.0
+	google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380
+	google.golang.org/grpc v1.28.1
+	google.golang.org/protobuf v1.23.0
 	gopkg.in/guregu/null.v3 v3.4.0
 	gopkg.in/segmentio/analytics-go.v3 v3.1.0
 	gotest.tools v2.1.0+incompatible