Add gRPC service definition and gRPC gateway

proto/Makefile

@@ -0,0 +1,19 @@
+# Makefile to generate api clients from proto.
+
+OUTPUT_MODE=import
+TMP_OUTPUT=/tmp
+
+.PHONY: generate
+generate:
+	mkdir -p go_client && mkdir -p swagger && \
+	protoc -I ./ \
+		-I ./third_party/ --go_out ${TMP_OUTPUT} --go_opt paths=${OUTPUT_MODE} \
+		--go-grpc_out ${TMP_OUTPUT} --go-grpc_opt paths=${OUTPUT_MODE} \
+		--grpc-gateway_out ${TMP_OUTPUT}  --grpc-gateway_opt paths=${OUTPUT_MODE} \
+		--openapiv2_opt logtostderr=true --openapiv2_out=:swagger ./*.proto && \
+	cp ${TMP_OUTPUT}/github.com/ray-project/kuberay/proto/go_client/* ./go_client
+
+.PHONY: clean
+clean:
+	rm -rf go_client && rm -rf swagger
+

proto/README.md

@@ -0,0 +1,65 @@
+# ProtoBuf Definitions
+
+This folder contains api definition written in protocol buffer. The api will be used for building services.
+
+## Prerequisite
+
+In order to run generation commands successfully, please make sure you have following tools installed in your environment. 
+
+- [Make](https://man7.org/linux/man-pages/man1/make.1.html)
+- [Docker](https://www.docker.com/)
+
+Currently, docker isnot being used for code generation. This is in the roadmap.
+User still have to install protoc, etc to successfully generate the code.
+
+```
+# 1. Install Protocol Buffer
+
+$ brew install protobuf
+$ protoc --version  # Ensure compiler version is 3+
+
+# 2. Install go plugin for protoc and grpc
+
+$ go get -u google.golang.org/grpc
+$ go get -u github.com/golang/protobuf/protoc-gen-go
+
+# 3. Install grpc gateway and openapi plugin for protoc 
+
+$ go get -u github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway
+$ go get -u github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 
+
+```
+
+## Auto Generation
+
+### Go Client and Swagger
+
+```
+make generate
+```
+
+### Clients
+
+We postpone the client generation until there's external service needs for communication. 
+
+### API reference documentation
+
+Use the tools [bootprint-openapi](https://github.com/bootprint/bootprint-monorepo/tree/master/packages/bootprint-openapi) 
+and [html-inline](https://github.com/substack/html-inline) to generate the API reference from the swagger files.
+
+### Third Party Protos
+
+Third party proto dependencies are sync back to `api/third_party` for easy development (IDE friendly).
+Actually, we should specify the directory in which to search for imports instead.
+
+```
+protoc -I. 
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/ \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options/ \
+``` 
+
+Source: 
+- [googleapis](https://github.com/googleapis/googleapis/tree/master/google/api)
+- [protoc-gen-openapiv2](https://github.com/grpc-ecosystem/grpc-gateway/tree/master/protoc-gen-openapiv2/options)
+

proto/cluster.proto

@@ -3,8 +3,94 @@ syntax = "proto3";
 option go_package = "github.com/ray-project/kuberay/proto/go_client";
 package proto;
 
+import "google/api/annotations.proto";
+import "google/protobuf/empty.proto";
 import "google/protobuf/timestamp.proto";
+import "protoc-gen-openapiv2/options/annotations.proto";
 
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
+  schemes: HTTP;
+  responses: {
+    key: "default";
+    value: {
+      schema: {
+        json_schema: {
+          ref: ".api.Status";
+        }
+      }
+    }
+  }
+};
+
+service ClusterService {
+  // Creates a new Cluster.
+  rpc CreateCluster(CreateClusterRequest) returns (Cluster) {
+    option (google.api.http) = {
+      post: "/apis/v1alpha1/clusters"
+      body: "cluster"
+    };
+  }
+
+  // Finds a specific Cluster by ID.
+  rpc GetCluster(GetClusterRequest) returns (Cluster) {
+    option (google.api.http) = {
+      get: "/apis/v1alpha1/clusters/{id}"
+    };
+  }
+
+  // Finds all Clusters. Supports pagination, and sorting on certain fields.
+  rpc ListCluster(ListClustersRequest) returns (ListClustersResponse) {
+    option (google.api.http) = {
+      get: "/apis/v1alpha1/clusters"
+    };
+  }
+
+  // Deletes an cluster without deleting the cluster's runs and jobs. To
+  // avoid unexpected behaviors, delete an cluster's runs and jobs before
+  // deleting the cluster.
+  rpc DeleteCluster(DeleteClusterRequest) returns (google.protobuf.Empty) {
+    option (google.api.http) = {
+      delete: "/apis/v1alpha1/clusters/{id}"
+    };
+  }
+}
+
+message CreateClusterRequest {
+  // The Cluster to be created.
+  Cluster cluster = 1;
+}
+
+message GetClusterRequest {
+  // The ID of the Cluster to be retrieved.
+  string id = 1;
+  // The name of the Cluster to be retrieved.
+  string name = 2;
+}
+
+message ListClustersRequest {
+  // A page token to request the next page of results. The token is acquried
+  // from the nextPageToken field of the response from the previous
+  // ListCluster call or can be omitted when fetching the first page.
+  // TODO: support this later
+}
+
+message ListClustersResponse {
+  // A list of Clusters returned.
+  repeated Cluster clusters = 1;
+
+  // The total number of Clusters for the given query.
+  // int32 total_size = 2;
+
+  // The token to list the next page of Clusters.
+  // string next_page_token = 3;
+}
+
+message DeleteClusterRequest {
+  // The ID of the cluster to be deleted.
+  string id = 1;
+  // The name of the Cluster to be retrieved.
+  string name = 2;
+}
 
 message Cluster {
   // Required input field. Unique Cluster name provided by user.

proto/config.proto

@@ -3,18 +3,159 @@ syntax = "proto3";
 option go_package = "github.com/ray-project/kuberay/proto/go_client";
 package proto;
 
+import "google/api/annotations.proto";
+import "google/protobuf/empty.proto";
+import "protoc-gen-openapiv2/options/annotations.proto";
+
+
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
+  schemes: HTTP;
+  responses: {
+    key: "default";
+    value: {
+      schema: {
+        json_schema: {
+          ref: ".api.Status";
+        }
+      }
+    }
+  }
+};
+
+service ComputeTemplateService {
+  // Creates a new Cluster.
+  rpc CreateComputeTemplate(CreateComputeTemplateRequest) returns (ComputeTemplate) {
+    option (google.api.http) = {
+      post: "/apis/v1alpha1/compute_templates"
+      body: "compute_template"
+    };
+  }
+
+  // Finds a specific Cluster by ID.
+  rpc GetComputeTemplate(GetComputeTemplateRequest) returns (ComputeTemplate) {
+    option (google.api.http) = {
+      get: "/apis/v1alpha1/compute_templates/{id}"
+    };
+  }
+
+  // Finds all Clusters. Supports pagination, and sorting on certain fields.
+  rpc ListComputeTemplates(ListComputeTemplatesRequest) returns (ListComputeTemplatesResponse) {
+    option (google.api.http) = {
+      get: "/apis/v1alpha1/compute_templates"
+    };
+  }
+
+  // Deletes an Cluster without deleting the Cluster's runs and jobs. To
+  // avoid unexpected behaviors, delete an Cluster's runs and jobs before
+  // deleting the Cluster.
+  rpc DeleteComputeTemplate(DeleteComputeTemplateRequest) returns (google.protobuf.Empty) {
+    option (google.api.http) = {
+      delete: "/apis/v1alpha1/compute_templates/{id}"
+    };
+  }
+}
+
+message CreateComputeTemplateRequest {
+  // The ComputeTemplate to be created.
+  ComputeTemplate compute_template = 1;
+}
+
+message GetComputeTemplateRequest {
+  // The ID of the ComputeTemplate to be retrieved. reserved field for database layer query later
+  string id = 1;
+  // The name of the ComputeTemplate to be retrieved.
+  string name = 2;
+}
+
+message ListComputeTemplatesRequest {
+  // TODO: support paganation later
+}
+
+message ListComputeTemplatesResponse {
+  repeated ComputeTemplate compute_templates = 1;
+}
+
+message DeleteComputeTemplateRequest {
+  // The ID of the ComputeTemplate to be deleted.
+  string id = 1;
+  // The name of the ComputeTemplate to be deleted.
+  string name = 2;
+}
+
 // ComputeTemplate can be reused by any compute units like worker group, workspace, image build job, etc
 message ComputeTemplate {
   // The ID of the compute template
-  string name = 1;
+  string id = 1;
+  // The name of the compute template
+  string name = 2;
   // Number of cpus
-  uint32 cpu = 2;
+  uint32 cpu = 3;
   // Number of memory
-  uint32 memory = 3;
+  uint32 memory = 4;
   // Number of gpus
-  uint32 gpu = 4;
+  uint32 gpu = 5;
   // The detail gpu accelerator type
-  string gpu_accelerator = 5;
+  string gpu_accelerator = 6;
+}
+
+
+service ImageTemplateService {
+  // Creates a new ImageTemplate.
+  rpc CreateImageTemplate(CreateImageTemplateRequest) returns (ImageTemplate) {
+    option (google.api.http) = {
+      post: "/apis/v1alpha1/image_templates"
+      body: "image_template"
+    };
+  }
+
+  // Finds a specific ImageTemplate by ID.
+  rpc GetImageTemplate(GetImageTemplateRequest) returns (ImageTemplate) {
+    option (google.api.http) = {
+      get: "/apis/v1alpha1/image_templates/{id}"
+    };
+  }
+
+  // Finds all ImageTemplates. Supports pagination, and sorting on certain fields.
+  rpc ListImageTemplates(ListImageTemplatesRequest) returns (ListImageTemplatesResponse) {
+    option (google.api.http) = {
+      get: "/apis/v1alpha1/image_templates"
+    };
+  }
+
+  // Deletes an ImageTemplate.
+  rpc DeleteImageTemplate(DeleteImageTemplateRequest) returns (google.protobuf.Empty) {
+    option (google.api.http) = {
+      delete: "/apis/v1alpha1/image_templates/{id}"
+    };
+  }
+}
+
+message CreateImageTemplateRequest {
+  // The Compute to be created.
+  ImageTemplate image_template = 1;
+}
+
+message GetImageTemplateRequest {
+  // The ID of the ImageTemplate to be retrieved. reserved field for database layer query later
+  string id = 1;
+  // The name of the ImageTemplate to be retrieved.
+  string name = 2;
+}
+
+message ListImageTemplatesRequest {
+  // TODO: support pagingation later
+}
+
+message ListImageTemplatesResponse {
+  // A list of Compute returned.
+  repeated ImageTemplate image_templates = 1;
+}
+
+message DeleteImageTemplateRequest {
+  // The ID of the ImageTemplate to be deleted.
+  string id = 1;
+  // The name of the ImageTemplate to be delete.
+  string name = 2;
 }
 
 // ImageTemplate can be used by worker group and workspce.

proto/go.mod

@@ -0,0 +1,18 @@
+module github.com/ray-project/kuberay/proto
+
+go 1.17
+
+require (
+	github.com/golang/protobuf v1.5.2
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0
+	google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af
+	google.golang.org/grpc v1.40.0
+	google.golang.org/protobuf v1.27.1
+)
+
+require (
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+	golang.org/x/text v0.3.5 // indirect
+)