feat: persistent storage

Signed-off-by: Artur Troian <[email protected]>

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/hashicorp/hcl v1.0.1-0.20191016231534-914dc3f8dd7c // indirect
 	github.com/jmhodges/levigo v1.0.1-0.20191019112844-b572e7f4cdac // indirect
 	github.com/libp2p/go-buffer-pool v0.0.3-0.20190619091711-d94255cb3dfc // indirect
-	github.com/moby/term v0.0.0-20200312100748-672ec06f55cd
+	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.11.0
@@ -41,12 +41,12 @@ require (
 	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
 	google.golang.org/grpc v1.38.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
-	k8s.io/api v0.19.3
-	k8s.io/apimachinery v0.20.2
-	k8s.io/client-go v0.19.3
-	k8s.io/code-generator v0.19.3
-	k8s.io/kubectl v0.19.3
-	k8s.io/metrics v0.19.3
+	k8s.io/api v0.21.3
+	k8s.io/apimachinery v0.21.3
+	k8s.io/client-go v0.21.3
+	k8s.io/code-generator v0.21.3
+	k8s.io/kubectl v0.21.3
+	k8s.io/metrics v0.21.3
 	sigs.k8s.io/kind v0.11.1
 )
 

integration/e2e_test.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	sdktest "github.com/cosmos/cosmos-sdk/testutil"
+
 	"github.com/ovrclk/akash/provider/gateway/rest"
 
 	"github.com/cosmos/cosmos-sdk/server"
@@ -985,7 +986,7 @@ func (s *E2EDeploymentUpdate) TestE2ELeaseShell() {
 
 func TestIntegrationTestSuite(t *testing.T) {
 	integrationTestOnly(t)
-	suite.Run(t, new(E2EContainerToContainer))
+	// suite.Run(t, new(E2EContainerToContainer))
 	suite.Run(t, new(E2EAppNodePort))
 	suite.Run(t, new(E2EDeploymentUpdate))
 	suite.Run(t, new(E2EApp))
@@ -1012,6 +1013,6 @@ func TestQueryApp(t *testing.T) {
 	integrationTestOnly(t)
 	host, appPort := appEnv(t)
 
-	appURL := fmt.Sprintf("https://%s:%s/", host, appPort)
+	appURL := fmt.Sprintf("http://%s:%s/", host, appPort)
 	queryApp(t, appURL, 1)
 }

integration/test_helpers.go

@@ -52,14 +52,14 @@ func queryAppWithRetries(t *testing.T, appURL string, appHost string, limit int)
 			DualStack: false,
 		}).DialContext,
 	}
-	client := &http.Client{
+	httpClient := &http.Client{
 		Transport: tr,
 	}
 
 	var resp *http.Response
 	const delay = 1 * time.Second
 	for i := 0; i != limit; i++ {
-		resp, err = client.Do(req)
+		resp, err = httpClient.Do(req)
 		if resp != nil {
 			t.Log("GET: ", appURL, resp.StatusCode)
 		}
@@ -94,15 +94,15 @@ func queryAppWithHostname(t *testing.T, appURL string, limit int, hostname strin
 	tr := &http.Transport{
 		DisableKeepAlives: false,
 	}
-	client := &http.Client{
+	httpClient := &http.Client{
 		Transport: tr,
 	}
 
 	// retry mechanism
 	var resp *http.Response
 	for i := 0; i < limit; i++ {
 		time.Sleep(1 * time.Second) // reduce absurdly long wait period
-		resp, err = client.Do(req)
+		resp, err = httpClient.Do(req)
 		if err != nil {
 			t.Log(err)
 			continue
@@ -112,13 +112,13 @@ func queryAppWithHostname(t *testing.T, appURL string, limit int, hostname strin
 			break
 		}
 	}
-	assert.NoError(t, err)
-	assert.NotNil(t, resp)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	require.NoError(t, err)
+	require.NotNil(t, resp)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
 
-	bytes, err := ioutil.ReadAll(resp.Body)
-	assert.NoError(t, err)
-	assert.Contains(t, string(bytes), "The Future of The Cloud is Decentralized")
+	body, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Contains(t, string(body), "The Future of The Cloud is Decentralized")
 }
 
 // appEnv asserts that there is an addressable docker container for KinD

manifest/types.go

@@ -46,6 +46,16 @@ func (g Group) GetResources() []types.Resources {
 	return resources
 }
 
+type StorageParams struct {
+	Name     string `json:"name"`
+	Mount    string `json:"readOnly"`
+	ReadOnly bool   `json:"mount"`
+}
+
+type ServiceParams struct {
+	Storage []StorageParams
+}
+
 // Service stores name, image, args, env, unit, count and expose list of service
 type Service struct {
 	Name      string
@@ -56,9 +66,10 @@ type Service struct {
 	Resources types.ResourceUnits
 	Count     uint32
 	Expose    []ServiceExpose
+	Params    *ServiceParams
 }
 
-// GetResourcesUnit returns resources unit of service
+// GetResourceUnits returns resources unit of service
 func (s Service) GetResourceUnits() types.ResourceUnits {
 	return s.Resources
 }

pkg/apis/akash.network/v1/crd.yaml

@@ -68,8 +68,10 @@ spec:
                                 type: string
                                 format: uint64
                               storage:
-                                type: string
-                                format: uint64
+                                type: array
+                                items:
+                                  type: string
+                                  format: uint64
                           count:
                             type: number
                             format: uint64
@@ -94,4 +96,48 @@ spec:
                                   type: array
                                   items:
                                     type: string
-
+                          params:
+                            type: object
+                            nullable: true
+                            properties:
+                              storage:
+                                type: object
+                                nullable: true
+                                properties:
+                                  name:
+                                    type: string
+                                  readOnly:
+                                    type: boolean
+                                  mount:
+                                    type: string
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: storageclassesstate.akash.network
+spec:
+  group: akash.network
+  scope: Cluster
+  names:
+    plural: storageclassesstate
+    singular: storageclassstate
+    kind: StorageClassState
+    shortNames:
+      - scs
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                capacity:
+                  type: string
+                  format: uint64
+                available:
+                  type: string
+                  format: uint64

pkg/apis/akash.network/v1/register.go

@@ -25,6 +25,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&Manifest{},
 		&ManifestList{},
+		&StorageClassState{},
+		&StorageClassStateList{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

pkg/apis/akash.network/v1/types.go

@@ -40,7 +40,30 @@ type ManifestSpec struct {
 	Group   ManifestGroup `json:"group"`
 }
 
-// type ResourceUnits types.ResourceUnits
+// +genclient
+// +genclient:nonNamespaced
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type StorageClassState struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard object's metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata"`
+
+	Spec   StorageClassStateSpec   `json:"spec,omitempty"`
+	Status StorageClassStateStatus `json:"status,omitempty"`
+}
+
+type StorageClassStateStatus struct {
+	State   string `json:"state,omitempty"`
+	Message string `json:"message,omitempty"`
+}
+
+type StorageClassStateSpec struct {
+	Capacity  uint64 `json:"capacity,omitempty"`
+	Available uint64 `json:"available,omitempty"`
+}
 
 // Deployment returns the cluster.Deployment that the saved manifest represents.
 func (m Manifest) Deployment() (ctypes.Deployment, error) {
@@ -91,6 +114,23 @@ func NewManifest(name string, lid mtypes.LeaseID, mgroup *manifest.Group) (*Mani
 	}, nil
 }
 
+// NewStorageClassState creates new storage class state with provided details. Returns error in case of failure.
+func NewStorageClassState(name string, capacity uint64, available uint64) (*StorageClassState, error) {
+	return &StorageClassState{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "StorageClassState",
+			APIVersion: "akash.network/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: StorageClassStateSpec{
+			Capacity:  capacity,
+			Available: available,
+		},
+	}, nil
+}
+
 // LeaseID stores deployment, group sequence, order, provider and metadata
 type LeaseID struct {
 	Owner    string `json:"owner"`
@@ -182,6 +222,10 @@ func manifestGroupFromAkash(m *manifest.Group) (ManifestGroup, error) {
 	return ma, nil
 }
 
+type ManifestServiceParams struct {
+	Storage []manifest.StorageParams `json:"storage,omitempty"`
+}
+
 // ManifestService stores name, image, args, env, unit, count and expose list of service
 type ManifestService struct {
 	// Service name
@@ -197,6 +241,8 @@ type ManifestService struct {
 	Count uint32 `json:"count,omitempty"`
 	// Overlay Network Links
 	Expose []ManifestServiceExpose `json:"expose,omitempty"`
+	// Miscellaneous service parameters
+	Params *ManifestServiceParams `json:"params,omitempty"`
 }
 
 func (ms ManifestService) toAkash() (manifest.Service, error) {
@@ -223,6 +269,14 @@ func (ms ManifestService) toAkash() (manifest.Service, error) {
 		ams.Expose = append(ams.Expose, value)
 	}
 
+	if ms.Params != nil {
+		ams.Params = &manifest.ServiceParams{
+			Storage: make([]manifest.StorageParams, len(ms.Params.Storage)),
+		}
+
+		copy(ams.Params.Storage, ms.Params.Storage)
+	}
+
 	return *ams, nil
 }
 
@@ -246,6 +300,14 @@ func manifestServiceFromAkash(ams manifest.Service) (ManifestService, error) {
 		ms.Expose = append(ms.Expose, manifestServiceExposeFromAkash(expose))
 	}
 
+	if ams.Params != nil {
+		ms.Params = &ManifestServiceParams{
+			Storage: make([]manifest.StorageParams, len(ams.Params.Storage)),
+		}
+
+		copy(ms.Params.Storage, ams.Params.Storage)
+	}
+
 	return ms, nil
 }
 
@@ -344,3 +406,12 @@ type ManifestList struct {
 	metav1.ListMeta `json:",inline"`
 	Items           []Manifest `json:"items"`
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// StorageClassStateList stores metadata and items list of storage class states
+type StorageClassStateList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:",inline"`
+	Items           []StorageClassState `json:"items"`
+}

proto/akash/base/v1beta1/endpoint.proto

@@ -11,9 +11,9 @@ message Endpoint {
   // This describes how the endpoint is implemented when the lease is deployed
   enum Kind {
     // Describes an endpoint that becomes a Kubernetes Ingress
-  	SHARED_HTTP = 0;
-  	// Describes an endpoint that becomes a Kubernetes NodePort
-  	RANDOM_PORT = 1;
+    SHARED_HTTP = 0;
+    // Describes an endpoint that becomes a Kubernetes NodePort
+    RANDOM_PORT = 1;
   }
   Kind kind = 1;
 }