feat: persistent storage

Signed-off-by: Artur Troian <[email protected]>

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/hashicorp/hcl v1.0.1-0.20191016231534-914dc3f8dd7c // indirect
 	github.com/jmhodges/levigo v1.0.1-0.20191019112844-b572e7f4cdac // indirect
 	github.com/libp2p/go-buffer-pool v0.0.3-0.20190619091711-d94255cb3dfc // indirect
-	github.com/moby/term v0.0.0-20200312100748-672ec06f55cd
+	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.8.0
@@ -41,12 +41,12 @@ require (
 	google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f
 	google.golang.org/grpc v1.35.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
-	k8s.io/api v0.19.3
-	k8s.io/apimachinery v0.20.2
-	k8s.io/client-go v0.19.3
-	k8s.io/code-generator v0.19.3
-	k8s.io/kubectl v0.19.3
-	k8s.io/metrics v0.19.3
+	k8s.io/api v0.21.3
+	k8s.io/apimachinery v0.21.3
+	k8s.io/client-go v0.21.3
+	k8s.io/code-generator v0.21.3
+	k8s.io/kubectl v0.21.3
+	k8s.io/metrics v0.21.3
 	sigs.k8s.io/kind v0.11.1
 )
 

manifest/types.go

@@ -58,7 +58,7 @@ type Service struct {
 	Expose    []ServiceExpose
 }
 
-// GetResourcesUnit returns resources unit of service
+// GetResourceUnits returns resources unit of service
 func (s Service) GetResourceUnits() types.ResourceUnits {
 	return s.Resources
 }

pkg/apis/akash.network/v1/crd.yaml

@@ -68,8 +68,24 @@ spec:
                                 type: string
                                 format: uint64
                               storage:
-                                type: string
-                                format: uint64
+                                type: array
+                                items:
+                                  type: object
+                                  properties:
+                                    id:
+                                      type: string
+                                    size:
+                                      type: string
+                                      format: uint64
+                                    attributes:
+                                      type: array
+                                      items:
+                                        type: object
+                                        properties:
+                                          key:
+                                            type: string
+                                          value:
+                                            type: string
                           count:
                             type: number
                             format: uint64
@@ -94,4 +110,34 @@ spec:
                                   type: array
                                   items:
                                     type: string
-
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: storageclass.akash.network
+spec:
+  group: akash.network
+  scope: Cluster
+  names:
+    plural: storageclassesstate
+    singular: storageclassstate
+    kind: StorageClassState
+    shortNames:
+      - scs
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                capacity:
+                  type: string
+                  format: uint64
+                available:
+                  type: string
+                  format: uint64

pkg/apis/akash.network/v1/register.go

@@ -25,6 +25,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&Manifest{},
 		&ManifestList{},
+		&StorageClassState{},
+		&StorageClassStateList{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

pkg/apis/akash.network/v1/types.go

@@ -40,7 +40,30 @@ type ManifestSpec struct {
 	Group   ManifestGroup `json:"group"`
 }
 
-// type ResourceUnits types.ResourceUnits
+// +genclient
+// +genclient:nonNamespaced
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type StorageClassState struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard object's metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata"`
+
+	Spec   StorageClassStateSpec
+	Status StorageClassStateStatus
+}
+
+type StorageClassStateStatus struct {
+	State   string `json:"state,omitempty"`
+	Message string `json:"message,omitempty"`
+}
+
+type StorageClassStateSpec struct {
+	Capacity  uint64 `json:"capacity"`
+	Available uint64 `json:"available"`
+}
 
 // Deployment returns the cluster.Deployment that the saved manifest represents.
 func (m Manifest) Deployment() (ctypes.Deployment, error) {
@@ -91,6 +114,23 @@ func NewManifest(name string, lid mtypes.LeaseID, mgroup *manifest.Group) (*Mani
 	}, nil
 }
 
+// NewStorageClassState creates new storage class state with provided details. Returns error in case of failure.
+func NewStorageClassState(name string, capacity uint64, available uint64) (*StorageClassState, error) {
+	return &StorageClassState{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "StorageClassState",
+			APIVersion: "akash.network/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: StorageClassStateSpec{
+			Capacity:  capacity,
+			Available: available,
+		},
+	}, nil
+}
+
 // LeaseID stores deployment, group sequence, order, provider and metadata
 type LeaseID struct {
 	Owner    string `json:"owner"`
@@ -344,3 +384,12 @@ type ManifestList struct {
 	metav1.ListMeta `json:",inline"`
 	Items           []Manifest `json:"items"`
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// StorageClassStateList stores metadata and items list of storage class states
+type StorageClassStateList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:",inline"`
+	Items           []StorageClassState `json:"items"`
+}

proto/akash/base/v1beta1/endpoint.proto

@@ -11,9 +11,9 @@ message Endpoint {
   // This describes how the endpoint is implemented when the lease is deployed
   enum Kind {
     // Describes an endpoint that becomes a Kubernetes Ingress
-  	SHARED_HTTP = 0;
-  	// Describes an endpoint that becomes a Kubernetes NodePort
-  	RANDOM_PORT = 1;
+    SHARED_HTTP = 0;
+    // Describes an endpoint that becomes a Kubernetes NodePort
+    RANDOM_PORT = 1;
   }
   Kind kind = 1;
 }

proto/akash/base/v1beta1/resource.proto

@@ -12,10 +12,11 @@ option go_package = "github.com/ovrclk/akash/types";
 message CPU {
   option (gogoproto.equal)      = true;
   ResourceValue      units      = 1 [(gogoproto.nullable) = false];
-  repeated Attribute attributes = 2 [
+  repeated akash.base.v1beta1.Attribute attributes = 2 [
     (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "Attributes",
     (gogoproto.jsontag)  = "attributes,omitempty",
-    (gogoproto.moretags) = "yaml:\"cpu,omitempty\""
+    (gogoproto.moretags) = "yaml:\"attributes,omitempty\""
   ];
 }
 
@@ -24,22 +25,31 @@ message Memory {
   option (gogoproto.equal) = true;
   ResourceValue quantity   = 1
       [(gogoproto.nullable) = false, (gogoproto.jsontag) = "size", (gogoproto.moretags) = "yaml:\"size\""];
-  repeated Attribute attributes = 2 [
+  repeated akash.base.v1beta1.Attribute attributes = 2 [
     (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "Attributes",
     (gogoproto.jsontag)  = "attributes,omitempty",
-    (gogoproto.moretags) = "yaml:\"cpu,omitempty\""
+    (gogoproto.moretags) = "yaml:\"attributes,omitempty\""
   ];
 }
 
 // Storage stores resource quantity and storage attributes
 message Storage {
   option (gogoproto.equal) = true;
-  ResourceValue quantity   = 1
-      [(gogoproto.nullable) = false, (gogoproto.jsontag) = "size", (gogoproto.moretags) = "yaml:\"size\""];
-  repeated Attribute attributes = 2 [
+  string name = 1 [
+    (gogoproto.jsontag) = "name",
+    (gogoproto.moretags) = "yaml:\"name\""
+  ];
+  ResourceValue quantity = 2 [
     (gogoproto.nullable) = false,
-    (gogoproto.jsontag)  = "attributes,omitempty",
-    (gogoproto.moretags) = "yaml:\"cpu,omitempty\""
+    (gogoproto.jsontag) = "size",
+    (gogoproto.moretags) = "yaml:\"size\""
+  ];
+  repeated akash.base.v1beta1.Attribute attributes = 3 [
+    (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "Attributes",
+    (gogoproto.jsontag) = "attributes,omitempty",
+    (gogoproto.moretags) = "yaml:\"attributes,omitempty\""
   ];
 }
 
@@ -58,11 +68,16 @@ message ResourceUnits {
     (gogoproto.jsontag)  = "memory,omitempty",
     (gogoproto.moretags) = "yaml:\"memory,omitempty\""
   ];
-  Storage storage = 3 [
-    (gogoproto.nullable) = true,
+  repeated Storage storage = 3 [
+    (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "Volumes",
     (gogoproto.jsontag)  = "storage,omitempty",
     (gogoproto.moretags) = "yaml:\"storage,omitempty\""
   ];
-  repeated akash.base.v1beta1.Endpoint endpoints = 4
-      [(gogoproto.nullable) = false, (gogoproto.jsontag) = "endpoints", (gogoproto.moretags) = "yaml:\"endpoints\""];
+  repeated akash.base.v1beta1.Endpoint endpoints = 4 [
+    (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "Endpoints",
+    (gogoproto.jsontag) = "endpoints",
+    (gogoproto.moretags) = "yaml:\"endpoints\""
+  ];
 }

provider/bidengine/config.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
+
 	"github.com/ovrclk/akash/types"
 )
 

provider/bidengine/order.go

@@ -6,21 +6,24 @@ import (
 	"regexp"
 	"time"
 
-	lifecycle "github.com/boz/go-lifecycle"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	atypes "github.com/ovrclk/akash/x/audit/types"
+
+	"github.com/boz/go-lifecycle"
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/tendermint/tendermint/libs/log"
+
 	"github.com/ovrclk/akash/provider/cluster"
 	ctypes "github.com/ovrclk/akash/provider/cluster/types"
 	"github.com/ovrclk/akash/provider/event"
 	"github.com/ovrclk/akash/provider/session"
 	"github.com/ovrclk/akash/pubsub"
 	metricsutils "github.com/ovrclk/akash/util/metrics"
 	"github.com/ovrclk/akash/util/runner"
-	atypes "github.com/ovrclk/akash/x/audit/types"
 	dtypes "github.com/ovrclk/akash/x/deployment/types"
 	mtypes "github.com/ovrclk/akash/x/market/types"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/tendermint/tendermint/libs/log"
 )
 
 // order manages bidding and general lifecycle handling of an order.
@@ -79,6 +82,7 @@ var (
 func newOrder(svc *service, oid mtypes.OrderID, cfg Config, pass ProviderAttrSignatureService, checkForExistingBid bool) (*order, error) {
 	return newOrderInternal(svc, oid, cfg, pass, checkForExistingBid, nil)
 }
+
 func newOrderInternal(svc *service, oid mtypes.OrderID, cfg Config, pass ProviderAttrSignatureService, checkForExistingBid bool, reservationFulfilledNotify chan<- int) (*order, error) {
 	// Create a subscription that will see all events that have not been read from e.sub.Events()
 	sub, err := svc.sub.Clone()
@@ -408,7 +412,6 @@ loop:
 }
 
 func (o *order) shouldBid(group *dtypes.Group) (bool, error) {
-
 	// does provider have required attributes?
 	if !group.GroupSpec.MatchAttributes(o.session.Provider().Attributes) {
 		o.log.Debug("unable to fulfill: incompatible provider attributes")
@@ -421,6 +424,17 @@ func (o *order) shouldBid(group *dtypes.Group) (bool, error) {
 		return false, nil
 	}
 
+	attr, err := o.pass.GetAttributes()
+	if err != nil {
+		return false, err
+	}
+
+	// does provider have required capabilities?
+	if !group.GroupSpec.MatchResourcesRequirements(attr) {
+		o.log.Debug("unable to fulfill: incompatible attributes for resources requirements")
+		return false, nil
+	}
+
 	signatureRequirements := group.GroupSpec.Requirements.SignedBy
 	if signatureRequirements.Size() != 0 {
 		// Check that the signature requirements are met for each attribute

provider/bidengine/order_test.go

@@ -3,12 +3,12 @@ package bidengine
 import (
 	"context"
 	"errors"
+	"testing"
 	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
-	"github.com/ovrclk/akash/sdkutil"
 
-	"testing"
+	"github.com/ovrclk/akash/sdkutil"
 
 	"github.com/stretchr/testify/mock"
 
@@ -57,13 +57,16 @@ func makeMocks(s *orderTestScaffold) {
 	memory := atypes.Memory{}
 	memory.Quantity = atypes.NewResourceValue(dtypes.GetValidationConfig().MinUnitMemory)
 
-	storage := atypes.Storage{}
-	storage.Quantity = atypes.NewResourceValue(dtypes.GetValidationConfig().MinUnitStorage)
+	storage := atypes.Volumes{
+		atypes.Storage{
+			Quantity: atypes.NewResourceValue(dtypes.GetValidationConfig().MinUnitStorage),
+		},
+	}
 
 	clusterResources := atypes.ResourceUnits{
 		CPU:     &cpu,
 		Memory:  &memory,
-		Storage: &storage,
+		Storage: storage,
 	}
 	price := sdk.NewInt64Coin(testutil.CoinDenom, 23)
 	resource := dtypes.Resource{
@@ -114,6 +117,10 @@ func (nullProviderAttrSignatureService) GetAuditorAttributeSignatures(auditor st
 	return nil, nil // Return no attributes & no error
 }
 
+func (nullProviderAttrSignatureService) GetAttributes() (atypes.Attributes, error) {
+	return nil, nil // Return no attributes & no error
+}
+
 func makeOrderForTest(t *testing.T, checkForExistingBid bool, pricing BidPricingStrategy, callerConfig *Config) (*order, orderTestScaffold, <-chan int) {
 	if pricing == nil {
 		var err error