Skip to content

Commit

Permalink
dataset add storageSize (#4178)
Browse files Browse the repository at this point in the history 
Signed-off-by: wangshulei098 <[email protected]>
  • Loading branch information
@wangshulei098
wangshulei098 authored Jul 1, 2024
1 parent a11dab1 commit 2680812
Show file tree
Hide file tree
Showing 4 changed files with 126 additions and 5 deletions.
9 changes: 7 additions & 2 deletions pkg/ddc/efc/create_volume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
volumehelper "github.com/fluid-cloudnative/fluid/pkg/utils/dataset/volume"
"github.com/fluid-cloudnative/fluid/pkg/utils/kubeclient"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

Expand Down Expand Up @@ -82,6 +81,11 @@ func (e *EFCEngine) createPersistentVolumeForRuntime(runtime base.RuntimeInfoInt
if err != nil {
return err
}

storageCapacity, err := utils.GetPVCStorageCapacityOfDataset(e.Client, runtime.GetName(), runtime.GetNamespace())
if err != nil {
return err
}

pvName := runtime.GetPersistentVolumeName()

Expand All @@ -103,8 +107,9 @@ func (e *EFCEngine) createPersistentVolumeForRuntime(runtime base.RuntimeInfoInt
Spec: corev1.PersistentVolumeSpec{
AccessModes: accessModes,
Capacity: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"),
corev1.ResourceName(corev1.ResourceStorage): storageCapacity,
},

StorageClassName: common.FluidStorageClass,
PersistentVolumeSource: corev1.PersistentVolumeSource{
CSI: &corev1.CSIPersistentVolumeSource{
Expand Down
30 changes: 30 additions & 0 deletions pkg/utils/dataset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,17 @@ import (
datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
"github.com/fluid-cloudnative/fluid/pkg/common"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
PVCStorageAnnotation = "pvc.fluid.io/resources.requests.storage"
DefaultStorageCapacity = "100Pi"
)

// GetDataset gets the dataset.
// It returns a pointer to the dataset if successful.
func GetDataset(client client.Client, name, namespace string) (*datav1alpha1.Dataset, error) {
Expand Down Expand Up @@ -71,6 +77,30 @@ func GetAccessModesOfDataset(client client.Client, name, namespace string) (acce

}

func GetPVCStorageCapacityOfDataset(client client.Client, name, namespace string) (storageCapacity resource.Quantity, err error) {
dataset, err := GetDataset(client, name, namespace)
if err != nil {
return storageCapacity, fmt.Errorf("failed to get dataset %s/%s: %w", namespace, name, err)
}
annotations := dataset.GetObjectMeta().GetAnnotations()
if annotations == nil {
storageCapacity = resource.MustParse(DefaultStorageCapacity)
return
}
size := annotations[PVCStorageAnnotation]
if size == "" {
storageCapacity = resource.MustParse(DefaultStorageCapacity)
return
}

storageCapacity, err = resource.ParseQuantity(size)
if err != nil {
log.Info("failed to parse storage capacity '%s', using default '%s': %v\n", size, DefaultStorageCapacity, err)
return resource.MustParse(DefaultStorageCapacity), nil
}
return
}

// IsTargetPathUnderFluidNativeMounts checks if targetPath is a subpath under some given native mount point.
// We check this for the reason that native mount points need extra metadata sync alluxioOperations.
func IsTargetPathUnderFluidNativeMounts(targetPath string, dataset datav1alpha1.Dataset) bool {
Expand Down
15 changes: 12 additions & 3 deletions pkg/utils/dataset/volume/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"

Expand All @@ -45,6 +44,11 @@ func CreatePersistentVolumeForRuntime(client client.Client,
return err
}

storageCapacity, err := utils.GetPVCStorageCapacityOfDataset(client, runtime.GetName(), runtime.GetNamespace())
if err != nil {
return err
}

pvName := runtime.GetPersistentVolumeName()

found, err := kubeclient.IsPersistentVolumeExist(client, pvName, common.ExpectedFluidAnnotations)
Expand All @@ -65,7 +69,7 @@ func CreatePersistentVolumeForRuntime(client client.Client,
Spec: corev1.PersistentVolumeSpec{
AccessModes: accessModes,
Capacity: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"),
corev1.ResourceName(corev1.ResourceStorage): storageCapacity,
},
StorageClassName: common.FluidStorageClass,
PersistentVolumeSource: corev1.PersistentVolumeSource{
Expand Down Expand Up @@ -179,6 +183,11 @@ func CreatePersistentVolumeClaimForRuntime(client client.Client,
return err
}

storageCapacity, err := utils.GetPVCStorageCapacityOfDataset(client, runtime.GetName(), runtime.GetNamespace())
if err != nil {
return err
}

found, err := kubeclient.IsPersistentVolumeClaimExist(client, runtime.GetName(), runtime.GetNamespace(), common.ExpectedFluidAnnotations)
if err != nil {
return err
Expand All @@ -204,7 +213,7 @@ func CreatePersistentVolumeClaimForRuntime(client client.Client,
AccessModes: accessModes,
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"),
corev1.ResourceName(corev1.ResourceStorage): storageCapacity,
},
},
},
Expand Down
77 changes: 77 additions & 0 deletions pkg/utils/dataset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
"github.com/fluid-cloudnative/fluid/pkg/utils/fake"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
Expand Down Expand Up @@ -174,6 +175,71 @@ func TestGetAccessModesOfDataset(t *testing.T) {
}
}

func TestGetPVCStorageCapacityOfDataset(t *testing.T) {

testCases := map[string]struct {
name string
getName string
namespace string
storageCapacity string
wantStorageCapacity resource.Quantity
notFound bool
}{
"test get dataset PVC storage capacity case 1": {
name: "dataset-1",
getName: "dataset-1",
notFound: false,
namespace: "default",
storageCapacity: "",
wantStorageCapacity: resource.MustParse("100Pi"),
},
"test get dataset PVC storage capacity case 2": {
name: "dataset-1",
getName: "dataset-1",
notFound: false,
namespace: "default",
storageCapacity: "1Gi",
wantStorageCapacity: resource.MustParse("1Gi"),
},
"test get dataset PVC storage capacity case 3": {
name: "dataset-1",
getName: "dataset-1-notexist",
notFound: true,
namespace: "default",
storageCapacity: "",
wantStorageCapacity: resource.Quantity{},
},
"test get dataset PVC storage capacity case 4": {
name: "dataset-1",
getName: "dataset-1",
notFound: false,
namespace: "default",
storageCapacity: "formatError",
wantStorageCapacity: resource.MustParse("100Pi"),
},
}

for k, item := range testCases {
dataset := mockDatasetWithPVCStorageCapacity(item.name, item.namespace, item.storageCapacity)
s := runtime.NewScheme()
s.AddKnownTypes(datav1alpha1.GroupVersion, dataset)

fakeClient := fake.NewFakeClientWithScheme(s, dataset)

gotStorageCapacity, err := GetPVCStorageCapacityOfDataset(fakeClient, item.getName, item.namespace)

if item.notFound {
if err == nil {
t.Errorf("%s check failure,want err but got nil", k)
}
} else {
if !reflect.DeepEqual(gotStorageCapacity, item.wantStorageCapacity) {
t.Errorf("%s check failure, want:%v,got:%v", k, item.wantStorageCapacity, gotStorageCapacity)
}
}
}
}

func TestIsTargetPathUnderFluidNativeMounts(t *testing.T) {
testCases := map[string]struct {
targetPath string
Expand Down Expand Up @@ -315,6 +381,17 @@ func mockDatasetWithAccessModel(name, ns string, accessModel []v1.PersistentVolu
return dataset
}

func mockDatasetWithPVCStorageCapacity(name, ns, storageCapacity string) *datav1alpha1.Dataset {
dataset := &datav1alpha1.Dataset{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns,
Annotations: map[string]string{"pvc.fluid.io/resources.requests.storage": storageCapacity},
},
}
return dataset
}

func mockDatasetWithCondition(name, ns string, conditions []datav1alpha1.DatasetCondition) *datav1alpha1.Dataset {
dataset := &datav1alpha1.Dataset{
ObjectMeta: metav1.ObjectMeta{
Expand Down

0 comments on commit 2680812

Please sign in to comment.