From 1cb71801265602854bd138f2fa68ec59ee9a946c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20G=C3=B6ktu=C4=9F=20=C3=96zt=C3=BCrk?= Date: Fri, 21 Jun 2019 11:24:50 +0000 Subject: [PATCH] sdk/instancesdk: optionally delete instance's data volumes during instance Delete() and update gRPC API to migrate this feature. resolves #1069. --- protobuf/api/instance.pb.go | 76 +++++++++++++++++++++---------------- protobuf/api/instance.proto | 1 + sdk/instance/instance.go | 14 ++++++- sdk/instance/volume.go | 43 +++++++++++++++++++++ server/grpc/instance.go | 2 +- 5 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 sdk/instance/volume.go diff --git a/protobuf/api/instance.pb.go b/protobuf/api/instance.pb.go index 5c7834960..4ebfeeda6 100644 --- a/protobuf/api/instance.pb.go +++ b/protobuf/api/instance.pb.go @@ -35,7 +35,7 @@ func (m *GetInstanceRequest) Reset() { *m = GetInstanceRequest{} } func (m *GetInstanceRequest) String() string { return proto.CompactTextString(m) } func (*GetInstanceRequest) ProtoMessage() {} func (*GetInstanceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_instance_b39c234cbed136ac, []int{0} + return fileDescriptor_instance_2701e6acce0e6202, []int{0} } func (m *GetInstanceRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetInstanceRequest.Unmarshal(m, b) @@ -73,7 +73,7 @@ func (m *GetInstanceResponse) Reset() { *m = GetInstanceResponse{} } func (m *GetInstanceResponse) String() string { return proto.CompactTextString(m) } func (*GetInstanceResponse) ProtoMessage() {} func (*GetInstanceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_instance_b39c234cbed136ac, []int{1} + return fileDescriptor_instance_2701e6acce0e6202, []int{1} } func (m *GetInstanceResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetInstanceResponse.Unmarshal(m, b) @@ -111,7 +111,7 @@ func (m *ListInstancesRequest) Reset() { *m = ListInstancesRequest{} } func (m *ListInstancesRequest) String() string { return proto.CompactTextString(m) } func (*ListInstancesRequest) ProtoMessage() {} func (*ListInstancesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_instance_b39c234cbed136ac, []int{2} + return fileDescriptor_instance_2701e6acce0e6202, []int{2} } func (m *ListInstancesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListInstancesRequest.Unmarshal(m, b) @@ -149,7 +149,7 @@ func (m *ListInstancesResponse) Reset() { *m = ListInstancesResponse{} } func (m *ListInstancesResponse) String() string { return proto.CompactTextString(m) } func (*ListInstancesResponse) ProtoMessage() {} func (*ListInstancesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_instance_b39c234cbed136ac, []int{3} + return fileDescriptor_instance_2701e6acce0e6202, []int{3} } func (m *ListInstancesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListInstancesResponse.Unmarshal(m, b) @@ -188,7 +188,7 @@ func (m *CreateInstanceRequest) Reset() { *m = CreateInstanceRequest{} } func (m *CreateInstanceRequest) String() string { return proto.CompactTextString(m) } func (*CreateInstanceRequest) ProtoMessage() {} func (*CreateInstanceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_instance_b39c234cbed136ac, []int{4} + return fileDescriptor_instance_2701e6acce0e6202, []int{4} } func (m *CreateInstanceRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateInstanceRequest.Unmarshal(m, b) @@ -233,7 +233,7 @@ func (m *CreateInstanceResponse) Reset() { *m = CreateInstanceResponse{} func (m *CreateInstanceResponse) String() string { return proto.CompactTextString(m) } func (*CreateInstanceResponse) ProtoMessage() {} func (*CreateInstanceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_instance_b39c234cbed136ac, []int{5} + return fileDescriptor_instance_2701e6acce0e6202, []int{5} } func (m *CreateInstanceResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateInstanceResponse.Unmarshal(m, b) @@ -262,6 +262,7 @@ func (m *CreateInstanceResponse) GetInstance() *definition.Instance { type DeleteInstanceRequest struct { Hash string `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` + DeleteData bool `protobuf:"varint,2,opt,name=deleteData,proto3" json:"deleteData,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -271,7 +272,7 @@ func (m *DeleteInstanceRequest) Reset() { *m = DeleteInstanceRequest{} } func (m *DeleteInstanceRequest) String() string { return proto.CompactTextString(m) } func (*DeleteInstanceRequest) ProtoMessage() {} func (*DeleteInstanceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_instance_b39c234cbed136ac, []int{6} + return fileDescriptor_instance_2701e6acce0e6202, []int{6} } func (m *DeleteInstanceRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteInstanceRequest.Unmarshal(m, b) @@ -298,6 +299,13 @@ func (m *DeleteInstanceRequest) GetHash() string { return "" } +func (m *DeleteInstanceRequest) GetDeleteData() bool { + if m != nil { + return m.DeleteData + } + return false +} + type DeleteInstanceResponse struct { Hash string `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -309,7 +317,7 @@ func (m *DeleteInstanceResponse) Reset() { *m = DeleteInstanceResponse{} func (m *DeleteInstanceResponse) String() string { return proto.CompactTextString(m) } func (*DeleteInstanceResponse) ProtoMessage() {} func (*DeleteInstanceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_instance_b39c234cbed136ac, []int{7} + return fileDescriptor_instance_2701e6acce0e6202, []int{7} } func (m *DeleteInstanceResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteInstanceResponse.Unmarshal(m, b) @@ -519,29 +527,31 @@ var _Instance_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("protobuf/api/instance.proto", fileDescriptor_instance_b39c234cbed136ac) -} - -var fileDescriptor_instance_b39c234cbed136ac = []byte{ - // 319 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x93, 0xcd, 0x4e, 0xc2, 0x40, - 0x10, 0x80, 0x81, 0x12, 0x02, 0xc3, 0xc5, 0x8c, 0x80, 0xb5, 0x5c, 0x9a, 0x3d, 0x35, 0xd1, 0x14, - 0x53, 0x2f, 0xc6, 0x8b, 0x07, 0x4c, 0xea, 0xdf, 0xa9, 0x6f, 0xb0, 0xe0, 0x10, 0x36, 0x31, 0xdb, - 0xda, 0x5d, 0x78, 0x0f, 0xdf, 0xd8, 0xf4, 0x1f, 0xdb, 0x35, 0x26, 0xde, 0x36, 0x33, 0x5f, 0xbf, - 0x99, 0x9d, 0xe9, 0xc2, 0x32, 0x49, 0x63, 0x1d, 0x6f, 0x0e, 0xbb, 0x15, 0x4f, 0xc4, 0x4a, 0x48, - 0xa5, 0xb9, 0xdc, 0x92, 0x9f, 0x47, 0xd1, 0xe2, 0x89, 0x70, 0x58, 0x4d, 0xbc, 0xd3, 0x4e, 0x48, - 0xa1, 0x45, 0x2c, 0x5b, 0x20, 0xf3, 0x00, 0x43, 0xd2, 0xcf, 0x65, 0x30, 0xa2, 0xcf, 0x03, 0x29, - 0x8d, 0x08, 0xc3, 0x3d, 0x57, 0x7b, 0xbb, 0xef, 0xf6, 0xbd, 0x49, 0x94, 0x9f, 0x59, 0x08, 0xe7, - 0x3f, 0x48, 0x95, 0xc4, 0x52, 0x11, 0xde, 0xc0, 0xb8, 0x52, 0xe6, 0xf8, 0x34, 0x98, 0xf9, 0x4d, - 0x39, 0xbf, 0xe6, 0x6b, 0x8a, 0xdd, 0xc1, 0xec, 0x4d, 0xa8, 0xda, 0xa4, 0xaa, 0xa2, 0x2e, 0x4c, - 0x15, 0xa5, 0x47, 0xb1, 0xa5, 0xa7, 0xa6, 0xf6, 0x69, 0x88, 0xbd, 0xc2, 0xbc, 0xf5, 0x65, 0xd9, - 0x44, 0x00, 0x93, 0x4a, 0xaf, 0xec, 0xbe, 0x6b, 0xfd, 0xda, 0x45, 0x83, 0x65, 0xb2, 0x75, 0x4a, - 0x5c, 0x53, 0xfb, 0xf2, 0x7f, 0xf6, 0x81, 0x67, 0x60, 0x91, 0x3c, 0xda, 0x03, 0xd7, 0xf2, 0x26, - 0x51, 0x76, 0x64, 0x2f, 0xb0, 0x68, 0xcb, 0xfe, 0x3d, 0x9f, 0x2b, 0x98, 0x3f, 0xd2, 0x07, 0x75, - 0x1b, 0x33, 0x6d, 0xe5, 0x1a, 0x16, 0x6d, 0xb8, 0x2c, 0x6c, 0xa0, 0x83, 0xaf, 0x01, 0x8c, 0x2b, - 0x10, 0xef, 0xc1, 0x0a, 0x49, 0xe3, 0x85, 0xcf, 0x13, 0xe1, 0x77, 0x7f, 0x02, 0xc7, 0xee, 0x26, - 0x0a, 0x35, 0xeb, 0xe1, 0x03, 0x0c, 0xb3, 0x4d, 0xe0, 0x65, 0xce, 0x98, 0xd6, 0xe9, 0x38, 0xa6, - 0x54, 0x2d, 0x58, 0xc3, 0xa8, 0x18, 0x18, 0x16, 0x9c, 0x71, 0x15, 0xce, 0xd2, 0x98, 0x3b, 0x95, - 0x14, 0x97, 0x2f, 0x25, 0xc6, 0xb1, 0x95, 0x12, 0xf3, 0x94, 0x58, 0x6f, 0x33, 0xca, 0x1f, 0xc2, - 0xed, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2f, 0xeb, 0x23, 0x16, 0x50, 0x03, 0x00, 0x00, + proto.RegisterFile("protobuf/api/instance.proto", fileDescriptor_instance_2701e6acce0e6202) +} + +var fileDescriptor_instance_2701e6acce0e6202 = []byte{ + // 337 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x93, 0xcf, 0x4e, 0xf2, 0x40, + 0x10, 0xc0, 0x29, 0x25, 0x04, 0x86, 0xcb, 0x97, 0xf9, 0x00, 0x6b, 0x49, 0x4c, 0xb3, 0xa7, 0x1e, + 0x4c, 0x31, 0x78, 0x31, 0x5e, 0x3c, 0x40, 0x82, 0x8a, 0xa7, 0xbe, 0xc1, 0x02, 0x43, 0xd8, 0xc4, + 0x6c, 0x2b, 0xbb, 0xf0, 0x1e, 0xbe, 0xb1, 0x61, 0xfb, 0x87, 0xda, 0xae, 0x31, 0xf1, 0xb6, 0x99, + 0xf9, 0xf5, 0x37, 0xb3, 0x33, 0x5d, 0x98, 0xa4, 0x87, 0x44, 0x27, 0xeb, 0xe3, 0x6e, 0xca, 0x53, + 0x31, 0x15, 0x52, 0x69, 0x2e, 0x37, 0x14, 0x99, 0x28, 0xba, 0x3c, 0x15, 0x3e, 0x2b, 0x89, 0x2d, + 0xed, 0x84, 0x14, 0x5a, 0x24, 0xb2, 0x06, 0xb2, 0x10, 0x70, 0x49, 0xfa, 0x25, 0x0f, 0xc6, 0xf4, + 0x71, 0x24, 0xa5, 0x11, 0xa1, 0xb3, 0xe7, 0x6a, 0xef, 0x39, 0x81, 0x13, 0xf6, 0x63, 0x73, 0x66, + 0x4b, 0xf8, 0xff, 0x8d, 0x54, 0x69, 0x22, 0x15, 0xe1, 0x1d, 0xf4, 0x0a, 0xa5, 0xc1, 0x07, 0xb3, + 0x61, 0x74, 0x29, 0x17, 0x95, 0x7c, 0x49, 0xb1, 0x07, 0x18, 0xbe, 0x09, 0x55, 0x9a, 0x54, 0x51, + 0x34, 0x80, 0x81, 0xa2, 0xc3, 0x49, 0x6c, 0xe8, 0xf9, 0x52, 0xbb, 0x1a, 0x62, 0x2b, 0x18, 0xd5, + 0xbe, 0xcc, 0x9b, 0x98, 0x41, 0xbf, 0xd0, 0x2b, 0xcf, 0x09, 0xdc, 0x1f, 0xbb, 0xb8, 0x60, 0x67, + 0xd9, 0xfc, 0x40, 0x5c, 0x53, 0xfd, 0xf2, 0xbf, 0xf6, 0x81, 0xff, 0xc0, 0x25, 0x79, 0xf2, 0xda, + 0x81, 0x1b, 0xf6, 0xe3, 0xf3, 0x91, 0xbd, 0xc2, 0xb8, 0x2e, 0xfb, 0xf3, 0x7c, 0x56, 0x30, 0x5a, + 0xd0, 0x3b, 0x35, 0x1b, 0xb3, 0x6c, 0x05, 0x6f, 0x00, 0xb6, 0x06, 0x5e, 0x70, 0xcd, 0xbd, 0x76, + 0xe0, 0x84, 0xbd, 0xb8, 0x12, 0x61, 0xb7, 0x30, 0xae, 0xcb, 0xf2, 0xc6, 0x2c, 0xb6, 0xd9, 0x67, + 0x1b, 0x7a, 0x05, 0x88, 0x8f, 0xe0, 0x2e, 0x49, 0xe3, 0x55, 0xc4, 0x53, 0x11, 0x35, 0x7f, 0x12, + 0xdf, 0x6b, 0x26, 0x32, 0x35, 0x6b, 0xe1, 0x13, 0x74, 0xce, 0x9b, 0xc2, 0x6b, 0xc3, 0xd8, 0xd6, + 0xed, 0xfb, 0xb6, 0x54, 0x29, 0x98, 0x43, 0x37, 0x1b, 0x28, 0x66, 0x9c, 0x75, 0x55, 0xfe, 0xc4, + 0x9a, 0xab, 0x4a, 0xb2, 0xcb, 0xe7, 0x12, 0xeb, 0x58, 0x73, 0x89, 0x7d, 0x4a, 0xac, 0xb5, 0xee, + 0x9a, 0x87, 0x72, 0xff, 0x15, 0x00, 0x00, 0xff, 0xff, 0xbb, 0x18, 0x2e, 0xfb, 0x70, 0x03, 0x00, + 0x00, } diff --git a/protobuf/api/instance.proto b/protobuf/api/instance.proto index 32ee8cefe..134359161 100644 --- a/protobuf/api/instance.proto +++ b/protobuf/api/instance.proto @@ -38,6 +38,7 @@ message CreateInstanceResponse { message DeleteInstanceRequest { string hash = 1; // Instance hash + bool deleteData = 2; // if enabled, any persistent data (volumes) that belongs to instance and its dependencies will also be deleted. } message DeleteInstanceResponse { diff --git a/sdk/instance/instance.go b/sdk/instance/instance.go index 8ccde7157..de09894f3 100644 --- a/sdk/instance/instance.go +++ b/sdk/instance/instance.go @@ -119,8 +119,10 @@ func (i *Instance) GetAllByService(serviceHash string) ([]*instance.Instance, er return i.instanceDB.GetAllByService(serviceHash) } -// Delete an instance -func (i *Instance) Delete(hash string) error { +// Delete deletes an instance. +// if deleteData is enabled, any persistent data that belongs to +// the instance and to its dependencies will also be deleted. +func (i *Instance) Delete(hash string, deleteData bool) error { inst, err := i.instanceDB.Get(hash) if err != nil { return err @@ -128,5 +130,13 @@ func (i *Instance) Delete(hash string) error { if err := i.stop(inst); err != nil { return err } + // delete volumes first before the instance. this way if + // deleting volumes fails, process can be retried by the user again + // because instance still will be in the db. + if deleteData { + if err := i.deleteData(inst); err != nil { + return err + } + } return i.instanceDB.Delete(hash) } diff --git a/sdk/instance/volume.go b/sdk/instance/volume.go new file mode 100644 index 000000000..7dda3b11f --- /dev/null +++ b/sdk/instance/volume.go @@ -0,0 +1,43 @@ +package instancesdk + +import ( + "sync" + + "github.com/docker/docker/client" + "github.com/mesg-foundation/core/instance" + "github.com/mesg-foundation/core/x/xerrors" +) + +// deleteData deletes the data volumes of instance and its dependencies. +// TODO: right now deleteData() is not compatible to work with multiple instances of same +// service since extractVolumes() accepts service, not instance. same applies in the start +// api as well. make it work with multiple instances. +func (i *Instance) deleteData(inst *instance.Instance) error { + s, err := i.serviceDB.Get(inst.ServiceHash) + if err != nil { + return err + } + var ( + wg sync.WaitGroup + errs xerrors.SyncErrors + ) + for _, d := range append(s.Dependencies, s.Configuration) { + // Service.Configuration can be nil so, here is a check for it. + if d == nil { + continue + } + for _, volume := range extractVolumes(s, d) { + wg.Add(1) + go func(source string) { + defer wg.Done() + // if service is never started before, data volume won't be created and Docker Engine + // will return with the not found error. therefore, we can safely ignore it. + if err := i.container.DeleteVolume(source); err != nil && !client.IsErrNotFound(err) { + errs.Append(err) + } + }(volume.Source) + } + } + wg.Wait() + return errs.ErrorOrNil() +} diff --git a/server/grpc/instance.go b/server/grpc/instance.go index dfc6c5064..0bff9c7f6 100644 --- a/server/grpc/instance.go +++ b/server/grpc/instance.go @@ -49,7 +49,7 @@ func (s *InstanceServer) Get(ctx context.Context, request *protobuf_api.GetInsta // Delete an instance func (s *InstanceServer) Delete(ctx context.Context, request *protobuf_api.DeleteInstanceRequest) (*protobuf_api.DeleteInstanceResponse, error) { - err := s.sdk.Instance.Delete(request.Hash) + err := s.sdk.Instance.Delete(request.Hash, request.DeleteData) if err != nil { return nil, err }