From 161549915ad9f9a4b94407069253d45d6c9fa714 Mon Sep 17 00:00:00 2001 From: nameczz Date: Fri, 18 Mar 2022 21:05:11 +0800 Subject: [PATCH 1/3] add submodule for milvus proto Signed-off-by: nameczz --- .gitmodules | 3 + build.js | 6 +- grpc-proto/common.proto | 201 ---------- grpc-proto/milvus.proto | 815 ---------------------------------------- grpc-proto/schema.proto | 125 ------ milvus/Collection.ts | 2 +- milvus/Data.ts | 2 +- milvus/index.ts | 2 +- package.json | 2 +- proto | 1 + tsconfig.json | 2 +- 11 files changed, 13 insertions(+), 1148 deletions(-) create mode 100644 .gitmodules delete mode 100644 grpc-proto/common.proto delete mode 100644 grpc-proto/milvus.proto delete mode 100644 grpc-proto/schema.proto create mode 160000 proto diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..bdc6a011 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "proto"] + path = proto + url = https://github.com/milvus-io/milvus-proto.git diff --git a/build.js b/build.js index c7478ef7..31daff5d 100644 --- a/build.js +++ b/build.js @@ -17,7 +17,9 @@ function copyFileSync(source, target) { function copyFolderRecursiveSync(source, target) { let files = []; - + if (!fs.existsSync(target)){ + fs.mkdirSync(target, { recursive: true }); +} // Check if folder needs to be created or integrated let targetFolder = path.join(target, path.basename(source)); if (!fs.existsSync(targetFolder)) { @@ -65,7 +67,7 @@ function writeSdkJson(path) { } } -copyFolderRecursiveSync("./grpc-proto", "./dist"); +copyFolderRecursiveSync("./proto/proto", "./dist/proto"); // if dist has package.json need delete it. // otherwise npm publish will use package.json inside dist then will missing files. diff --git a/grpc-proto/common.proto b/grpc-proto/common.proto deleted file mode 100644 index 8c6f8542..00000000 --- a/grpc-proto/common.proto +++ /dev/null @@ -1,201 +0,0 @@ -syntax = "proto3"; - -package milvus.proto.common; -option go_package="github.com/milvus-io/milvus/internal/proto/commonpb"; - - -enum ErrorCode { - Success = 0; - UnexpectedError = 1; - ConnectFailed = 2; - PermissionDenied = 3; - CollectionNotExists = 4; - IllegalArgument = 5; - IllegalDimension = 7; - IllegalIndexType = 8; - IllegalCollectionName = 9; - IllegalTOPK = 10; - IllegalRowRecord = 11; - IllegalVectorID = 12; - IllegalSearchResult = 13; - FileNotFound = 14; - MetaFailed = 15; - CacheFailed = 16; - CannotCreateFolder = 17; - CannotCreateFile = 18; - CannotDeleteFolder = 19; - CannotDeleteFile = 20; - BuildIndexError = 21; - IllegalNLIST = 22; - IllegalMetricType = 23; - OutOfMemory = 24; - IndexNotExist = 25; - EmptyCollection = 26; - - // internal error code. - DDRequestRace = 1000; -} - -enum IndexState { - IndexStateNone = 0; - Unissued = 1; - InProgress = 2; - Finished = 3; - Failed = 4; -} - -enum SegmentState { - SegmentStateNone = 0; - NotExist = 1; - Growing = 2; - Sealed = 3; - Flushed = 4; - Flushing = 5; - Dropped = 6; -} - -message Status { - ErrorCode error_code = 1; - string reason = 2; -} - -message KeyValuePair { - string key = 1; - string value = 2; -} - -message KeyDataPair { - string key = 1; - bytes data = 2; -} - -message Blob { - bytes value = 1; -} - -message Address { - string ip = 1; - int64 port = 2; -} - -enum MsgType { - Undefined = 0; - /* DEFINITION REQUESTS: COLLECTION */ - CreateCollection = 100; - DropCollection = 101; - HasCollection = 102; - DescribeCollection = 103; - ShowCollections = 104; - GetSystemConfigs = 105; - LoadCollection = 106; - ReleaseCollection = 107; - CreateAlias = 108; - DropAlias = 109; - AlterAlias = 110; - - - /* DEFINITION REQUESTS: PARTITION */ - CreatePartition = 200; - DropPartition = 201; - HasPartition = 202; - DescribePartition = 203; - ShowPartitions = 204; - LoadPartitions = 205; - ReleasePartitions = 206; - - /* DEFINE REQUESTS: SEGMENT */ - ShowSegments = 250; - DescribeSegment = 251; - LoadSegments = 252; - ReleaseSegments = 253; - HandoffSegments = 254; - LoadBalanceSegments = 255; - - /* DEFINITION REQUESTS: INDEX */ - CreateIndex = 300; - DescribeIndex = 301; - DropIndex = 302; - - /* MANIPULATION REQUESTS */ - Insert = 400; - Delete = 401; - Flush = 402; - - /* QUERY */ - Search = 500; - SearchResult = 501; - GetIndexState = 502; - GetIndexBuildProgress = 503; - GetCollectionStatistics = 504; - GetPartitionStatistics = 505; - Retrieve = 506; - RetrieveResult = 507; - WatchDmChannels = 508; - RemoveDmChannels = 509; - WatchQueryChannels = 510; - RemoveQueryChannels = 511; - SealedSegmentsChangeInfo = 512; - WatchDeltaChannels = 513; - - /* DATA SERVICE */ - SegmentInfo = 600; - SystemInfo = 601; - GetRecoveryInfo = 602; - GetSegmentState = 603; - - /* SYSTEM CONTROL */ - TimeTick = 1200; - QueryNodeStats = 1201; // GOOSE TODO: Remove kQueryNodeStats - LoadIndex = 1202; - RequestID = 1203; - RequestTSO = 1204; - AllocateSegment = 1205; - SegmentStatistics = 1206; - SegmentFlushDone = 1207; - - DataNodeTt = 1208; -} - -message MsgBase { - MsgType msg_type = 1; - int64 msgID = 2; - uint64 timestamp = 3; - int64 sourceID = 4; -} - -enum DslType { - Dsl = 0; - BoolExprV1 = 1; -} - -// Don't Modify This. @czs -message MsgHeader { - common.MsgBase base = 1; -} - -// Don't Modify This. @czs -message DMLMsgHeader { - common.MsgBase base = 1; - string shardName = 2; -} - -enum CompactionState { - UndefiedState = 0; - Executing = 1; - Completed = 2; -} - -enum ConsistencyLevel { - Strong = 0; - Session = 1; // default in PyMilvus - Bounded = 2; - Eventually = 3; - Customized = 4; // Users pass their own `guarantee_timestamp`. -} - -enum ImportState { - ImportPending = 0; - ImportExecuting = 1; - ImportCompleted = 2; - ImportFailed = 3; -} \ No newline at end of file diff --git a/grpc-proto/milvus.proto b/grpc-proto/milvus.proto deleted file mode 100644 index 35272d18..00000000 --- a/grpc-proto/milvus.proto +++ /dev/null @@ -1,815 +0,0 @@ -syntax = "proto3"; -package milvus.proto.milvus; - -option go_package = "github.com/milvus-io/milvus/internal/proto/milvuspb"; - -import "common.proto"; -import "schema.proto"; - -service MilvusService { - rpc CreateCollection(CreateCollectionRequest) returns (common.Status) {} - rpc DropCollection(DropCollectionRequest) returns (common.Status) {} - rpc HasCollection(HasCollectionRequest) returns (BoolResponse) {} - rpc LoadCollection(LoadCollectionRequest) returns (common.Status) {} - rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {} - rpc DescribeCollection(DescribeCollectionRequest) returns (DescribeCollectionResponse) {} - rpc GetCollectionStatistics(GetCollectionStatisticsRequest) returns (GetCollectionStatisticsResponse) {} - rpc ShowCollections(ShowCollectionsRequest) returns (ShowCollectionsResponse) {} - - rpc CreatePartition(CreatePartitionRequest) returns (common.Status) {} - rpc DropPartition(DropPartitionRequest) returns (common.Status) {} - rpc HasPartition(HasPartitionRequest) returns (BoolResponse) {} - rpc LoadPartitions(LoadPartitionsRequest) returns (common.Status) {} - rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {} - rpc GetPartitionStatistics(GetPartitionStatisticsRequest) returns (GetPartitionStatisticsResponse) {} - rpc ShowPartitions(ShowPartitionsRequest) returns (ShowPartitionsResponse) {} - - rpc CreateAlias(CreateAliasRequest) returns (common.Status) {} - rpc DropAlias(DropAliasRequest) returns (common.Status) {} - rpc AlterAlias(AlterAliasRequest) returns (common.Status) {} - - rpc CreateIndex(CreateIndexRequest) returns (common.Status) {} - rpc DescribeIndex(DescribeIndexRequest) returns (DescribeIndexResponse) {} - rpc GetIndexState(GetIndexStateRequest) returns (GetIndexStateResponse) {} - rpc GetIndexBuildProgress(GetIndexBuildProgressRequest) returns (GetIndexBuildProgressResponse) {} - rpc DropIndex(DropIndexRequest) returns (common.Status) {} - - rpc Insert(InsertRequest) returns (MutationResult) {} - rpc Delete(DeleteRequest) returns (MutationResult) {} - rpc Search(SearchRequest) returns (SearchResults) {} - rpc Flush(FlushRequest) returns (FlushResponse) {} - rpc Query(QueryRequest) returns (QueryResults) {} - rpc CalcDistance(CalcDistanceRequest) returns (CalcDistanceResults) {} - - rpc GetFlushState(GetFlushStateRequest) returns (GetFlushStateResponse) {} - rpc GetPersistentSegmentInfo(GetPersistentSegmentInfoRequest) returns (GetPersistentSegmentInfoResponse) {} - rpc GetQuerySegmentInfo(GetQuerySegmentInfoRequest) returns (GetQuerySegmentInfoResponse) {} - - rpc Dummy(DummyRequest) returns (DummyResponse) {} - - // TODO: remove - rpc RegisterLink(RegisterLinkRequest) returns (RegisterLinkResponse) {} - - // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy - rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} - rpc LoadBalance(LoadBalanceRequest) returns (common.Status) {} - rpc GetCompactionState(GetCompactionStateRequest) returns (GetCompactionStateResponse) {} - rpc ManualCompaction(ManualCompactionRequest) returns (ManualCompactionResponse) {} - rpc GetCompactionStateWithPlans(GetCompactionPlansRequest) returns (GetCompactionPlansResponse) {} - - // https://wiki.lfaidata.foundation/display/MIL/MEP+24+--+Support+bulk+load - rpc Import(ImportRequest) returns (ImportResponse) {} - rpc GetImportState(GetImportStateRequest) returns (GetImportStateResponse) {} -} - -message CreateAliasRequest { - common.MsgBase base = 1; - string db_name = 2; - string collection_name = 3; - string alias = 4; -} - -message DropAliasRequest { - common.MsgBase base = 1; - string db_name = 2; - string alias = 3; -} - -message AlterAliasRequest{ - common.MsgBase base = 1; - string db_name = 2; - string collection_name = 3; - string alias = 4; -} - -/** -* Create collection in milvus -*/ -message CreateCollectionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The unique collection name in milvus.(Required) - string collection_name = 3; - // The serialized `schema.CollectionSchema`(Required) - bytes schema = 4; - // Once set, no modification is allowed (Optional) - // https://github.com/milvus-io/milvus/issues/6690 - int32 shards_num = 5; - // The consistency level that the collection used, modification is not supported now. - common.ConsistencyLevel consistency_level = 6; -} - -/** -* Drop collection in milvus, also will drop data in collection. -*/ -message DropCollectionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The unique collection name in milvus.(Required) - string collection_name = 3; -} - -/** -* Check collection exist in milvus or not. -*/ -message HasCollectionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name you want to check. - string collection_name = 3; - // If time_stamp is not zero, will return true when time_stamp >= created collection timestamp, otherwise will return false. - uint64 time_stamp = 4; -} - - -message BoolResponse { - common.Status status = 1; - bool value = 2; -} - -message StringResponse { - common.Status status = 1; - string value = 2; -} - -/** -* Get collection meta datas like: schema, collectionID, shards number ... -*/ -message DescribeCollectionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name you want to describe, you can pass collection_name or collectionID - string collection_name = 3; - // The collection ID you want to describe - int64 collectionID = 4; - // If time_stamp is not zero, will describe collection success when time_stamp >= created collection timestamp, otherwise will throw error. - uint64 time_stamp = 5; -} - -/** -* DescribeCollection Response -*/ -message DescribeCollectionResponse { - // Contain error_code and reason - common.Status status = 1; - // The schema param when you created collection. - schema.CollectionSchema schema = 2; - // The collection id - int64 collectionID = 3; - // System design related, users should not perceive - repeated string virtual_channel_names = 4; - // System design related, users should not perceive - repeated string physical_channel_names = 5; - // Hybrid timestamp in milvus - uint64 created_timestamp = 6; - // The utc timestamp calculated by created_timestamp - uint64 created_utc_timestamp = 7; - // The shards number you set. - int32 shards_num = 8; - // The aliases of this collection - repeated string aliases = 9; - // The message ID/posititon when collection is created - repeated common.KeyDataPair start_positions = 10; - // The consistency level that the collection used, modification is not supported now. - common.ConsistencyLevel consistency_level = 11; - // The collection name - string collection_name = 12; -} - -/** -* Load collection data into query nodes, then you can do vector search on this collection. -*/ -message LoadCollectionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name you want to load - string collection_name = 3; -} - -/** -* Release collection data from query nodes, then you can't do vector search on this collection. -*/ -message ReleaseCollectionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name you want to release - string collection_name = 3; -} - -/** -* Get collection statistics like row_count. -*/ -message GetCollectionStatisticsRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name you want get statistics - string collection_name = 3; -} - -/** -* Will return collection statistics in stats field like [{key:"row_count",value:"1"}] -*/ -message GetCollectionStatisticsResponse { - // Contain error_code and reason - common.Status status = 1; - // Collection statistics data - repeated common.KeyValuePair stats = 2; -} - -/* -* This is for ShowCollectionsRequest type field. -*/ -enum ShowType { - // Will return all colloections - All = 0; - // Will return loaded collections with their inMemory_percentages - InMemory = 1; -} - -/* -* List collections -*/ -message ShowCollectionsRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // Not useful for now - uint64 time_stamp = 3; - // Decide return Loaded collections or All collections(Optional) - ShowType type = 4; - // When type is InMemory, will return these collection's inMemory_percentages.(Optional) - repeated string collection_names = 5; -} - -/* -* Return basic collection infos. -*/ -message ShowCollectionsResponse { - // Contain error_code and reason - common.Status status = 1; - // Collection name array - repeated string collection_names = 2; - // Collection Id array - repeated int64 collection_ids = 3; - // Hybrid timestamps in milvus - repeated uint64 created_timestamps = 4; - // The utc timestamp calculated by created_timestamp - repeated uint64 created_utc_timestamps = 5; - // Load percentage on querynode when type is InMemory - repeated int64 inMemory_percentages = 6; -} - -/* -* Create partition in created collection. -*/ -message CreatePartitionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name in milvus - string collection_name = 3; - // The partition name you want to create. - string partition_name = 4; -} - -/* -* Drop partition in created collection. -*/ -message DropPartitionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name in milvus - string collection_name = 3; - // The partition name you want to drop - string partition_name = 4; -} - -/* -* Check if partition exist in collection or not. -*/ -message HasPartitionRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name in milvus - string collection_name = 3; - // The partition name you want to check - string partition_name = 4; -} - -/* -* Load specific partitions data of one collection into query nodes -* Then you can get these data as result when you do vector search on this collection. -*/ -message LoadPartitionsRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name in milvus - string collection_name = 3; - // The partition names you want to load - repeated string partition_names = 4; -} - -/* -* Release specific partitions data of one collection from query nodes. -* Then you can not get these data as result when you do vector search on this collection. -*/ -message ReleasePartitionsRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name in milvus - string collection_name = 3; - // The partition names you want to release - repeated string partition_names = 4; -} - -/* -* Get partition statistics like row_count. -*/ -message GetPartitionStatisticsRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name in milvus - string collection_name = 3; - // The partition name you want to collect statistics - string partition_name = 4; -} - -message GetPartitionStatisticsResponse { - common.Status status = 1; - repeated common.KeyValuePair stats = 2; -} - -/* -* List all partitions for particular collection -*/ -message ShowPartitionsRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The collection name you want to describe, you can pass collection_name or collectionID - string collection_name = 3; - // The collection id in milvus - int64 collectionID = 4; - // When type is InMemory, will return these patitions's inMemory_percentages.(Optional) - repeated string partition_names = 5; - // Decide return Loaded partitions or All partitions(Optional) - ShowType type = 6; -} - -/* -* List all partitions for particular collection response. -* The returned datas are all rows, we can format to columns by therir index. -*/ -message ShowPartitionsResponse { - // Contain error_code and reason - common.Status status = 1; - // All partition names for this collection - repeated string partition_names = 2; - // All partition ids for this collection - repeated int64 partitionIDs = 3; - // All hybrid timestamps - repeated uint64 created_timestamps = 4; - // All utc timestamps calculated by created_timestamps - repeated uint64 created_utc_timestamps = 5; - // Load percentage on querynode - repeated int64 inMemory_percentages = 6; -} - -message DescribeSegmentRequest { - common.MsgBase base = 1; - int64 collectionID = 2; - int64 segmentID = 3; -} - -message DescribeSegmentResponse { - common.Status status = 1; - int64 indexID = 2; - int64 buildID = 3; - bool enable_index = 4; - int64 fieldID = 5; -} - -message ShowSegmentsRequest { - common.MsgBase base = 1; - int64 collectionID = 2; - int64 partitionID = 3; -} - -message ShowSegmentsResponse { - common.Status status = 1; - repeated int64 segmentIDs = 2; -} - -/* -* Create index for vector datas -*/ -message CreateIndexRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The particular collection name you want to create index. - string collection_name = 3; - // The vector field name in this particular collection - string field_name = 4; - // Support keys: index_type,metric_type, params. Different index_type may has different params. - repeated common.KeyValuePair extra_params = 5; -} - -/* -* Get created index information. -* Current release of Milvus only supports showing latest built index. -*/ -message DescribeIndexRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2; - // The particular collection name in Milvus - string collection_name = 3; - // The vector field name in this particular collection - string field_name = 4; - // No need to set up for now @2021.06.30 - string index_name = 5; -} - -/* -* Index informations -*/ -message IndexDescription { - // Index name - string index_name = 1; - // Index id - int64 indexID = 2; - // Will return index_type, metric_type, params(like nlist). - repeated common.KeyValuePair params = 3; - // The vector field name - string field_name = 4; -} - -/* -* Describe index response -*/ -message DescribeIndexResponse { - // Response status - common.Status status = 1; - // All index informations, for now only return tha latest index you created for the collection. - repeated IndexDescription index_descriptions = 2; -} - -/* -* Get index building progress -*/ -message GetIndexBuildProgressRequest { - // Not useful for now - common.MsgBase base = 1; - // Not useful for now - string db_name = 2 ; - // The collection name in milvus - string collection_name = 3; - // The vector field name in this collection - string field_name = 4; - // Not useful for now - string index_name = 5; -} - -message GetIndexBuildProgressResponse { - common.Status status = 1; - int64 indexed_rows = 2; - int64 total_rows = 3; -} - -message GetIndexStateRequest { - common.MsgBase base = 1; // must - string db_name = 2 ; - string collection_name = 3; // must - string field_name = 4; - string index_name = 5; // No need to set up for now @2021.06.30 -} - -message GetIndexStateResponse { - common.Status status = 1; - common.IndexState state = 2; - string fail_reason = 3; -} - -message DropIndexRequest { - common.MsgBase base = 1; // must - string db_name = 2; - string collection_name = 3; // must - string field_name = 4; - string index_name = 5; // No need to set up for now @2021.06.30 -} - -message InsertRequest { - common.MsgBase base = 1; - string db_name = 2; - string collection_name = 3; - string partition_name = 4; - repeated schema.FieldData fields_data = 5; - repeated uint32 hash_keys = 6; - uint32 num_rows = 7; -} - -message MutationResult { - common.Status status = 1; - schema.IDs IDs = 2; // required for insert, delete - repeated uint32 succ_index = 3; // error indexes indicate - repeated uint32 err_index = 4; // error indexes indicate - bool acknowledged = 5; - int64 insert_cnt = 6; - int64 delete_cnt = 7; - int64 upsert_cnt = 8; - uint64 timestamp = 9; -} - -message DeleteRequest { - common.MsgBase base = 1; - string db_name = 2; - string collection_name = 3; - string partition_name = 4; - string expr = 5; - repeated uint32 hash_keys = 6; -} - -enum PlaceholderType { - None = 0; - BinaryVector = 100; - FloatVector = 101; -} - -message PlaceholderValue { - string tag = 1; - PlaceholderType type = 2; - // values is a 2d-array, every array contains a vector - repeated bytes values = 3; -} - -message PlaceholderGroup { - repeated PlaceholderValue placeholders = 1; -} - -message SearchRequest { - common.MsgBase base = 1; // must - string db_name = 2; - string collection_name = 3; // must - repeated string partition_names = 4; // must - string dsl = 5; // must - // serialized `PlaceholderGroup` - bytes placeholder_group = 6; // must - common.DslType dsl_type = 7; // must - repeated string output_fields = 8; - repeated common.KeyValuePair search_params = 9; // must - uint64 travel_timestamp = 10; - uint64 guarantee_timestamp = 11; // guarantee_timestamp -} - -message Hits { - repeated int64 IDs = 1; - repeated bytes row_data = 2; - repeated float scores = 3; -} - -message SearchResults { - common.Status status = 1; - schema.SearchResultData results = 2; - string collection_name = 3; -} - -message FlushRequest { - common.MsgBase base = 1; - string db_name = 2; - repeated string collection_names = 3; -} - -message FlushResponse{ - common.Status status = 1; - string db_name = 2; - map coll_segIDs = 3; -} - -message QueryRequest { - common.MsgBase base = 1; - string db_name = 2; - string collection_name = 3; - string expr = 4; - repeated string output_fields = 5; - repeated string partition_names = 6; - uint64 travel_timestamp = 7; - uint64 guarantee_timestamp = 8; // guarantee_timestamp -} - -message QueryResults { - common.Status status = 1; - repeated schema.FieldData fields_data = 2; - string collection_name = 3; -} - -message VectorIDs { - string collection_name = 1; - string field_name = 2; - schema.IDs id_array = 3; - repeated string partition_names = 4; -} - -message VectorsArray { - oneof array { - VectorIDs id_array = 1; // vector ids - schema.VectorField data_array = 2; // vectors data - } -} - -message CalcDistanceRequest { - common.MsgBase base = 1; - VectorsArray op_left = 2; // vectors on the left of operator - VectorsArray op_right = 3; // vectors on the right of operator - repeated common.KeyValuePair params = 4; // "metric":"L2"/"IP"/"HAMMIN"/"TANIMOTO" -} - -message CalcDistanceResults { - common.Status status = 1; - // num(op_left)*num(op_right) distance values, "HAMMIN" return integer distance - oneof array { - schema.IntArray int_dist = 2; - schema.FloatArray float_dist = 3; - } -} - -message PersistentSegmentInfo { - int64 segmentID = 1; - int64 collectionID = 2; - int64 partitionID = 3; - int64 num_rows = 4; - common.SegmentState state = 5; -} - -message GetPersistentSegmentInfoRequest { - common.MsgBase base = 1; // must - string dbName = 2; - string collectionName = 3; // must -} - -message GetPersistentSegmentInfoResponse { - common.Status status = 1; - repeated PersistentSegmentInfo infos = 2; -} - -message QuerySegmentInfo { - int64 segmentID = 1; - int64 collectionID = 2; - int64 partitionID = 3; - int64 mem_size = 4; - int64 num_rows = 5; - string index_name = 6; - int64 indexID = 7; - int64 nodeID = 8; - common.SegmentState state = 9; -} - -message GetQuerySegmentInfoRequest { - common.MsgBase base = 1; // must - string dbName = 2; - string collectionName = 3; // must -} - -message GetQuerySegmentInfoResponse { - common.Status status = 1; - repeated QuerySegmentInfo infos = 2; -} - -message DummyRequest { - string request_type = 1; -} - -message DummyResponse { - string response = 1; -} - -message RegisterLinkRequest { -} - -message RegisterLinkResponse { - common.Address address = 1; - common.Status status = 2; -} - -message GetMetricsRequest { - common.MsgBase base = 1; - string request = 2; // request is of jsonic format -} - -message GetMetricsResponse { - common.Status status = 1; - string response = 2; // response is of jsonic format - string component_name = 3; // metrics from which component -} - -/* -* Do load balancing operation from src_nodeID to dst_nodeID. -*/ -message LoadBalanceRequest { - common.MsgBase base = 1; - int64 src_nodeID = 2; - repeated int64 dst_nodeIDs = 3; - repeated int64 sealed_segmentIDs = 4; -} - -message ManualCompactionRequest { - int64 collectionID = 1; - uint64 timetravel = 2; -} - -message ManualCompactionResponse { - common.Status status = 1; - int64 compactionID = 2; -} - -message GetCompactionStateRequest { - int64 compactionID = 1; -} - -message GetCompactionStateResponse { - common.Status status = 1; - common.CompactionState state = 2; - int64 executingPlanNo = 3; - int64 timeoutPlanNo = 4; - int64 completedPlanNo = 5; -} - -message GetCompactionPlansRequest { - int64 compactionID = 1; -} - -message GetCompactionPlansResponse { - common.Status status = 1; - common.CompactionState state = 2; - repeated CompactionMergeInfo mergeInfos = 3; -} - -message CompactionMergeInfo { - repeated int64 sources = 1; - int64 target = 2; -} - -message GetFlushStateRequest { - repeated int64 segmentIDs = 1; -} - -message GetFlushStateResponse { - common.Status status = 1; - bool flushed = 2; -} - -message ImportRequest { - string collection_name = 1; // target collection - string partition_name = 2; // target partition - bool row_based = 3; // the file is row-based or column-based - repeated string files = 4; // file paths to be imported - repeated common.KeyValuePair options = 5; // import options -} - -message ImportResponse { - common.Status status = 1; - repeated int64 tasks = 2; // id array of import tasks -} - -message GetImportStateRequest { - int64 task = 1; // id of an import task -} - -message GetImportStateResponse { - common.Status status = 1; - common.ImportState state = 2; // is this import task finished or not - int64 row_count = 3; // if the task is finished, this value is how many rows are imported. if the task is not finished, this value is how many rows are parsed. -} - -service ProxyService { - rpc RegisterLink(RegisterLinkRequest) returns (RegisterLinkResponse) {} -} diff --git a/grpc-proto/schema.proto b/grpc-proto/schema.proto deleted file mode 100644 index c72d64d6..00000000 --- a/grpc-proto/schema.proto +++ /dev/null @@ -1,125 +0,0 @@ -syntax = "proto3"; - -package milvus.proto.schema; -option go_package = "github.com/milvus-io/milvus/internal/proto/schemapb"; - -import "common.proto"; - -/** - * @brief Field data type - */ -enum DataType { - None = 0; - Bool = 1; - Int8 = 2; - Int16 = 3; - Int32 = 4; - Int64 = 5; - - Float = 10; - Double = 11; - - String = 20; - - BinaryVector = 100; - FloatVector = 101; -} - -/** - * @brief Field schema - */ -message FieldSchema { - int64 fieldID = 1; - string name = 2; - bool is_primary_key = 3; - string description = 4; - DataType data_type = 5; - repeated common.KeyValuePair type_params = 6; - repeated common.KeyValuePair index_params = 7; - bool autoID = 8; -} - -/** - * @brief Collection schema - */ -message CollectionSchema { - string name = 1; - string description = 2; - bool autoID = 3; // deprecated later, keep compatible with c++ part now - repeated FieldSchema fields = 4; -} - -message BoolArray { - repeated bool data = 1; -} - -message IntArray { - repeated int32 data = 1; -} - -message LongArray { - repeated int64 data = 1; -} - -message FloatArray { - repeated float data = 1; -} - -message DoubleArray { - repeated double data = 1; -} - -// For special fields such as bigdecimal, array... -message BytesArray { - repeated bytes data = 1; -} - -message StringArray { - repeated string data = 1; -} - -message ScalarField { - oneof data { - BoolArray bool_data = 1; - IntArray int_data = 2; - LongArray long_data = 3; - FloatArray float_data = 4; - DoubleArray double_data = 5; - StringArray string_data = 6; - BytesArray bytes_data = 7; - } -} - -message VectorField { - int64 dim = 1; - oneof data { - FloatArray float_vector = 2; - bytes binary_vector = 3; - } -} - -message FieldData { - DataType type = 1; - string field_name = 2; - oneof field { - ScalarField scalars = 3; - VectorField vectors = 4; - } - int64 field_id = 5; -} - -message IDs { - oneof id_field { - LongArray int_id = 1; - StringArray str_id = 2; - } -} - -message SearchResultData { - int64 num_queries = 1; - int64 top_k = 2; - repeated FieldData fields_data = 3; - repeated float scores = 4; - IDs ids = 5; - repeated int64 topks = 6; -} diff --git a/milvus/Collection.ts b/milvus/Collection.ts index eb063d9c..201f8e08 100644 --- a/milvus/Collection.ts +++ b/milvus/Collection.ts @@ -36,7 +36,7 @@ import path from "path"; import { formatKeyValueData, parseToKeyValue } from "./utils/Format"; import { Client } from "./Client"; -const schemaPath = path.resolve(__dirname, "../grpc-proto/schema.proto"); +const schemaPath = path.resolve(__dirname, "../proto/proto/schema.proto"); /** * See all [collection operation examples](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/Collection.ts). diff --git a/milvus/Data.ts b/milvus/Data.ts index 6541957d..30d03b74 100644 --- a/milvus/Data.ts +++ b/milvus/Data.ts @@ -41,7 +41,7 @@ import { import path from "path"; import { formatNumberPrecision, parseToKeyValue } from "./utils/Format"; -const protoPath = path.resolve(__dirname, "../grpc-proto/milvus.proto"); +const protoPath = path.resolve(__dirname, "../proto/proto/milvus.proto"); export class Data extends Client { vectorTypes: number[]; diff --git a/milvus/index.ts b/milvus/index.ts index 81c7fe97..7c0ab71a 100644 --- a/milvus/index.ts +++ b/milvus/index.ts @@ -9,7 +9,7 @@ import sdkInfo from "../sdk.json"; import { ERROR_REASONS } from "./const/ErrorReason"; import { ErrorCode } from "./types/Response"; -const protoPath = path.resolve(__dirname, "../grpc-proto/milvus.proto"); +const protoPath = path.resolve(__dirname, "../proto/proto/milvus.proto"); export class MilvusClient { client: Client; collectionManager: Collection; diff --git a/package.json b/package.json index 50f38faa..07b4cec3 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ ], "scripts": { "build": "tsc --declaration && node build.js", - "test": "jest", + "test": "jest --testPathIgnorePatterns=/test/build/", "coverage": "jest --coverage=true --testPathIgnorePatterns=/test/build/", "build-test": "yarn build && jest test/build/Collection.spec.ts", "example": "npx ts-node", diff --git a/proto b/proto new file mode 160000 index 00000000..aec1f8a8 --- /dev/null +++ b/proto @@ -0,0 +1 @@ +Subproject commit aec1f8a8f58fc7336d3705fa35b3ad1cf3ad69b3 diff --git a/tsconfig.json b/tsconfig.json index 7e970d43..c81dd67a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -69,5 +69,5 @@ "skipLibCheck": true /* Skip type checking of declaration files. */, "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ }, - "include": ["milvus/**/*", "milvus/grpc-proto/*.proto"] + "include": ["milvus/**/*", "milvus/proto/proto/*.proto"] } From d33c91c84110d7a7ceefc91314477777566e42fd Mon Sep 17 00:00:00 2001 From: nameczz Date: Sat, 19 Mar 2022 12:05:41 +0800 Subject: [PATCH 2/3] add submodule ci Signed-off-by: nameczz --- .github/workflows/check.yml | 5 +++++ .github/workflows/ci.yml | 5 +++++ .github/workflows/merge.yml | 5 +++++ README.md | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index f5733683..efd8acbb 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -21,6 +21,11 @@ jobs: wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/standalone/docker-compose.yml -O docker-compose.yml docker-compose up -d + - name: update submodules + run: | + git submodule init + git submodule update --remote + - name: Run Test coverage run: | yarn diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fad1f96..8af01c62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,11 @@ jobs: - name: Install dependencies run: yarn + + - name: update submodules + run: | + git submodule init + git submodule update --remote - name: Build app run: | diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 3e6fd668..993f41e0 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -19,6 +19,11 @@ jobs: run: | wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/standalone/docker-compose.yml -O docker-compose.yml docker-compose up -d + + - name: update submodules + run: | + git submodule init + git submodule update --remote - name: Run Test coverage run: | diff --git a/README.md b/README.md index 5b3affab..a34daa00 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,11 @@ Node: v12+ | compact | true | Do compaction for the collection | | getCompactionState | true | Get compaction states by compact id | +## How to dev +1. yarn +2. git submodule init (if this is your first time) +3. git submodule update --remote +4. In milvus folder has all milvus related functions. ## Example 1. [Hello World](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/HelloMilvus.ts) From 0731f94a3207fa0eb1be02dd7c0fd436228a87f9 Mon Sep 17 00:00:00 2001 From: nameczz Date: Sat, 19 Mar 2022 12:16:45 +0800 Subject: [PATCH 3/3] fix ci Signed-off-by: nameczz --- .github/workflows/merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 993f41e0..503af25f 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -20,7 +20,7 @@ jobs: wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/standalone/docker-compose.yml -O docker-compose.yml docker-compose up -d - - name: update submodules + - name: update submodules run: | git submodule init git submodule update --remote