From 03dbf7ab4e373047658ad656128c1be301491790 Mon Sep 17 00:00:00 2001 From: nameczz Date: Tue, 7 Jun 2022 12:04:36 +0800 Subject: [PATCH] fix test Signed-off-by: nameczz --- milvus/Data.ts | 110 ++++++++++++++++++++-------------------- proto | 2 +- test/Collection.spec.ts | 105 +++++++++++++++++++------------------- 3 files changed, 109 insertions(+), 108 deletions(-) diff --git a/milvus/Data.ts b/milvus/Data.ts index 30d03b74..55ac4a33 100644 --- a/milvus/Data.ts +++ b/milvus/Data.ts @@ -1,10 +1,10 @@ -import protobuf from "protobufjs"; -import { promisify } from "../utils"; -import { Client } from "./Client"; -import { Collection } from "./Collection"; -import { ERROR_REASONS } from "./const/ErrorReason"; +import protobuf from 'protobufjs'; +import { promisify } from '../utils'; +import { Client } from './Client'; +import { Collection } from './Collection'; +import { ERROR_REASONS } from './const/ErrorReason'; -import { DataType, DataTypeMap, DslType } from "./types/Common"; +import { DataType, DataTypeMap, DslType } from './types/Common'; import { CalcDistanceReq, DeleteEntitiesReq, @@ -13,7 +13,7 @@ import { GetQuerySegmentInfoReq, InsertReq, LoadBalanceReq, -} from "./types/Data"; +} from './types/Data'; import { CalcDistanceResponse, ErrorCode, @@ -25,23 +25,23 @@ import { QueryResults, ResStatus, SearchResults, -} from "./types/Response"; +} from './types/Response'; import { GetMetricsRequest, QueryReq, QueryRes, SearchReq, SearchRes, -} from "./types/Search"; -import { findKeyValue, sleep } from "./utils/index"; +} from './types/Search'; +import { findKeyValue, sleep } from './utils/index'; import { parseBinaryVectorToBytes, parseFloatVectorToBytes, -} from "./utils/Blob"; -import path from "path"; -import { formatNumberPrecision, parseToKeyValue } from "./utils/Format"; +} from './utils/Blob'; +import path from 'path'; +import { formatNumberPrecision, parseToKeyValue } from './utils/Format'; -const protoPath = path.resolve(__dirname, "../proto/proto/milvus.proto"); +const protoPath = path.resolve(__dirname, '../proto/proto/milvus.proto'); export class Data extends Client { vectorTypes: number[]; @@ -106,11 +106,11 @@ export class Data extends Client { // Tip: The field data sequence needs to be set same as `collectionInfo.schema.fields`. // If primarykey is set `autoid = true`, you cannot insert the data. const fieldsData = collectionInfo.schema.fields - .filter((v) => !v.is_primary_key || !v.autoID) - .map((v) => ({ + .filter(v => !v.is_primary_key || !v.autoID) + .map(v => ({ name: v.name, type: v.data_type, - dim: Number(findKeyValue(v.type_params, "dim")), + dim: Number(findKeyValue(v.type_params, 'dim')), value: [] as number[], })); @@ -122,8 +122,8 @@ export class Data extends Client { // Set the key as the field name to get all names in a row. const fieldNames = Object.keys(v); - fieldNames.forEach((name) => { - const target = fieldsData.find((item) => item.name === name); + fieldNames.forEach(name => { + const target = fieldsData.find(item => item.name === name); if (!target) { throw new Error(`${ERROR_REASONS.INSERT_CHECK_WRONG_FIELD} ${i}`); } @@ -151,35 +151,35 @@ export class Data extends Client { }); }); - params.fields_data = fieldsData.map((v) => { + params.fields_data = fieldsData.map(v => { // milvus return string for field type, so we define the DataTypeMap to the value we need. // but if milvus change the string, may casue we cant find value. const type = DataTypeMap[v.type.toLowerCase()]; - const key = this.vectorTypes.includes(type) ? "vectors" : "scalars"; - let dataKey = "float_vector"; + const key = this.vectorTypes.includes(type) ? 'vectors' : 'scalars'; + let dataKey = 'float_vector'; switch (type) { case DataType.FloatVector: - dataKey = "float_vector"; + dataKey = 'float_vector'; break; case DataType.BinaryVector: - dataKey = "binary_vector"; + dataKey = 'binary_vector'; break; case DataType.Double: - dataKey = "double_data"; + dataKey = 'double_data'; break; case DataType.Float: - dataKey = "float_data"; + dataKey = 'float_data'; break; case DataType.Int64: - dataKey = "long_data"; + dataKey = 'long_data'; break; case DataType.Int32: case DataType.Int16: case DataType.Int8: - dataKey = "int_data"; + dataKey = 'int_data'; break; case DataType.Bool: - dataKey = "bool_data"; + dataKey = 'bool_data'; break; default: throw new Error(ERROR_REASONS.INSERT_CHECK_WRONG_DATA_TYPE); @@ -208,7 +208,7 @@ export class Data extends Client { }; }); - const promise = await promisify(this.client, "Insert", params); + const promise = await promisify(this.client, 'Insert', params); return promise; } @@ -243,7 +243,7 @@ export class Data extends Client { if (!data || !data.collection_name || !data.expr) { throw new Error(ERROR_REASONS.DELETE_PARAMS_CHECK); } - const promise = await promisify(this.client, "Delete", data); + const promise = await promisify(this.client, 'Delete', data); return promise; } @@ -309,13 +309,13 @@ export class Data extends Client { // anns_field is the vector field column user want to compare. const targetField = collectionInfo.schema.fields.find( - (v) => v.name === data.search_params.anns_field + v => v.name === data.search_params.anns_field ); if (!targetField) { throw new Error(ERROR_REASONS.SEARCH_NOT_FIND_VECTOR_FIELD); } - const dim = findKeyValue(targetField.type_params, "dim"); + const dim = findKeyValue(targetField.type_params, 'dim'); const vectorType = DataTypeMap[targetField.data_type.toLowerCase()]; const dimension = vectorType === DataType.BinaryVector ? Number(dim) / 8 : Number(dim); @@ -326,15 +326,15 @@ export class Data extends Client { // when data type is bytes , we need use protobufjs to transform data to buffer bytes. const PlaceholderGroup = root.lookupType( - "milvus.proto.milvus.PlaceholderGroup" + 'milvus.proto.common.PlaceholderGroup' ); // tag $0 is hard code in milvus, when dsltype is expr const placeholderGroupParams = PlaceholderGroup.create({ placeholders: [ { - tag: "$0", + tag: '$0', type: data.vector_type, - values: data.vectors.map((v) => + values: data.vectors.map(v => data.vector_type === DataType.BinaryVector ? parseBinaryVectorToBytes(v) : parseFloatVectorToBytes(v) @@ -347,9 +347,9 @@ export class Data extends Client { placeholderGroupParams ).finish(); - const promise: SearchRes = await promisify(this.client, "Search", { + const promise: SearchRes = await promisify(this.client, 'Search', { ...data, - dsl: data.expr || "", + dsl: data.expr || '', dsl_type: DslType.BoolExprV1, placeholder_group: placeholderGroupBytes, search_params: parseToKeyValue(data.search_params), @@ -376,7 +376,7 @@ export class Data extends Client { return { type: item.type, field_name: item.field_name, - data: value ? value[value?.data].data : "", + data: value ? value[value?.data].data : '', }; }); // verctor id support int / str id. @@ -393,15 +393,15 @@ export class Data extends Client { scores.splice(0, topk).forEach((score, scoreIndex) => { const i = index === 0 ? scoreIndex : scoreIndex + topk; const fixedScore = - typeof round_decimal === "undefined" || round_decimal === -1 + typeof round_decimal === 'undefined' || round_decimal === -1 ? score : formatNumberPrecision(score, round_decimal); const result: any = { score: fixedScore, - id: idData ? idData[i] : "", + id: idData ? idData[i] : '', }; - fieldsData.forEach((field) => { + fieldsData.forEach(field => { result[field.field_name] = field.data[i]; }); results.push(result); @@ -444,7 +444,7 @@ export class Data extends Client { ) { throw new Error(ERROR_REASONS.COLLECTION_NAME_IS_REQUIRED); } - const res = await promisify(this.client, "Flush", data); + const res = await promisify(this.client, 'Flush', data); return res; } @@ -479,10 +479,10 @@ export class Data extends Client { throw new Error(ERROR_REASONS.COLLECTION_NAME_IS_REQUIRED); } // copy flushed collection names - const res = await promisify(this.client, "Flush", data); + const res = await promisify(this.client, 'Flush', data); // After flush will return collection segment ids, need use GetPersistentSegmentInfo to check segment flush status. const segIDs = Object.keys(res.coll_segIDs) - .map((v) => res.coll_segIDs[v].data) + .map(v => res.coll_segIDs[v].data) .reduce((pre, cur) => [...pre, ...cur], []); let isFlushed = false; @@ -528,7 +528,7 @@ export class Data extends Client { */ async query(data: QueryReq): Promise { this.checkCollectionName(data); - const promise: QueryRes = await promisify(this.client, "Query", data); + const promise: QueryRes = await promisify(this.client, 'Query', data); const results: { [x: string]: any }[] = []; /** * type: DataType @@ -539,16 +539,16 @@ export class Data extends Client { * scalars: scalar data */ const fieldsData = promise.fields_data.map((item, i) => { - if (item.field === "vectors") { + if (item.field === 'vectors') { const key = item.vectors!.data; const vectorValue = - key === "float_vector" + key === 'float_vector' ? item.vectors![key]!.data : item.vectors![key]!.toJSON().data; // if binary vector , need use dim / 8 to split vector data const dim = - item.vectors?.data === "float_vector" + item.vectors?.data === 'float_vector' ? Number(item.vectors!.dim) : Number(item.vectors!.dim) / 8; const data: number[][] = []; @@ -578,7 +578,7 @@ export class Data extends Client { }); // parse column data to [{fieldname:value}] - fieldsData.forEach((v) => { + fieldsData.forEach(v => { v.data.forEach((d: string | number[], i: number) => { if (!results[i]) { results[i] = { @@ -609,7 +609,7 @@ export class Data extends Client { if (!data || !data.request || !data.request.metric_type) { throw new Error(ERROR_REASONS.GET_METRIC_CHECK_PARAMS); } - const res: GetMetricsResponse = await promisify(this.client, "GetMetrics", { + const res: GetMetricsResponse = await promisify(this.client, 'GetMetrics', { request: JSON.stringify(data.request), }); @@ -624,7 +624,7 @@ export class Data extends Client { * @param data */ async calcDistance(data: CalcDistanceReq): Promise { - const res = await promisify(this.client, "CalcDistance", data); + const res = await promisify(this.client, 'CalcDistance', data); return res; } @@ -657,7 +657,7 @@ export class Data extends Client { if (!data || !data.segmentIDs) { throw new Error(ERROR_REASONS.GET_FLUSH_STATE_CHECK_PARAMS); } - const res = await promisify(this.client, "GetFlushState", data); + const res = await promisify(this.client, 'GetFlushState', data); return res; } @@ -692,7 +692,7 @@ export class Data extends Client { if (!data || !data.src_nodeID) { throw new Error(ERROR_REASONS.LOAD_BALANCE_CHECK_PARAMS); } - const res = await promisify(this.client, "LoadBalance", data); + const res = await promisify(this.client, 'LoadBalance', data); return res; } @@ -727,7 +727,7 @@ export class Data extends Client { if (!data || !data.collectionName) { throw new Error(ERROR_REASONS.COLLECTION_NAME_IS_REQUIRED); } - const res = await promisify(this.client, "GetQuerySegmentInfo", data); + const res = await promisify(this.client, 'GetQuerySegmentInfo', data); return res; } } diff --git a/proto b/proto index f5682874..3b772e76 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit f56828744189e6b7b71d104ff2d8b37a13e089d7 +Subproject commit 3b772e76d2040f1a7951efe7a792d22e538811f3 diff --git a/test/Collection.spec.ts b/test/Collection.spec.ts index e79527ec..cf1370ba 100644 --- a/test/Collection.spec.ts +++ b/test/Collection.spec.ts @@ -1,22 +1,22 @@ -import { MilvusClient } from "../milvus"; +import { MilvusClient } from '../milvus'; -import { GENERATE_NAME, IP } from "../const"; -import { DataType } from "../milvus/types/Common"; -import { ErrorCode } from "../milvus/types/Response"; -import { ShowCollectionsType } from "../milvus/types/Collection"; -import { ERROR_REASONS } from "../milvus/const/ErrorReason"; -import { genCollectionParams, VECTOR_FIELD_NAME } from "../utils/test"; +import { GENERATE_NAME, IP } from '../const'; +import { DataType } from '../milvus/types/Common'; +import { ErrorCode } from '../milvus/types/Response'; +import { ShowCollectionsType } from '../milvus/types/Collection'; +import { ERROR_REASONS } from '../milvus/const/ErrorReason'; +import { genCollectionParams, VECTOR_FIELD_NAME } from '../utils/test'; const milvusClient = new MilvusClient(IP); const collectionManager = milvusClient.collectionManager; const COLLECTION_NAME = GENERATE_NAME(); const LOAD_COLLECTION_NAME = GENERATE_NAME(); -describe("Collection Api", () => { +describe('Collection Api', () => { it(`Create Collection Successful`, async () => { const res = await collectionManager.createCollection({ - ...genCollectionParams(COLLECTION_NAME, "128"), - consistency_level: "Eventually", + ...genCollectionParams(COLLECTION_NAME, '128'), + consistency_level: 'Eventually', }); expect(res.error_code).toEqual(ErrorCode.SUCCESS); }); @@ -24,7 +24,7 @@ describe("Collection Api", () => { it(`Create Collection validate fields`, async () => { try { await collectionManager.createCollection({ - collection_name: "zxc", + collection_name: 'zxc', } as any); } catch (error) { expect(error.message).toEqual( @@ -33,11 +33,11 @@ describe("Collection Api", () => { } try { await collectionManager.createCollection({ - collection_name: "zxc", + collection_name: 'zxc', fields: [ { - name: "vector_01", - description: "vector field", + name: 'vector_01', + description: 'vector field', data_type: DataType.FloatVector, }, ], @@ -50,11 +50,11 @@ describe("Collection Api", () => { try { await collectionManager.createCollection({ - collection_name: "zxc", + collection_name: 'zxc', fields: [ { - name: "age", - description: "", + name: 'age', + description: '', data_type: DataType.Int64, is_primary_key: true, }, @@ -70,16 +70,16 @@ describe("Collection Api", () => { it(`Create Collection expect dim error`, async () => { try { await collectionManager.createCollection({ - collection_name: "zxc", + collection_name: 'zxc', fields: [ { - name: "vector_01", - description: "vector field", + name: 'vector_01', + description: 'vector field', data_type: DataType.FloatVector, }, { - name: "age", - description: "", + name: 'age', + description: '', data_type: DataType.Int64, is_primary_key: true, }, @@ -93,7 +93,7 @@ describe("Collection Api", () => { try { await collectionManager.createCollection( - genCollectionParams("any", "10") + genCollectionParams('any', '10') ); } catch (error) { expect(error.message).toEqual( @@ -102,22 +102,22 @@ describe("Collection Api", () => { } }); - it("Create collection should throw CREATE_COLLECTION_CHECK_BINARY_DIM", async () => { + it('Create collection should throw CREATE_COLLECTION_CHECK_BINARY_DIM', async () => { try { await collectionManager.createCollection({ - collection_name: "zxc", + collection_name: 'zxc', fields: [ { - name: "vector_01", - description: "vector field", + name: 'vector_01', + description: 'vector field', data_type: DataType.BinaryVector, type_params: { - dim: "7", + dim: '7', }, }, { - name: "age", - description: "", + name: 'age', + description: '', data_type: DataType.Int64, is_primary_key: true, }, @@ -130,7 +130,7 @@ describe("Collection Api", () => { } }); - it("Create collection should throw check params error", async () => { + it('Create collection should throw check params error', async () => { try { await collectionManager.createCollection({} as any); } catch (error) { @@ -142,7 +142,7 @@ describe("Collection Api", () => { it(`Create load Collection Successful`, async () => { const res = await collectionManager.createCollection( - genCollectionParams(LOAD_COLLECTION_NAME, "128") + genCollectionParams(LOAD_COLLECTION_NAME, '128') ); console.log(res); expect(res.error_code).toEqual(ErrorCode.SUCCESS); @@ -160,12 +160,12 @@ describe("Collection Api", () => { const res = await collectionManager.hasCollection({ collection_name: COLLECTION_NAME, }); - console.log("----has collection", res); + console.log('----has collection', res); expect(res.status.error_code).toEqual(ErrorCode.SUCCESS); expect(res.value).toEqual(true); }); - it("Has collection should throw check params error", async () => { + it('Has collection should throw check params error', async () => { try { await collectionManager.hasCollection({} as any); } catch (error) { @@ -175,7 +175,7 @@ describe("Collection Api", () => { it(`Has collection not exist`, async () => { const res = await collectionManager.hasCollection({ - collection_name: "collection_not_exist", + collection_name: 'collection_not_exist', }); expect(res.value).toEqual(false); }); @@ -184,9 +184,7 @@ describe("Collection Api", () => { const res = await collectionManager.showCollections(); console.log(res); expect(res.status.error_code).toEqual(ErrorCode.SUCCESS); - expect(res.data.filter((v) => v.name === COLLECTION_NAME).length).toEqual( - 1 - ); + expect(res.data.filter(v => v.name === COLLECTION_NAME).length).toEqual(1); }); it(`Get Collection Statistics should throw error`, async () => { @@ -202,21 +200,21 @@ describe("Collection Api", () => { collection_name: COLLECTION_NAME, }); expect(res.status.error_code).toEqual(ErrorCode.SUCCESS); - expect(res.stats[0].value).toEqual("0"); - expect(res.data.row_count).toEqual("0"); + expect(res.stats[0].value).toEqual('0'); + expect(res.data.row_count).toEqual('0'); }); - it("Describe Collection info", async () => { + it('Describe Collection info', async () => { const res = await collectionManager.describeCollection({ collection_name: COLLECTION_NAME, }); - console.log("---- describe collection ---", res); + console.log('---- describe collection ---', res); expect(res.status.error_code).toEqual(ErrorCode.SUCCESS); - expect(res.consistency_level).toEqual("Eventually"); + expect(res.consistency_level).toEqual('Eventually'); expect(res.schema.name).toEqual(COLLECTION_NAME); expect(res.schema.fields.length).toEqual(2); expect(res.schema.fields[0].name).toEqual(VECTOR_FIELD_NAME); - expect(res.schema.fields[1].name).toEqual("age"); + expect(res.schema.fields[1].name).toEqual('age'); }); it(`Load Collection Sync throw COLLECTION_NAME_IS_REQUIRED`, async () => { @@ -238,11 +236,11 @@ describe("Collection Api", () => { const fakeClient = new MilvusClient(IP); fakeClient.collectionManager.showCollections = () => { - return new Promise((res) => { + return new Promise(res => { res({ status: { - error_code: "error", - reason: "123", + error_code: 'error', + reason: '123', }, } as any); }); @@ -252,7 +250,7 @@ describe("Collection Api", () => { collection_name: LOAD_COLLECTION_NAME, }); } catch (error) { - expect(typeof error.message).toBe("string"); + expect(typeof error.message).toBe('string'); } finally { fakeClient.closeConnection(); } @@ -267,6 +265,9 @@ describe("Collection Api", () => { }); it(`Load Collection Async success`, async () => { + await collectionManager.releaseCollection({ + collection_name: LOAD_COLLECTION_NAME, + }); const res = await collectionManager.loadCollection({ collection_name: LOAD_COLLECTION_NAME, }); @@ -280,11 +281,11 @@ describe("Collection Api", () => { expect(res.status.error_code).toEqual(ErrorCode.SUCCESS); expect( - res.data.filter((v) => v.name === LOAD_COLLECTION_NAME).length + res.data.filter(v => v.name === LOAD_COLLECTION_NAME).length ).toEqual(1); }); - it("Compact collection and get state expect success", async () => { + it('Compact collection and get state expect success', async () => { const res = await collectionManager.compact({ collection_name: LOAD_COLLECTION_NAME, }); @@ -299,7 +300,7 @@ describe("Collection Api", () => { expect(stateWithPlan.status.error_code).toEqual(ErrorCode.SUCCESS); }); - it("Compact collection expect throw error", async () => { + it('Compact collection expect throw error', async () => { try { await collectionManager.compact({ collection_name: undefined as any, @@ -309,7 +310,7 @@ describe("Collection Api", () => { } }); - it("Get Compaction State and with plan throw error", async () => { + it('Get Compaction State and with plan throw error', async () => { try { await collectionManager.getCompactionState({ compactionID: undefined as any,