Merge pull request #119 from milvus-io/dev

Dev

.github/workflows/check.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches:
       - main
-    types: [opened]
+    types: [opened, synchronize]
 
 jobs:
   publish:

.github/workflows/devCheck.yml

@@ -0,0 +1,34 @@
+name: Milvus2 Node sdk check when open pr
+
+on:
+  pull_request:
+    branches:
+      - dev
+    types: [opened, synchronize]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14
+
+      - name: Run Milvus
+        run: |
+          wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/standalone/docker-compose.yml -O docker-compose.yml
+          sed -i -e 's/milvusdb.*$/milvusdb\/milvus:master-latest/g' docker-compose.yml
+          docker-compose up -d
+
+      - name: update submodules
+        run: |
+          git submodule init
+          yarn pre
+
+      - name: Run Test coverage
+        run: |
+          yarn
+          yarn build-test
+          yarn coverage

.github/workflows/pre-release.yml

@@ -0,0 +1,31 @@
+name: Node sdk 2.x pre release
+
+on:
+  release:
+    types: [prereleased]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14
+
+      - name: Install dependencies
+        run: yarn
+
+      - name: update submodules
+        run: |
+          git submodule init
+          yarn pre
+
+      - name: Build app
+        run: |
+          yarn build
+
+      - uses: JS-DevTools/npm-publish@v1
+        with:
+          token: ${{ secrets.NPM_TOKEN }}

README.md

@@ -5,7 +5,7 @@
 
 # Milvus2-sdk-node
 
-This is node sdk for [Milvus](https://github.com/milvus-io/milvus).
+Node.js sdk for [Milvus](https://github.com/milvus-io/milvus).
 
 ## Compatibility
 
@@ -15,62 +15,49 @@ The following collection shows Milvus versions and recommended @zilliz/milvus2-s
 | :------------: | :------------------------------------------: |
 |     2.0.0      |                    2.0.0                     |
 |     2.0.1      |                 2.0.0, 2.0.1                 |
+|     2.1.x      |                    2.1.x                     |
+|     2.2.x      |                    2.2.x                     |
 
 ## Dependencies
 
-Milvus: v2+
-
-Node: v12+
+- Milvus: v2+
+- Node: v12+
 
 ## Installation
 
 ```javascript
-   npm install @zilliz/milvus2-sdk-node
+npm install @zilliz/milvus2-sdk-node
 ```
 
-## API Support
-
-| Name                    | Async |               Description                |
-| :---------------------- | :---- | :--------------------------------------: |
-| createCollection        | false |       Create collection in milvus        |
-| hasCollection           | false |      Check collection exist or not       |
-| showCollections         | false |           List all collections           |
-| describeCollection      | false | Get collection detail, like name ,schema |
-| getCollectionStatistics | false | Get collection statistics like row count |
-| loadCollection          | true  |           Load data into cache           |
-| releaseCollection       | true  |         Release data from cache          |
-| dropCollection          | false |       Drop collection from milvus        |
-| createPartition         | false |    Create partition in one collection    |
-| hasPartition            | false |       Check partition exist or not       |
-| showPartitions          | false |  List all partitions for one collection  |
-| getPartitionStatistics  | false | Get partition statistics like row_count  |
-| loadPartitions          | true  |           Load data into cache           |
-| releasePartitions       | true  |         Release data from cache          |
-| dropPartition           | false |        Drop partition from milvus        |
-| createIndex             | true  |       Creat index on vector field        |
-| describeIndex           | false |              Get index info              |
-| getIndexState           | false |          Get index build state           |
-| getIndexBuildProgress   | false |       Get index building progress        |
-| dropIndex               | true  |                Drop index                |
-| insert                  | false |         Insert data into milvus          |
-| search                  | false |         Vector similarity search         |
-| flush                   | true  |                Flush Data                |
-| flushSync               | false |             Flush Data Sync              |
-| query                   | false |             Get data by expr             |
-| compact                 | true  |     Do compaction for the collection     |
-| getCompactionState      | true  |   Get compaction states by compact id    |
+## SDK
+
+- [Client](./milvus/Index.ts)
+- [Collection](./milvus/Collection.ts)
+- [Index](./milvus/MilvusIndex.ts)
+- [Partition](./milvus/Partition.ts)
+- [User](./milvus/User.ts)
+- [Utils](./milvus/Utils.ts)
+- [Data](./milvus/Data.ts)
+
+More documentation, you can refer [Milvus offical website](https://milvus.io/).
 
 ## How to dev
+
 1. yarn install
 2. Fetch milvus proto
-   1. ```git submodule init``` (if this is your first time)
-   2. ```git submodule update --remote```
+   1. `git submodule init` (if this is your first time)
+   2. `git submodule update --remote`
 3. Add feature in milvus folder.
-4. Run test ```yarn test -- test/Your-test-for-your-feature.spec.ts```
+4. Run test `yarn test -- test/Your-test-for-your-feature.spec.ts`
+
 ## Example
 
 1. [Hello World](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/HelloMilvus.ts)
 2. [How to operate collection](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/Collection.ts)
 3. [How to insert data](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/Insert.ts)
 4. [Vector similarity search on float field](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/Search.ts)
 5. [Vector similarity search on binary field](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/BinarySearch.ts)
+
+## Others
+
+[Attu](https://github.com/zilliztech/attu) which is a Milvus web interface tool, depends on Milvus node.js SDK.

jest.config.js

@@ -3,7 +3,7 @@ module.exports = {
   testEnvironment: 'node',
   testRegex: '/test/.*\\.(test|spec)?\\.(ts|tsx)$',
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
-  testTimeout: 30000,
+  testTimeout: 60000,
   // because user will cause other test fail, but we still have user spec
   coveragePathIgnorePatterns: ['/milvus/User.ts'],
 };

milvus/Collection.ts

@@ -379,12 +379,18 @@ export class Collection extends Client {
       data,
       data.timeout
     );
+
+    if (promise.error_code !== ErrorCode.SUCCESS) {
+      throw new Error(`ErrorCode: ${promise.error_code}. Reason: ${promise.reason}`);
+    }
+
     let loadedPercentage = 0;
     while (Number(loadedPercentage) < 100) {
       let res = await this.showCollections({
         collection_names: [data.collection_name],
         type: ShowCollectionsType.Loaded,
       });
+
       if (res.status.error_code !== ErrorCode.SUCCESS) {
         throw new Error(
           `ErrorCode: ${res.status.error_code}. Reason: ${res.status.reason}`
@@ -658,7 +664,7 @@ export class Collection extends Client {
    *  });
    *
    * ```
-   * 
+   *
    * Return
    * ```
    * {

milvus/Data.ts

@@ -6,16 +6,16 @@ import { ERROR_REASONS } from './const/ErrorReason';
 
 import { DataType, DataTypeMap, DslType } from './types/Common';
 import {
-  CalcDistanceReq,
   DeleteEntitiesReq,
   FlushReq,
   GetFlushStateReq,
   GetQuerySegmentInfoReq,
   InsertReq,
   LoadBalanceReq,
+  ImportReq,
+  ListImportTasksReq,
 } from './types/Data';
 import {
-  CalcDistanceResponse,
   ErrorCode,
   FlushResult,
   GetFlushStateResponse,
@@ -25,6 +25,8 @@ import {
   QueryResults,
   ResStatus,
   SearchResults,
+  ImportResponse,
+  ListImportTasksResponse,
 } from './types/Response';
 import {
   GetMetricsRequest,
@@ -373,13 +375,15 @@ export class Data extends Client {
       'Search',
       {
         ...data,
+        nq: data.nq || data.vectors.length,
         dsl: data.expr || '',
         dsl_type: DslType.BoolExprV1,
         placeholder_group: placeholderGroupBytes,
         search_params: parseToKeyValue(data.search_params),
       },
       data.timeout
     );
+
     const results: any[] = [];
     /**
      *  It will decide the score precision.
@@ -534,7 +538,7 @@ export class Data extends Client {
    *  | partitions_names(optional)   | String[] |       Array of partition names      |
    *  | output_fields                | String[] |       Vector or scalar field to be returned    |
    *  | timeout        | number |        An optional duration of time in millisecond to allow for the RPC. If it is set to undefined, the client keeps waiting until the server responds or error occurs. Default is undefined       |
-   *
+   *  | params | {key: value}[] | An optional key pair json array
    *
    * @return
    *  | Property    |           Description              |
@@ -555,12 +559,27 @@ export class Data extends Client {
    */
   async query(data: QueryReq): Promise<QueryResults> {
     this.checkCollectionName(data);
+
+    let limits: { limit: number } | undefined;
+    let offset: { offset: number } | undefined;
+
+    if (typeof data.limit === 'number') {
+      limits = { limit: data.limit };
+    }
+    if (typeof data.offset === 'number') {
+      offset = { offset: data.offset };
+    }
+
     const promise: QueryRes = await promisify(
       this.client,
       'Query',
-      data,
+      {
+        ...data,
+        query_params: parseToKeyValue({ ...limits, ...offset }),
+      },
       data.timeout
     );
+
     const results: { [x: string]: any }[] = [];
     /**
      * type: DataType
@@ -768,4 +787,98 @@ export class Data extends Client {
     );
     return res;
   }
+
+  /**
+   * Import data from files
+   *
+   * @param data
+   *  | Property                | Type   |           Description              |
+   *  | :---------------------- | :----  | :-------------------------------  |
+   *  | collection_name          | String |      The name of the collection      |
+   *  | files        | string[] |        File path array       |
+   *
+   *
+   * @return
+   *  | Property    |           Description              |
+   *  | :-----------| :-------------------------------  |
+   *  | status      |  { error_code: number,reason:string } |
+   *  | tasks       |  taskId array  |
+   *
+   *
+   * #### Example
+   *
+   * ```
+   *   const res = await dataManager.bulkInsert({
+   *      collection_name: COLLECTION,
+   *      files: [`path-to-data-file.json`]
+   *    });
+   * ```
+   */
+  async bulkInsert(data: ImportReq): Promise<ImportResponse> {
+    if (!data || !data.collection_name) {
+      throw new Error(ERROR_REASONS.COLLECTION_NAME_IS_REQUIRED);
+    }
+
+    if (!data || !data.files) {
+      throw new Error(ERROR_REASONS.IMPORT_FILE_CHECK);
+    }
+    const res = await promisify(
+      this.client,
+      'Import',
+      {
+        ...data,
+        options: data.options || [],
+      },
+      data.timeout
+    );
+    return res;
+  }
+
+  /**
+   * List import tasks
+   *
+   * @param data
+   *  | Property                | Type   |           Description              |
+   *  | :---------------------- | :----  | :-------------------------------  |
+   *  | collection_name          | String |      The name of the collection       |
+   *  | limit        | number |       optional, maximum number of tasks returned, list all tasks if the value is 0       |
+   *
+   *
+   * @return
+   *  | Property    |           Description              |
+   *  | :-----------| :-------------------------------  |
+   *  | status      |  { error_code: number,reason:string } |
+   *  | state | import state |
+   *  | row_count | how many rows to import|
+   *  | id_list| id lists |
+   *  | collection_id | collection to be imported to |
+   *  |
+   *  | tasks       |  taskId array  |
+   *
+   *
+   * #### Example
+   *
+   * ```
+   *   const res = await dataManager.listImportTasks({
+   *      collection_name: COLLECTION
+   *    });
+   * ```
+   */
+  async listImportTasks(
+    data: ListImportTasksReq
+  ): Promise<ListImportTasksResponse> {
+    if (!data || !data.collection_name) {
+      throw new Error(ERROR_REASONS.COLLECTION_NAME_IS_REQUIRED);
+    }
+    const res = await promisify(
+      this.client,
+      'ListImportTasks',
+      {
+        ...data,
+        limit: data.limit || 0,
+      },
+      data.timeout
+    );
+    return res;
+  }
 }

milvus/const/ErrorReason.ts

@@ -45,4 +45,5 @@ export enum ERROR_REASONS {
   TIMESTAMP_PARAM_CHECK = 'type should be string (only contains number) or bigint',
 
   DATE_TYPE_CHECK = 'type should be Date',
+  IMPORT_FILE_CHECK = 'files is required',
 }