Merge remote-tracking branch 'origin' into default-http-settings

ClickHouse · Feb 29, 2024 · bdfd32f · bdfd32f
2 parents f0e49d9 + bd5bd29
commit bdfd32f
Show file tree

Hide file tree

Showing 5 changed files with 276 additions and 44 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,7 +2,7 @@
 
 ### Breaking changes
 
-- Client will enable [send_progress_in_http_headers](https://clickhouse.com/docs/en/operations/settings/settings#send_progress_in_http_headers) and set http_headers_progress_interval_ms to `290000` by default. These settings in combination allow to avoid LB timeout issues in case of long-running queries without data coming in or out, such as `INSERT FROM SELECT` and similar ones, as the connection could be marked as idle by the LB and closed abruptly. Currently, 290s is chosen as a safe value, since AWS LB timeout is 350s by default and some other LBs might have 300s. It can be overridden when creating a client instance if your LB timeout value is even lower than that. See also: [AWS LB documentation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout).
+- Client will enable [send_progress_in_http_headers](https://clickhouse.com/docs/en/operations/settings/settings#send_progress_in_http_headers) and set `http_headers_progress_interval_ms` to `29000` (29 seconds) by default. These settings in combination allow to avoid LB timeout issues in case of long-running queries without data coming in or out, such as `INSERT FROM SELECT` and similar ones, as the connection could be marked as idle by the LB and closed abruptly. Currently, 29s is chosen as a safe value, since most LBs will have at least 30s of idle timeout. It can be overridden when creating a client instance if your LB timeout value is even lower than that by manually changing the `send_progress_in_http_headers` value.
 
   NB: these settings will be enabled only if the client instance was created without setting `readonly` flag (see below).
 
@@ -32,6 +32,16 @@ NB: this is not necessary if a user has `READONLY = 2` mode as it allows to modi
 
 See also: [readonly documentation](https://clickhouse.com/docs/en/operations/settings/permissions-for-queries#readonly).
 
+## 0.2.10 (Common, Node.js, Web)
+
+### New features
+
+- If `InsertParams.values` is an empty array, no request is sent to the server and `ClickHouseClient.insert` short-circuits itself. In this scenario, the newly added `InsertResult.executed` flag will be `false`, and `InsertResult.query_id` will be an empty string.
+
+### Bug fixes
+
+- Client no longer produces `Code: 354. inflate failed: buffer error` exception if request compression is enabled and `InsertParams.values` is an empty array (see above).
+
 ## 0.2.9 (Common, Node.js, Web)
 
 ### New features

diff --git a/packages/client-common/__tests__/integration/data_types.test.ts b/packages/client-common/__tests__/integration/data_types.test.ts
@@ -519,25 +519,133 @@ describe('data types', () => {
     await insertAndAssert(table, values)
   })
 
-  /** @see https://github.com/ClickHouse/clickhouse-js/issues/89 */
-  xit('should work with nested', async () => {
+  it('should work with nested', async () => {
+    const values = [
+      {
+        id: 1,
+        'n.id': [42],
+        'n.name': ['foo'],
+        'n.createdAt': ['2001-04-23 00:00:00'],
+        'n.roles': [['User']],
+      },
+      {
+        id: 2,
+        'n.id': [43],
+        'n.name': ['bar'],
+        'n.createdAt': ['2000-01-12 00:00:00'],
+        'n.roles': [['Admin']],
+      },
+    ]
+    const table = await createTableWithFields(
+      client,
+      'n Nested(id UInt32, name String, createdAt DateTime, ' +
+        `roles Array(Enum('User', 'Admin')))`
+    )
+    await client.insert({
+      table,
+      values,
+      format: 'JSONEachRow',
+    })
+    const result = await client
+      .query({
+        query: `SELECT n.id, n.name, n.createdAt, n.roles FROM ${table} ORDER BY id ASC`,
+        format: 'JSONEachRow',
+      })
+      .then((r) => r.json())
+    expect(result).toEqual([
+      {
+        'n.id': [42],
+        'n.name': ['foo'],
+        'n.createdAt': ['2001-04-23 00:00:00'],
+        'n.roles': [['User']],
+      },
+      {
+        'n.id': [43],
+        'n.name': ['bar'],
+        'n.createdAt': ['2000-01-12 00:00:00'],
+        'n.roles': [['Admin']],
+      },
+    ])
+  })
+
+  it('should work with nested (flatten_nested = 0)', async () => {
+    const values = [
+      {
+        id: 1,
+        n: [
+          {
+            id: 42,
+            name: 'foo',
+            createdAt: '2001-04-23 00:00:00',
+            roles: ['User'],
+          },
+        ],
+      },
+      {
+        id: 2,
+        n: [
+          {
+            id: 43,
+            name: 'bar',
+            createdAt: '2000-01-12 00:00:00',
+            roles: ['Admin'],
+          },
+        ],
+      },
+    ]
+    const table = await createTableWithFields(
+      client,
+      'n Nested(id UInt32, name String, createdAt DateTime, ' +
+        `roles Array(Enum('User', 'Admin')))`,
+      {
+        flatten_nested: 0,
+      }
+    )
+    await client.insert({
+      table,
+      values,
+      format: 'JSONEachRow',
+    })
+    const result = await client
+      .query({
+        query: `SELECT n.id, n.name, n.createdAt, n.roles FROM ${table} ORDER BY id ASC`,
+        format: 'JSONEachRow',
+      })
+      .then((r) => r.json())
+    expect(result).toEqual([
+      {
+        'n.id': [42],
+        'n.name': ['foo'],
+        'n.createdAt': ['2001-04-23 00:00:00'],
+        'n.roles': [['User']],
+      },
+      {
+        'n.id': [43],
+        'n.name': ['bar'],
+        'n.createdAt': ['2000-01-12 00:00:00'],
+        'n.roles': [['Admin']],
+      },
+    ])
+  })
+
+  it('should work with nested (input_format_import_nested_json = 1)', async () => {
     const values = [
       {
         id: 1,
         n: {
-          id: 42,
-          name: 'foo',
-          createdAt: '2001-04-23',
-          roles: ['User'],
+          id: [42],
+          name: ['foo'],
+          createdAt: ['2001-04-23 00:00:00'],
+          roles: [['User']],
         },
       },
       {
         id: 2,
         n: {
-          id: 43,
-          name: 'bar',
-          createdAt: '2000-01-12',
-          roles: ['Admin'],
+          id: [43],
+          name: ['bar'],
+          createdAt: ['2000-01-12 00:00:00'],
+          roles: [['Admin']],
         },
       },
     ]
@@ -549,6 +657,72 @@ describe('data types', () => {
     await client.insert({
       table,
       values,
+      clickhouse_settings: {
+        input_format_import_nested_json: 1,
+      },
+      format: 'JSONEachRow',
+    })
+    const result = await client
+      .query({
+        query: `SELECT n.id, n.name, n.createdAt, n.roles FROM ${table} ORDER BY id ASC`,
+        format: 'JSONEachRow',
+      })
+      .then((r) => r.json())
+    expect(result).toEqual([
+      {
+        'n.id': [42],
+        'n.name': ['foo'],
+        'n.createdAt': ['2001-04-23 00:00:00'],
+        'n.roles': [['User']],
+      },
+      {
+        'n.id': [43],
+        'n.name': ['bar'],
+        'n.createdAt': ['2000-01-12 00:00:00'],
+        'n.roles': [['Admin']],
+      },
+    ])
+  })
+
+  it('should work with nested with (flatten_nested = 0 and input_format_import_nested_json = 1)', async () => {
+    const values = [
+      {
+        id: 1,
+        n: [
+          {
+            id: 42,
+            name: 'foo',
+            createdAt: '2001-04-23 00:00:00',
+            roles: ['User'],
+          },
+        ],
+      },
+      {
+        id: 2,
+        n: [
+          {
+            id: 43,
+            name: 'bar',
+            createdAt: '2000-01-12 00:00:00',
+            roles: ['Admin'],
+          },
+        ],
+      },
+    ]
+    const table = await createTableWithFields(
+      client,
+      'n Nested(id UInt32, name String, createdAt DateTime, ' +
+        `roles Array(Enum('User', 'Admin')))`,
+      {
+        flatten_nested: 0,
+      }
+    )
+    await client.insert({
+      table,
+      values,
+      clickhouse_settings: {
+        input_format_import_nested_json: 1,
+      },
       format: 'JSONEachRow',
     })
     const result = await client
@@ -559,16 +733,16 @@ describe('data types', () => {
       .then((r) => r.json())
     expect(result).toEqual([
       {
-        'n.id': 42,
-        'n.name': 'foo',
-        'n.createdAt': '2001-04-23',
-        'n.roles': ['User'],
+        'n.id': [42],
+        'n.name': ['foo'],
+        'n.createdAt': ['2001-04-23 00:00:00'],
+        'n.roles': [['User']],
       },
       {
-        'n.id': 43,
-        'n.name': 'bar',
-        'n.createdAt': '2000-01-12',
-        'n.roles': ['Admin'],
+        'n.id': [43],
+        'n.name': ['bar'],
+        'n.createdAt': ['2000-01-12 00:00:00'],
+        'n.roles': [['Admin']],
       },
     ])
   })

diff --git a/packages/client-common/__tests__/integration/insert.test.ts b/packages/client-common/__tests__/integration/insert.test.ts
@@ -17,7 +17,7 @@ describe('insert', () => {
   })
 
   it('inserts values using JSON format', async () => {
-    const { query_id } = await client.insert({
+    const result = await client.insert({
       table: tableName,
       values: {
         meta: [
@@ -39,12 +39,13 @@ describe('insert', () => {
       format: 'JSON',
     })
     await assertJsonValues(client, tableName)
-    expect(validateUUID(query_id)).toBeTruthy()
+    expect(validateUUID(result.query_id)).toBeTruthy()
+    expect(result.executed).toBeTruthy()
   })
 
   it('should use provide query_id', async () => {
     const query_id = guid()
-    const { query_id: q_id } = await client.insert({
+    const result = await client.insert({
       table: tableName,
       query_id,
       values: {
@@ -67,7 +68,8 @@ describe('insert', () => {
       format: 'JSON',
     })
     await assertJsonValues(client, tableName)
-    expect(query_id).toEqual(q_id)
+    expect(result.query_id).toEqual(query_id)
+    expect(result.executed).toBeTruthy()
   })
 
   it('inserts values using JSONObjectEachRow format', async () => {

diff --git a/packages/client-common/src/client.ts b/packages/client-common/src/client.ts
@@ -5,7 +5,6 @@ import type {
   Connection,
   ConnectionParams,
   ConnExecResult,
-  ConnInsertResult,
   Logger,
   WithClickHouseSummary,
 } from '@clickhouse/client-common'
@@ -156,7 +155,21 @@ export interface ExecParams extends BaseQueryParams {
 export type CommandParams = ExecParams
 export type CommandResult = { query_id: string } & WithClickHouseSummary
 
-export type InsertResult = ConnInsertResult
+export type InsertResult = {
+  /**
+   * Indicates whether the INSERT statement was executed on the server.
+   * Will be `false` if there was no data to insert.
+   * For example: if {@link InsertParams.values} was an empty array,
+   * the client does not any requests to the server, and {@link executed} is false.
+   */
+  executed: boolean
+  /**
+   * Empty string if {@link executed} is false.
+   * Otherwise, either {@link InsertParams.query_id} if it was set, or the id that was generated by the client.
+   */
+  query_id: string
+} & WithClickHouseSummary
+
 export type ExecResult<Stream> = ConnExecResult<Stream>
 export type PingResult = ConnPingResult
 
@@ -277,15 +290,20 @@ export class ClickHouseClient<Stream = unknown> {
    * consider using {@link ClickHouseClient.command}, passing the entire raw query there (including FORMAT clause).
    */
   async insert<T>(params: InsertParams<Stream, T>): Promise<InsertResult> {
+    if (Array.isArray(params.values) && params.values.length === 0) {
+      return { executed: false, query_id: '' }
+    }
+
     const format = params.format || 'JSONCompactEachRow'
     this.valuesEncoder.validateInsertValues(params.values, format)
 
     const query = getInsertQuery(params, format)
-    return await this.connection.insert({
+    const result = await this.connection.insert({
       query,
       values: this.valuesEncoder.encodeValues(params.values, format),
       ...this.getQueryParams(params),
     })
+    return { ...result, executed: true }
   }
 
   /**