From 3fd43cfe621fca3f7e112de128b059ea99a2ed8c Mon Sep 17 00:00:00 2001 From: Serge Klochkov Date: Mon, 19 Aug 2024 13:41:14 +0200 Subject: [PATCH 1/2] Remove old JSON tests --- examples/insert_ephemeral_columns.ts | 39 +-- .../__tests__/integration/data_types.test.ts | 21 +- .../insert_specific_columns.test.ts | 228 +++++------------- 3 files changed, 70 insertions(+), 218 deletions(-) diff --git a/examples/insert_ephemeral_columns.ts b/examples/insert_ephemeral_columns.ts index c621be10..756d12f4 100644 --- a/examples/insert_ephemeral_columns.ts +++ b/examples/insert_ephemeral_columns.ts @@ -1,26 +1,20 @@ import { createClient } from '@clickhouse/client' // or '@clickhouse/client-web' // Ephemeral columns documentation: https://clickhouse.com/docs/en/sql-reference/statements/create/table#ephemeral -// This example is inspired by https://github.com/ClickHouse/clickhouse-js/issues/217 void (async () => { const tableName = 'insert_ephemeral_columns' - const client = createClient({ - clickhouse_settings: { - allow_experimental_object_type: 1, // allows JSON type usage - }, - }) + const client = createClient({}) await client.command({ query: ` CREATE OR REPLACE TABLE ${tableName} ( - event_type LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'type'), - repo_name LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'repo', 'name'), - message JSON DEFAULT message_raw, - message_raw String EPHEMERAL + id UInt64, + message String DEFAULT message_raw, + message_raw String EPHEMERAL ) ENGINE MergeTree() - ORDER BY (event_type, repo_name) + ORDER BY (id) `, }) @@ -28,20 +22,12 @@ void (async () => { table: tableName, values: [ { - message_raw: { - type: 'MyEventType', - repo: { - name: 'foo', - }, - }, + id: '42', + message_raw: 'foo', }, { - message_raw: { - type: 'SomeOtherType', - repo: { - name: 'bar', - }, - }, + id: '144', + message_raw: 'bar', }, ], format: 'JSONEachRow', @@ -51,10 +37,11 @@ void (async () => { }) const rows = await client.query({ - query: `SELECT * FROM ${tableName}`, - format: 'JSONCompactEachRowWithNames', + query: `SELECT * + FROM ${tableName}`, + format: 'JSONEachRow', }) - console.info(await rows.text()) + console.info(await rows.json()) await client.close() })() diff --git a/packages/client-common/__tests__/integration/data_types.test.ts b/packages/client-common/__tests__/integration/data_types.test.ts index 020c21ad..cbc1760d 100644 --- a/packages/client-common/__tests__/integration/data_types.test.ts +++ b/packages/client-common/__tests__/integration/data_types.test.ts @@ -4,7 +4,7 @@ import type { } from '@clickhouse/client-common' import { randomUUID } from '@test/utils/guid' import { createTableWithFields } from '../fixtures/table_with_fields' -import { createTestClient, getRandomInt, TestEnv, whenOnEnv } from '../utils' +import { createTestClient, getRandomInt } from '../utils' describe('data types', () => { let client: ClickHouseClient @@ -504,25 +504,6 @@ describe('data types', () => { await insertAndAssert(table, values) }) - // JSON cannot be used on a "modern" Cloud instance - whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster, TestEnv.Cloud).it( - 'should work with JSON', - async () => { - const values = [ - { - o: { a: 1, b: { c: 2, d: [1, 2, 3] } }, - }, - { - o: { a: 2, b: { c: 3, d: [4, 5, 6] } }, - }, - ] - const table = await createTableWithFields(client, 'o JSON', { - allow_experimental_object_type: 1, - }) - await insertAndAssert(table, values) - }, - ) - describe('Nested', () => { it('should work by default', async () => { const values = [ diff --git a/packages/client-common/__tests__/integration/insert_specific_columns.test.ts b/packages/client-common/__tests__/integration/insert_specific_columns.test.ts index 7a37435b..f2d3931e 100644 --- a/packages/client-common/__tests__/integration/insert_specific_columns.test.ts +++ b/packages/client-common/__tests__/integration/insert_specific_columns.test.ts @@ -1,97 +1,44 @@ import { type ClickHouseClient } from '@clickhouse/client-common' import { createTableWithFields } from '@test/fixtures/table_with_fields' -import { createTestClient, TestEnv, whenOnEnv } from '../utils' +import { createTestClient, guid } from '../utils' +import { createSimpleTable } from '../fixtures/simple_table' -// allow_experimental_object_type is not permitted on a modern Cloud instance -whenOnEnv( - TestEnv.LocalSingleNode, - TestEnv.LocalCluster, - TestEnv.Cloud, -).describe('Insert with specific columns', () => { +describe('Insert with specific columns', () => { let client: ClickHouseClient let table: string beforeEach(async () => { - client = createTestClient({ - clickhouse_settings: { - allow_experimental_object_type: 1, - }, - }) + client = createTestClient() }) afterEach(async () => { await client.close() }) - // Inspired by https://github.com/ClickHouse/clickhouse-js/issues/217 - // Specifying a particular insert column is especially useful with ephemeral types describe('list of columns', () => { beforeEach(async () => { - // `message_raw` will be used as a source for default values here - table = await createTableWithFields( - client, - ` - event_type LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'type'), - repo_name LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'repo', 'name'), - message JSON DEFAULT message_raw, - message_raw String EPHEMERAL - `, // `id UInt32` will be added as well - ) + table = `insert_specific_columns_${guid()}` + await createSimpleTable(client, table) }) + const row = { + id: '42', + name: 'foo', + sku: [144], + } + it('should work with a single column', async () => { await client.insert({ table, - values: [ - { - message_raw: { - type: 'MyEventType', - repo: { - name: 'foo', - }, - }, - }, - { - message_raw: { - type: 'SomeOtherType', - repo: { - name: 'bar', - }, - }, - }, - ], + values: [{ id: 42 }], format: 'JSONEachRow', - columns: ['message_raw'], + columns: ['id'], }) - - const result = await client - .query({ - query: `SELECT * FROM ${table} ORDER BY repo_name DESC`, - format: 'JSONEachRow', - }) - .then((r) => r.json()) - + const result = await select() expect(result).toEqual([ { - id: 0, // defaults for everything are taken from `message_raw` - event_type: 'MyEventType', - repo_name: 'foo', - message: { - type: 'MyEventType', - repo: { - name: 'foo', - }, - }, - }, - { - id: 0, - event_type: 'SomeOtherType', - repo_name: 'bar', - message: { - type: 'SomeOtherType', - repo: { - name: 'bar', - }, - }, + id: '42', + name: '', + sku: [], }, ]) }) @@ -101,119 +48,41 @@ whenOnEnv( table, values: [ { - id: 42, - message_raw: { - type: 'MyEventType', - repo: { - name: 'foo', - }, - }, - }, - { - id: 144, - message_raw: { - type: 'SomeOtherType', - repo: { - name: 'bar', - }, - }, + id: '42', + name: 'foo', }, ], format: 'JSONEachRow', - columns: ['id', 'message_raw'], + columns: ['id', 'name'], }) - - const result = await client - .query({ - query: `SELECT * FROM ${table} ORDER BY id`, - format: 'JSONEachRow', - }) - .then((r) => r.json()) - + const result = await select() expect(result).toEqual([ { - id: 42, // all defaults except `id` are taken from `message_raw` - event_type: 'MyEventType', - repo_name: 'foo', - message: { - type: 'MyEventType', - repo: { - name: 'foo', - }, - }, - }, - { - id: 144, - event_type: 'SomeOtherType', - repo_name: 'bar', - message: { - type: 'SomeOtherType', - repo: { - name: 'bar', - }, - }, + id: '42', + name: 'foo', + sku: [], }, ]) }) - // In this case, `message_raw` will be ignored, as expected it('should work when all the columns are specified', async () => { - const value1 = { - id: 42, - event_type: 'MyEventType', - repo_name: 'foo', - message: { foo: 'bar' }, - } - const value2 = { - id: 42, - event_type: 'MyEventType', - repo_name: 'foo', - message: { foo: 'bar' }, - } await client.insert({ table, - values: [ - { ...value1, message_raw: '{ "i": 42 }' }, - { ...value2, message_raw: '{ "j": 255 }' }, - ], + values: [row], format: 'JSONEachRow', - columns: ['id', 'event_type', 'repo_name', 'message', 'message_raw'], + columns: ['id', 'name', 'sku'], }) - - const result = await client - .query({ - query: `SELECT * FROM ${table} ORDER BY id`, - format: 'JSONEachRow', - }) - .then((r) => r.json()) - - expect(result).toEqual([value1, value2]) + const result = await select() + expect(result).toEqual([row]) }) it('should fail when an unknown column is specified', async () => { await expectAsync( client.insert({ table, - values: [ - { - message_raw: { - type: 'MyEventType', - repo: { - name: 'foo', - }, - }, - }, - { - message_raw: { - type: 'SomeOtherType', - repo: { - name: 'bar', - }, - }, - }, - ], + values: [row], format: 'JSONEachRow', - columns: ['foobar', 'message_raw'], + columns: ['foobar', 'id'], }), ).toBeRejectedWith( jasmine.objectContaining({ @@ -223,8 +92,6 @@ whenOnEnv( }) }) - // This is impossible to test with ephemeral columns (need to send at least `message_raw`), - // so for this corner case the tests are simplified. Essentially, just a fallback to the "normal" insert behavior. describe('list of columns corner cases', () => { beforeEach(async () => { table = await createTableWithFields( @@ -249,7 +116,9 @@ whenOnEnv( const result = await client .query({ - query: `SELECT * FROM ${table} ORDER BY id`, + query: `SELECT * + FROM ${table} + ORDER BY id`, format: 'JSONEachRow', }) .then((r) => r.json()) @@ -258,8 +127,6 @@ whenOnEnv( }) }) - // TODO: For some reason, ephemeral columns don't work well with EXCEPT (even from the CLI) - to be investigated. - // Thus, the tests for this case are simplified. describe('list of excluded columns', () => { beforeEach(async () => { table = await createTableWithFields( @@ -283,7 +150,9 @@ whenOnEnv( const result = await client .query({ - query: `SELECT * FROM ${table} ORDER BY id`, + query: `SELECT * + FROM ${table} + ORDER BY id`, format: 'JSONEachRow', }) .then((r) => r.json()) @@ -306,7 +175,9 @@ whenOnEnv( const result = await client .query({ - query: `SELECT * FROM ${table} ORDER BY s DESC`, + query: `SELECT * + FROM ${table} + ORDER BY s DESC`, format: 'JSONEachRow', }) .then((r) => r.json()) @@ -334,7 +205,8 @@ whenOnEnv( const result = await client .query({ - query: `SELECT * FROM ${table}`, + query: `SELECT * + FROM ${table}`, format: 'JSONEachRow', }) .then((r) => r.json()) @@ -354,7 +226,8 @@ whenOnEnv( const result = await client .query({ - query: `SELECT * FROM ${table}`, + query: `SELECT * + FROM ${table}`, format: 'JSONEachRow', }) .then((r) => r.json()) @@ -381,7 +254,8 @@ whenOnEnv( const result = await client .query({ - query: `SELECT * FROM ${table}`, + query: `SELECT * + FROM ${table}`, format: 'JSONEachRow', }) .then((r) => r.json()) @@ -389,4 +263,14 @@ whenOnEnv( expect(result).toEqual(values) }) }) + + async function select() { + const rs = await client.query({ + query: `SELECT * + FROM ${table} + ORDER BY id ASC`, + format: 'JSONEachRow', + }) + return rs.json<{ id: string; name: string; sku: number[] }>() + } }) From 80b40b5f885f06c29794e30e0bc5cce5bbb242ca Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Mon, 19 Aug 2024 14:42:14 +0200 Subject: [PATCH 2/2] Add the Object(json) test back --- examples/insert_ephemeral_columns.ts | 12 +++++------ .../__tests__/integration/data_types.test.ts | 21 ++++++++++++++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/examples/insert_ephemeral_columns.ts b/examples/insert_ephemeral_columns.ts index 756d12f4..f0b257c0 100644 --- a/examples/insert_ephemeral_columns.ts +++ b/examples/insert_ephemeral_columns.ts @@ -9,9 +9,9 @@ void (async () => { query: ` CREATE OR REPLACE TABLE ${tableName} ( - id UInt64, - message String DEFAULT message_raw, - message_raw String EPHEMERAL + id UInt64, + message String DEFAULT message_default, + message_default String EPHEMERAL ) ENGINE MergeTree() ORDER BY (id) @@ -23,17 +23,17 @@ void (async () => { values: [ { id: '42', - message_raw: 'foo', + message_default: 'foo', }, { id: '144', - message_raw: 'bar', + message_default: 'bar', }, ], format: 'JSONEachRow', // The name of the ephemeral column has to be specified here // to trigger the default values logic for the rest of the columns - columns: ['message_raw'], + columns: ['id', 'message_default'], }) const rows = await client.query({ diff --git a/packages/client-common/__tests__/integration/data_types.test.ts b/packages/client-common/__tests__/integration/data_types.test.ts index cbc1760d..0c6f0c55 100644 --- a/packages/client-common/__tests__/integration/data_types.test.ts +++ b/packages/client-common/__tests__/integration/data_types.test.ts @@ -4,7 +4,7 @@ import type { } from '@clickhouse/client-common' import { randomUUID } from '@test/utils/guid' import { createTableWithFields } from '../fixtures/table_with_fields' -import { createTestClient, getRandomInt } from '../utils' +import { createTestClient, getRandomInt, TestEnv, whenOnEnv } from '../utils' describe('data types', () => { let client: ClickHouseClient @@ -504,6 +504,25 @@ describe('data types', () => { await insertAndAssert(table, values) }) + // JSON cannot be used on a "modern" Cloud instance + whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster, TestEnv.Cloud).it( + 'should work with JSON', + async () => { + const values = [ + { + o: { a: 1, b: { c: 2, d: [1, 2, 3] } }, + }, + { + o: { a: 2, b: { c: 3, d: [4, 5, 6] } }, + }, + ] + const table = await createTableWithFields(client, `o Object('json')`, { + allow_experimental_object_type: 1, + }) + await insertAndAssert(table, values) + }, + ) + describe('Nested', () => { it('should work by default', async () => { const values = [