From 9e221ea891c766810927078bb808c460b873d800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 27 Sep 2024 17:24:28 +0200 Subject: [PATCH] SNOW-1374896 unify structured types string representation (#1882) Co-authored-by: sfc-gh-astachowski Build string representations of Snowflake structured types recursively to reuse existing converters designed for specific logical types (e.g. timestamps/binary) https://snowflakecomputing.atlassian.net/browse/SNOW-1374896 --- .../net/snowflake/client/core/ResultUtil.java | 2 +- .../client/core/arrow/ArrayConverter.java | 24 +++- .../core/arrow/ArrowVectorConverterUtil.java | 22 ++- .../core/arrow/BitToBooleanConverter.java | 2 +- .../client/core/arrow/MapConverter.java | 29 +++- .../client/core/arrow/StructConverter.java | 18 ++- ...ArrowArrayStringRepresentationBuilder.java | 19 +++ ...rrowObjectStringRepresentationBuilder.java | 21 +++ .../ArrowStringRepresentationBuilderBase.java | 53 +++++++ .../java/net/snowflake/client/TestUtil.java | 10 ++ .../core/arrow/BitToBooleanConverterTest.java | 2 +- .../client/core/json/StringConverterTest.java | 8 +- .../client/jdbc/ResultSetJsonVsArrowIT.java | 4 +- .../client/jdbc/ResultSetLatestIT.java | 4 +- .../ResultSetStructuredTypesLatestIT.java | 94 +++--------- ...ypesGetStringArrowJsonCompatibilityIT.java | 135 ++++++++++++++++++ .../StructuredTypesGetStringBaseIT.java | 68 +++++++++ .../sqldata/AllTypesClass.java | 37 +++++ 18 files changed, 459 insertions(+), 93 deletions(-) create mode 100644 src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowArrayStringRepresentationBuilder.java create mode 100644 src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowObjectStringRepresentationBuilder.java create mode 100644 src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java create mode 100644 src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java create mode 100644 src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java diff --git a/src/main/java/net/snowflake/client/core/ResultUtil.java b/src/main/java/net/snowflake/client/core/ResultUtil.java index b894f4259..6c00f513e 100644 --- a/src/main/java/net/snowflake/client/core/ResultUtil.java +++ b/src/main/java/net/snowflake/client/core/ResultUtil.java @@ -251,7 +251,7 @@ public static String getSFTimeAsString( * @return boolean in string */ public static String getBooleanAsString(boolean bool) { - return bool ? "TRUE" : "FALSE"; + return bool ? "true" : "false"; } /** diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index 08ce23eec..48b8fa083 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -2,7 +2,10 @@ import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; +import net.snowflake.client.core.arrow.tostringhelpers.ArrowArrayStringRepresentationBuilder; +import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.SnowflakeType; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.complex.ListVector; public class ArrayConverter extends AbstractArrowVectorConverter { @@ -21,6 +24,25 @@ public Object toObject(int index) throws SFException { @Override public String toString(int index) throws SFException { - return vector.getObject(index).toString(); + FieldVector vectorUnpacked = vector.getChildrenFromFields().get(0); + SnowflakeType logicalType = + ArrowVectorConverterUtil.getSnowflakeTypeFromFieldMetadata(vectorUnpacked.getField()); + + ArrowArrayStringRepresentationBuilder builder = + new ArrowArrayStringRepresentationBuilder(logicalType); + + final ArrowVectorConverter converter; + + try { + converter = ArrowVectorConverterUtil.initConverter(vectorUnpacked, context, columnIndex); + } catch (SnowflakeSQLException e) { + return vector.getObject(index).toString(); + } + + for (int i = vector.getElementStartIndex(index); i < vector.getElementEndIndex(index); i++) { + builder.appendValue(converter.toString(i)); + } + + return builder.toString(); } } diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrowVectorConverterUtil.java b/src/main/java/net/snowflake/client/core/arrow/ArrowVectorConverterUtil.java index 1aa84db8f..0231ebd51 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrowVectorConverterUtil.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrowVectorConverterUtil.java @@ -9,17 +9,28 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.SnowflakeType; import net.snowflake.common.core.SqlState; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.Field; @SnowflakeJdbcInternalApi public final class ArrowVectorConverterUtil { private ArrowVectorConverterUtil() {} + public static SnowflakeType getSnowflakeTypeFromFieldMetadata(Field field) { + Map customMeta = field.getMetadata(); + if (customMeta != null && customMeta.containsKey("logicalType")) { + return SnowflakeType.valueOf(customMeta.get("logicalType")); + } + + return null; + } + /** * Given an arrow vector (a single column in a single record batch), return an arrow vector * converter. Note, converter is built on top of arrow vector, so that arrow data can be converted @@ -51,12 +62,11 @@ public static ArrowVectorConverter initConverter( Types.MinorType type = Types.getMinorTypeForArrowType(vector.getField().getType()); // each column's metadata - Map customMeta = vector.getField().getMetadata(); + SnowflakeType st = getSnowflakeTypeFromFieldMetadata(vector.getField()); if (type == Types.MinorType.DECIMAL) { // Note: Decimal vector is different from others return new DecimalToScaledFixedConverter(vector, idx, context); - } else if (!customMeta.isEmpty()) { - SnowflakeType st = SnowflakeType.valueOf(customMeta.get("logicalType")); + } else if (st != null) { switch (st) { case ANY: case CHAR: @@ -216,4 +226,10 @@ public static ArrowVectorConverter initConverter( "Unexpected Arrow Field for ", type.toString()); } + + public static ArrowVectorConverter initConverter( + FieldVector vector, DataConversionContext context, int columnIndex) + throws SnowflakeSQLException { + return initConverter(vector, context, context.getSession(), columnIndex); + } } diff --git a/src/main/java/net/snowflake/client/core/arrow/BitToBooleanConverter.java b/src/main/java/net/snowflake/client/core/arrow/BitToBooleanConverter.java index 2f5a8cf83..640ff68ca 100644 --- a/src/main/java/net/snowflake/client/core/arrow/BitToBooleanConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/BitToBooleanConverter.java @@ -58,7 +58,7 @@ public Object toObject(int index) { @Override public String toString(int index) { - return isNull(index) ? null : toBoolean(index) ? "TRUE" : "FALSE"; + return isNull(index) ? null : toBoolean(index) ? "true" : "false"; } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index 433792294..8ef1cdccf 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -4,7 +4,10 @@ import java.util.stream.Collectors; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; +import net.snowflake.client.core.arrow.tostringhelpers.ArrowObjectStringRepresentationBuilder; +import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.SnowflakeType; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.util.JsonStringHashMap; @@ -28,6 +31,30 @@ public Object toObject(int index) throws SFException { @Override public String toString(int index) throws SFException { - return vector.getObject(index).toString(); + ArrowObjectStringRepresentationBuilder builder = new ArrowObjectStringRepresentationBuilder(); + + FieldVector vectorUnpacked = vector.getChildrenFromFields().get(0); + + FieldVector keys = vectorUnpacked.getChildrenFromFields().get(0); + FieldVector values = vectorUnpacked.getChildrenFromFields().get(1); + final ArrowVectorConverter keyConverter; + final ArrowVectorConverter valueConverter; + + SnowflakeType valueLogicalType = + ArrowVectorConverterUtil.getSnowflakeTypeFromFieldMetadata(values.getField()); + + try { + keyConverter = ArrowVectorConverterUtil.initConverter(keys, context, columnIndex); + valueConverter = ArrowVectorConverterUtil.initConverter(values, context, columnIndex); + } catch (SnowflakeSQLException e) { + return vector.getObject(index).toString(); + } + + for (int i = vector.getElementStartIndex(index); i < vector.getElementEndIndex(index); i++) { + builder.appendKeyValue( + keyConverter.toString(i), valueConverter.toString(i), valueLogicalType); + } + + return builder.toString(); } } diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index 84ccd7c0f..4c0516c51 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -3,7 +3,10 @@ import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; import net.snowflake.client.core.SnowflakeJdbcInternalApi; +import net.snowflake.client.core.arrow.tostringhelpers.ArrowObjectStringRepresentationBuilder; +import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.SnowflakeType; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.complex.StructVector; @SnowflakeJdbcInternalApi @@ -23,6 +26,19 @@ public Object toObject(int index) throws SFException { @Override public String toString(int index) throws SFException { - return structVector.getObject(index).toString(); + ArrowObjectStringRepresentationBuilder builder = new ArrowObjectStringRepresentationBuilder(); + for (String childName : structVector.getChildFieldNames()) { + FieldVector fieldVector = structVector.getChild(childName); + SnowflakeType logicalType = + ArrowVectorConverterUtil.getSnowflakeTypeFromFieldMetadata(fieldVector.getField()); + try { + ArrowVectorConverter converter = + ArrowVectorConverterUtil.initConverter(fieldVector, context, columnIndex); + builder.appendKeyValue(childName, converter.toString(index), logicalType); + } catch (SnowflakeSQLException e) { + return structVector.getObject(index).toString(); + } + } + return builder.toString(); } } diff --git a/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowArrayStringRepresentationBuilder.java b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowArrayStringRepresentationBuilder.java new file mode 100644 index 000000000..7ee6a07aa --- /dev/null +++ b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowArrayStringRepresentationBuilder.java @@ -0,0 +1,19 @@ +package net.snowflake.client.core.arrow.tostringhelpers; + +import net.snowflake.client.core.SnowflakeJdbcInternalApi; +import net.snowflake.client.jdbc.SnowflakeType; + +@SnowflakeJdbcInternalApi +public class ArrowArrayStringRepresentationBuilder extends ArrowStringRepresentationBuilderBase { + + private final SnowflakeType valueType; + + public ArrowArrayStringRepresentationBuilder(SnowflakeType valueType) { + super(",", "[", "]"); + this.valueType = valueType; + } + + public ArrowStringRepresentationBuilderBase appendValue(String value) { + return add(quoteIfNeeded(value, valueType)); + } +} diff --git a/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowObjectStringRepresentationBuilder.java b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowObjectStringRepresentationBuilder.java new file mode 100644 index 000000000..53513836b --- /dev/null +++ b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowObjectStringRepresentationBuilder.java @@ -0,0 +1,21 @@ +package net.snowflake.client.core.arrow.tostringhelpers; + +import java.util.StringJoiner; +import net.snowflake.client.core.SnowflakeJdbcInternalApi; +import net.snowflake.client.jdbc.SnowflakeType; + +@SnowflakeJdbcInternalApi +public class ArrowObjectStringRepresentationBuilder extends ArrowStringRepresentationBuilderBase { + + public ArrowObjectStringRepresentationBuilder() { + super(",", "{", "}"); + } + + public ArrowStringRepresentationBuilderBase appendKeyValue( + String key, String value, SnowflakeType valueType) { + StringJoiner joiner = new StringJoiner(": "); + joiner.add('"' + key + '"'); + joiner.add(quoteIfNeeded(value, valueType)); + return add(joiner.toString()); + } +} diff --git a/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java new file mode 100644 index 000000000..5b83f4497 --- /dev/null +++ b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java @@ -0,0 +1,53 @@ +package net.snowflake.client.core.arrow.tostringhelpers; + +import java.util.HashSet; +import java.util.Set; +import java.util.StringJoiner; +import net.snowflake.client.core.SnowflakeJdbcInternalApi; +import net.snowflake.client.jdbc.SnowflakeType; + +@SnowflakeJdbcInternalApi +public abstract class ArrowStringRepresentationBuilderBase { + private final StringJoiner joiner; + private static final Set quotableTypes; + + static { + quotableTypes = new HashSet<>(); + quotableTypes.add(SnowflakeType.ANY); + quotableTypes.add(SnowflakeType.CHAR); + quotableTypes.add(SnowflakeType.TEXT); + quotableTypes.add(SnowflakeType.VARIANT); + quotableTypes.add(SnowflakeType.BINARY); + quotableTypes.add(SnowflakeType.DATE); + quotableTypes.add(SnowflakeType.TIME); + quotableTypes.add(SnowflakeType.TIMESTAMP_LTZ); + quotableTypes.add(SnowflakeType.TIMESTAMP_NTZ); + quotableTypes.add(SnowflakeType.TIMESTAMP_TZ); + } + + public ArrowStringRepresentationBuilderBase(String delimiter, String prefix, String suffix) { + joiner = new StringJoiner(delimiter, prefix, suffix); + } + + protected ArrowStringRepresentationBuilderBase add(String string) { + joiner.add(string); + return this; + } + + private boolean shouldQuoteValue(SnowflakeType type) { + return quotableTypes.contains(type); + } + + protected String quoteIfNeeded(String string, SnowflakeType type) { + if (shouldQuoteValue(type)) { + return '"' + string + '"'; + } + + return string; + } + + @Override + public String toString() { + return joiner.toString(); + } +} diff --git a/src/test/java/net/snowflake/client/TestUtil.java b/src/test/java/net/snowflake/client/TestUtil.java index 76487bcb4..ba73dbb01 100644 --- a/src/test/java/net/snowflake/client/TestUtil.java +++ b/src/test/java/net/snowflake/client/TestUtil.java @@ -144,4 +144,14 @@ public static void expectSnowflakeLoggedFeatureNotSupportedException(MethodRaise assertEquals(ex.getClass().getSimpleName(), "SnowflakeLoggedFeatureNotSupportedException"); } } + + /** + * Compares two string values both values are cleaned of whitespaces + * + * @param expected expected value + * @param actual actual value + */ + public static void assertEqualsIgnoringWhitespace(String expected, String actual) { + assertEquals(expected.replaceAll("\\s+", ""), actual.replaceAll("\\s+", "")); + } } diff --git a/src/test/java/net/snowflake/client/core/arrow/BitToBooleanConverterTest.java b/src/test/java/net/snowflake/client/core/arrow/BitToBooleanConverterTest.java index e5091d6fc..1fd65d911 100644 --- a/src/test/java/net/snowflake/client/core/arrow/BitToBooleanConverterTest.java +++ b/src/test/java/net/snowflake/client/core/arrow/BitToBooleanConverterTest.java @@ -73,7 +73,7 @@ public void testConvertToString() throws SFException { } else { assertThat(boolVal, is(expectedValues.get(i))); assertThat(objectVal, is(expectedValues.get(i))); - assertThat(stringVal, is(expectedValues.get(i).toString().toUpperCase())); + assertThat(stringVal, is(expectedValues.get(i).toString())); if (boolVal) { assertThat((byte) 0x1, is(converter.toBytes(i)[0])); } else { diff --git a/src/test/java/net/snowflake/client/core/json/StringConverterTest.java b/src/test/java/net/snowflake/client/core/json/StringConverterTest.java index 5fe3dd2cb..748548966 100644 --- a/src/test/java/net/snowflake/client/core/json/StringConverterTest.java +++ b/src/test/java/net/snowflake/client/core/json/StringConverterTest.java @@ -59,10 +59,10 @@ public void testConvertingString() throws SFException { @Test public void testConvertingBoolean() throws SFException { - assertEquals("TRUE", stringConverter.getString(true, Types.BOOLEAN, Types.BOOLEAN, 0)); - assertEquals("TRUE", stringConverter.getString("true", Types.BOOLEAN, Types.BOOLEAN, 0)); - assertEquals("FALSE", stringConverter.getString(false, Types.BOOLEAN, Types.BOOLEAN, 0)); - assertEquals("FALSE", stringConverter.getString("false", Types.BOOLEAN, Types.BOOLEAN, 0)); + assertEquals("true", stringConverter.getString(true, Types.BOOLEAN, Types.BOOLEAN, 0)); + assertEquals("true", stringConverter.getString("true", Types.BOOLEAN, Types.BOOLEAN, 0)); + assertEquals("false", stringConverter.getString(false, Types.BOOLEAN, Types.BOOLEAN, 0)); + assertEquals("false", stringConverter.getString("false", Types.BOOLEAN, Types.BOOLEAN, 0)); } @Test diff --git a/src/test/java/net/snowflake/client/jdbc/ResultSetJsonVsArrowIT.java b/src/test/java/net/snowflake/client/jdbc/ResultSetJsonVsArrowIT.java index 65cc27242..0dd8edd47 100644 --- a/src/test/java/net/snowflake/client/jdbc/ResultSetJsonVsArrowIT.java +++ b/src/test/java/net/snowflake/client/jdbc/ResultSetJsonVsArrowIT.java @@ -1501,12 +1501,12 @@ public void testBoolean() throws SQLException { ResultSet rs = statement.executeQuery("select * from " + table)) { assertTrue(rs.next()); assertTrue(rs.getBoolean(1)); - assertEquals("TRUE", rs.getString(1)); + assertEquals("true", rs.getString(1)); assertTrue(rs.next()); assertFalse(rs.getBoolean(1)); assertTrue(rs.next()); assertFalse(rs.getBoolean(1)); - assertEquals("FALSE", rs.getString(1)); + assertEquals("false", rs.getString(1)); assertFalse(rs.next()); statement.execute("drop table if exists " + table); } diff --git a/src/test/java/net/snowflake/client/jdbc/ResultSetLatestIT.java b/src/test/java/net/snowflake/client/jdbc/ResultSetLatestIT.java index d82cd9ff2..3ad105cea 100644 --- a/src/test/java/net/snowflake/client/jdbc/ResultSetLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/ResultSetLatestIT.java @@ -1176,9 +1176,9 @@ public void testGetObjectWithType() throws SQLException { assertResultValueAndType(statement, Double.valueOf("1.1"), "f", Double.class); assertResultValueAndType(statement, Double.valueOf("2.2"), "d", Double.class); assertResultValueAndType(statement, BigDecimal.valueOf(3.3), "bd", BigDecimal.class); - assertResultValueAndType(statement, "FALSE", "bool", String.class); + assertResultValueAndType(statement, "false", "bool", String.class); assertResultValueAndType(statement, Boolean.FALSE, "bool", Boolean.class); - assertResultValueAndType(statement, Long.valueOf(0), "bool", Long.class); + assertResultValueAndType(statement, 0L, "bool", Long.class); assertResultValueAsString( statement, new SnowflakeTimestampWithTimezone( diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java index b1da95b99..2857634f8 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java @@ -26,6 +26,7 @@ import java.util.Map; import net.snowflake.client.ConditionalIgnoreRule; import net.snowflake.client.RunningOnGithubAction; +import net.snowflake.client.TestUtil; import net.snowflake.client.ThrowingConsumer; import net.snowflake.client.category.TestCategoryResultSet; import net.snowflake.client.core.structs.SnowflakeObjectTypeFactories; @@ -144,43 +145,7 @@ public void testMapStructAllTypes() throws SQLException { try (Connection connection = init(); Statement statement = connection.createStatement()) { statement.execute("ALTER SESSION SET TIMEZONE = 'Europe/Warsaw'"); - try (ResultSet resultSet = - statement.executeQuery( - "select {" - + "'string': 'a', " - + "'b': 1, " - + "'s': 2, " - + "'i': 3, " - + "'l': 4, " - + "'f': 1.1, " - + "'d': 2.2, " - + "'bd': 3.3, " - + "'bool': true, " - + "'timestamp_ltz': '2021-12-22 09:43:44'::TIMESTAMP_LTZ, " - + "'timestamp_ntz': '2021-12-23 09:44:44'::TIMESTAMP_NTZ, " - + "'timestamp_tz': '2021-12-24 09:45:45 +0800'::TIMESTAMP_TZ, " - + "'date': '2023-12-24'::DATE, " - + "'time': '12:34:56'::TIME, " - + "'binary': TO_BINARY('616263', 'HEX'), " - + "'simpleClass': {'string': 'b', 'intValue': 2}" - + "}::OBJECT(" - + "string VARCHAR, " - + "b TINYINT, " - + "s SMALLINT, " - + "i INTEGER, " - + "l BIGINT, " - + "f FLOAT, " - + "d DOUBLE, " - + "bd DOUBLE, " - + "bool BOOLEAN, " - + "timestamp_ltz TIMESTAMP_LTZ, " - + "timestamp_ntz TIMESTAMP_NTZ, " - + "timestamp_tz TIMESTAMP_TZ, " - + "date DATE, " - + "time TIME, " - + "binary BINARY, " - + "simpleClass OBJECT(string VARCHAR, intValue INTEGER)" - + ")"); ) { + try (ResultSet resultSet = statement.executeQuery(AllTypesClass.ALL_TYPES_QUERY); ) { resultSet.next(); AllTypesClass object = resultSet.getObject(1, AllTypesClass.class); assertEquals("a", object.getString()); @@ -213,6 +178,14 @@ public void testMapStructAllTypes() throws SQLException { assertTrue(object.getBool()); assertEquals("b", object.getSimpleClass().getString()); assertEquals(Integer.valueOf(2), object.getSimpleClass().getIntValue()); + + if (queryResultFormat == ResultSetFormatType.NATIVE_ARROW) { + // Only verify getString for Arrow since JSON representations have difficulties with + // floating point toString conversion (3.300000000000000e+00 vs 3.3 in native arrow) + String expectedArrowGetStringResult = + "{\"string\": \"a\",\"b\": 1,\"s\": 2,\"i\": 3,\"l\": 4,\"f\": 1.1,\"d\": 2.2,\"bd\": 3.3,\"bool\": true,\"timestamp_ltz\": \"Wed, 22 Dec 2021 09:43:44 +0100\",\"timestamp_ntz\": \"Thu, 23 Dec 2021 09:44:44 Z\",\"timestamp_tz\": \"Fri, 24 Dec 2021 09:45:45 +0800\",\"date\": \"2023-12-24\",\"time\": \"12:34:56\",\"binary\": \"616263\",\"simpleClass\": {\"string\": \"b\",\"intValue\": 2}}"; + assertEquals(expectedArrowGetStringResult, resultSet.getString(1)); + } } } } @@ -234,43 +207,7 @@ public void testReturnStructAsStringIfTypeWasNotIndicated() throws SQLException + "TIMESTAMP_LTZ_OUTPUT_FORMAT='YYYY-MM-DD HH24:MI:SS.FF3 TZHTZM'," + "TIMESTAMP_NTZ_OUTPUT_FORMAT='YYYY-MM-DD HH24:MI:SS.FF3'"); - try (ResultSet resultSet = - statement.executeQuery( - "select {" - + "'string': 'a', " - + "'b': 1, " - + "'s': 2, " - + "'i': 3, " - + "'l': 4, " - + "'f': 1.1, " - + "'d': 2.2, " - + "'bd': 3.3, " - + "'bool': true, " - + "'timestamp_ltz': '2021-12-22 09:43:44'::TIMESTAMP_LTZ, " - + "'timestamp_ntz': '2021-12-23 09:44:44'::TIMESTAMP_NTZ, " - + "'timestamp_tz': '2021-12-24 09:45:45 +0800'::TIMESTAMP_TZ, " - + "'date': '2023-12-24'::DATE, " - + "'time': '12:34:56'::TIME, " - + "'binary': TO_BINARY('616263', 'HEX'), " - + "'simpleClass': {'string': 'b', 'intValue': 2}" - + "}::OBJECT(" - + "string VARCHAR, " - + "b TINYINT, " - + "s SMALLINT, " - + "i INTEGER, " - + "l BIGINT, " - + "f FLOAT, " - + "d DOUBLE, " - + "bd DOUBLE, " - + "bool BOOLEAN, " - + "timestamp_ltz TIMESTAMP_LTZ, " - + "timestamp_ntz TIMESTAMP_NTZ, " - + "timestamp_tz TIMESTAMP_TZ, " - + "date DATE, " - + "time TIME, " - + "binary BINARY, " - + "simpleClass OBJECT(string VARCHAR, intValue INTEGER)" - + ")"); ) { + try (ResultSet resultSet = statement.executeQuery(AllTypesClass.ALL_TYPES_QUERY); ) { resultSet.next(); String object = (String) resultSet.getObject(1); String expected = @@ -849,7 +786,7 @@ public void testMapArrayOfArrays() throws SQLException { @Test @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) public void testMapNestedStructures() throws SQLException { - withFirstRow( + String structSelectStatement = "SELECT {'simpleClass': {'string': 'a', 'intValue': 2}, " + "'simpleClasses': ARRAY_CONSTRUCT({'string': 'a', 'intValue': 2}, {'string': 'b', 'intValue': 2}), " + "'arrayOfSimpleClasses': ARRAY_CONSTRUCT({'string': 'a', 'intValue': 2}, {'string': 'b', 'intValue': 2}), " @@ -863,7 +800,11 @@ public void testMapNestedStructures() throws SQLException { + "mapOfSimpleClasses MAP(VARCHAR, OBJECT(string VARCHAR, intValue INTEGER))," + "texts ARRAY(VARCHAR)," + "arrayOfDates ARRAY(DATE)," - + "mapOfIntegers MAP(VARCHAR, INTEGER))", + + "mapOfIntegers MAP(VARCHAR, INTEGER))"; + String expectedQueryResult = + "{\"simpleClass\": {\"string\": \"a\",\"intValue\": 2},\"simpleClasses\": [{\"string\": \"a\",\"intValue\": 2},{\"string\": \"b\",\"intValue\": 2}],\"arrayOfSimpleClasses\": [{\"string\": \"a\",\"intValue\": 2},{\"string\": \"b\",\"intValue\": 2}],\"mapOfSimpleClasses\": {\"x\": {\"string\": \"c\",\"intValue\": 2},\"y\": {\"string\": \"d\",\"intValue\": 2}},\"texts\": [\"string\",\"a\"],\"arrayOfDates\": [\"2023-12-24\",\"2023-12-25\"],\"mapOfIntegers\": {\"x\": 3,\"y\": 4}}"; + withFirstRow( + structSelectStatement, (resultSet) -> { NestedStructSqlData nestedStructSqlData = resultSet.getObject(1, NestedStructSqlData.class); @@ -908,6 +849,7 @@ public void testMapNestedStructures() throws SQLException { assertEquals(Integer.valueOf(3), nestedStructSqlData.getMapOfIntegers().get("x")); assertEquals(Integer.valueOf(4), nestedStructSqlData.getMapOfIntegers().get("y")); + TestUtil.assertEqualsIgnoringWhitespace(expectedQueryResult, resultSet.getString(1)); }); } diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java new file mode 100644 index 000000000..a4bdb1194 --- /dev/null +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java @@ -0,0 +1,135 @@ +package net.snowflake.client.jdbc.structuredtypes; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import net.snowflake.client.ConditionalIgnoreRule; +import net.snowflake.client.RunningOnGithubAction; +import net.snowflake.client.category.TestCategoryResultSet; +import net.snowflake.client.jdbc.ResultSetFormatType; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +@Category(TestCategoryResultSet.class) +public class StructuredTypesGetStringArrowJsonCompatibilityIT + extends StructuredTypesGetStringBaseIT { + + private final String expectedStructureTypeRepresentation; + private final String selectSql; + private static Map connections = new HashMap<>(); + + public StructuredTypesGetStringArrowJsonCompatibilityIT( + ResultSetFormatType queryResultFormat, + String selectSql, + String expectedStructureTypeRepresentation) { + super(queryResultFormat); + this.selectSql = selectSql; + this.expectedStructureTypeRepresentation = expectedStructureTypeRepresentation; + } + + @Before + public void setUpConnection() throws SQLException { + // We initialize connection here since we need to set server properties that cannot be set in GH + // actions and before class is running even when all the tests have conditional ignore of tests + Connection connection = connections.get(queryResultFormat); + if (connection == null) { + connections.put(queryResultFormat, initConnection(queryResultFormat)); + } + } + + @AfterClass + public static void closeConnections() throws SQLException { + for (Connection connection : connections.values()) { + connection.close(); + } + } + + @Test + @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) + public void testRunAsGetString() throws SQLException { + withFirstRow( + connections.get(queryResultFormat), + selectSql, + (resultSet) -> assertGetStringIsCompatible(resultSet, expectedStructureTypeRepresentation)); + } + + @Parameterized.Parameters(name = "format={0},sql={1}") + public static Collection data() { + Map samples = new LinkedHashMap<>(); + samples.put("select {'a':3}::map(text, int);", "{\"a\":3}"); + samples.put( + "select {'a':'zażółć gęślą jaźń'}::map(text, text);", "{\"a\":\"zażółć gęślą jaźń\"}"); + samples.put("select {'a':'bla'}::map(text, text);", "{\"a\":\"bla\"}"); + samples.put("select {'1':'bla'}::map(int, text);", "{\"1\":\"bla\"}"); + samples.put("select {'1':[1,2,3]}::map(int, ARRAY(int));", "{\"1\":[1,2,3]}"); + samples.put( + "select {'1':{'string':'a'}}::map(int, OBJECT(string VARCHAR));", + "{\"1\":{\"string\":\"a\"}}"); + samples.put( + "select {'1':{'string':'a'}}::map(int, map(string, string));", + "{\"1\":{\"string\":\"a\"}}"); + samples.put( + "select {'1':[{'string':'a'},{'bla':'ble'}]}::map(int, array(map(string, string)));", + "{\"1\":[{\"string\":\"a\"},{\"bla\":\"ble\"}]}"); + samples.put("select [1,2,3]::array(int)", "[1,2,3]"); + samples.put( + "select [{'a':'a'}, {'b':'b'}]::array(map(string, string))", + "[{\"a\":\"a\"}, {\"b\":\"b\"}]"); + samples.put( + "select [{'string':'a'}, {'string':'b'}]::array(object(string varchar))", + "[{\"string\":\"a\"}, {\"string\":\"b\"}]"); + samples.put("select {'string':'a'}::object(string varchar)", "{\"string\":\"a\"}"); + samples.put( + "select {'x':'a','b':'a','c':'a','d':'a','e':'a'}::object(x varchar,b varchar,c varchar,d varchar,e varchar)", + "{\"x\":\"a\",\"b\":\"a\",\"c\":\"a\",\"d\":\"a\",\"e\":\"a\"}"); + samples.put("select {'string':[1,2,3]}::object(string array(int))", "{\"string\":[1,2,3]}"); + samples.put( + "select {'string':{'a':15}}::object(string object(a int))", "{\"string\":{\"a\":15}}"); + samples.put( + "select {'string':{'a':15}}::object(string map(string,int))", "{\"string\":{\"a\":15}}"); + samples.put( + "select {'string':{'a':{'b':15}}}::object(string object(a map(string, int)))", + "{\"string\":{\"a\":{\"b\":15}}}"); + + samples.put( + "select {'string':{'a':{'b':[{'c': 15}]}}}::object(string map(string, object(b array(object(c int)))))", + "{\"string\":{\"a\":{\"b\":[{\"c\":15}]}}}"); + // DY, DD MON YYYY HH24:MI:SS TZHTZM + samples.put( + "select {'ltz': '2024-05-20 11:22:33'::TIMESTAMP_LTZ}::object(ltz TIMESTAMP_LTZ)", + "{\"ltz\":\"Mon, 20 May 2024 11:22:33 +0200\"}"); + samples.put( + "select {'ntz': '2024-05-20 11:22:33'::TIMESTAMP_NTZ}::object(ntz TIMESTAMP_NTZ)", + "{\"ntz\":\"Mon, 20 May 2024 11:22:33 Z\"}"); + samples.put( + "select {'tz': '2024-05-20 11:22:33+0800'::TIMESTAMP_TZ}::object(tz TIMESTAMP_TZ)", + "{\"tz\":\"Mon, 20 May 2024 11:22:33 +0800\"}"); + samples.put( + "select {'date': '2024-05-20'::DATE}::object(date DATE)", "{\"date\":\"2024-05-20\"}"); + samples.put("select {'time': '22:14:55'::TIME}::object(time TIME)", "{\"time\":\"22:14:55\"}"); + samples.put("select {'bool': TRUE}::object(bool BOOLEAN)", "{\"bool\":true}"); + samples.put("select {'bool': 'y'}::object(bool BOOLEAN)", "{\"bool\":true}"); + samples.put( + "select {'binary': TO_BINARY('616263', 'HEX')}::object(binary BINARY)", + "{\"binary\":\"616263\"}"); + samples.put("select [1,2,3]::VECTOR(INT, 3)", "[1,2,3]"); + samples.put("select ['a','b','c']::ARRAY(varchar)", "[\"a\",\"b\",\"c\"]"); + + Collection parameters = new ArrayList<>(); + for (ResultSetFormatType resultSetFormatType : ResultSetFormatType.values()) { + samples.forEach( + (sql, expected) -> parameters.add(new Object[] {resultSetFormatType, sql, expected})); + } + + return parameters; + } +} diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java new file mode 100644 index 000000000..d9d5c15e2 --- /dev/null +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -0,0 +1,68 @@ +package net.snowflake.client.jdbc.structuredtypes; + +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import net.snowflake.client.TestUtil; +import net.snowflake.client.ThrowingConsumer; +import net.snowflake.client.jdbc.BaseJDBCTest; +import net.snowflake.client.jdbc.ResultSetFormatType; + +abstract class StructuredTypesGetStringBaseIT extends BaseJDBCTest { + + protected final ResultSetFormatType queryResultFormat; + + public StructuredTypesGetStringBaseIT(ResultSetFormatType queryResultFormat) { + this.queryResultFormat = queryResultFormat; + } + + protected Connection init() throws SQLException { + return initConnection(this.queryResultFormat); + } + + protected static Connection initConnection(ResultSetFormatType queryResultFormat) + throws SQLException { + Connection conn = BaseJDBCTest.getConnection(BaseJDBCTest.DONT_INJECT_SOCKET_TIMEOUT); + try (Statement stmt = conn.createStatement()) { + stmt.execute("alter session set USE_CACHED_RESULT = false"); + stmt.execute("alter session set ENABLE_STRUCTURED_TYPES_IN_CLIENT_RESPONSE = true"); + stmt.execute("alter session set IGNORE_CLIENT_VESRION_IN_STRUCTURED_TYPES_RESPONSE = true"); + stmt.execute("ALTER SESSION SET TIMEZONE = 'Europe/Warsaw'"); + stmt.execute( + "alter session set " + + "TIMESTAMP_TYPE_MAPPING='TIMESTAMP_LTZ'," + + "TIMESTAMP_OUTPUT_FORMAT='DY, DD MON YYYY HH24:MI:SS TZHTZM'," + + "TIMESTAMP_TZ_OUTPUT_FORMAT='DY, DD MON YYYY HH24:MI:SS TZHTZM'," + + "TIMESTAMP_LTZ_OUTPUT_FORMAT='DY, DD MON YYYY HH24:MI:SS TZHTZM'," + + "TIMESTAMP_NTZ_OUTPUT_FORMAT='DY, DD MON YYYY HH24:MI:SS TZHTZM'"); + stmt.execute( + "alter session set jdbc_query_result_format = '" + + queryResultFormat.sessionParameterTypeValue + + "'"); + if (queryResultFormat == ResultSetFormatType.NATIVE_ARROW) { + stmt.execute("alter session set ENABLE_STRUCTURED_TYPES_NATIVE_ARROW_FORMAT = true"); + stmt.execute("alter session set FORCE_ENABLE_STRUCTURED_TYPES_NATIVE_ARROW_FORMAT = true"); + } + } + return conn; + } + + protected void assertGetStringIsCompatible(ResultSet resultSet, String expected) + throws SQLException { + String result = resultSet.getString(1); + TestUtil.assertEqualsIgnoringWhitespace(expected, result); + } + + protected void withFirstRow( + Connection connection, String sqlText, ThrowingConsumer consumer) + throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet rs = statement.executeQuery(sqlText); ) { + assertTrue(rs.next()); + consumer.accept(rs); + } + } +} diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/sqldata/AllTypesClass.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/sqldata/AllTypesClass.java index 3f12a9f63..f8b494a87 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/sqldata/AllTypesClass.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/sqldata/AllTypesClass.java @@ -11,6 +11,43 @@ import net.snowflake.client.jdbc.SnowflakeColumn; public class AllTypesClass implements SQLData { + public static String ALL_TYPES_QUERY = + "select {" + + "'string': 'a', " + + "'b': 1, " + + "'s': 2, " + + "'i': 3, " + + "'l': 4, " + + "'f': 1.1, " + + "'d': 2.2, " + + "'bd': 3.3, " + + "'bool': true, " + + "'timestamp_ltz': '2021-12-22 09:43:44'::TIMESTAMP_LTZ, " + + "'timestamp_ntz': '2021-12-23 09:44:44'::TIMESTAMP_NTZ, " + + "'timestamp_tz': '2021-12-24 09:45:45 +0800'::TIMESTAMP_TZ, " + + "'date': '2023-12-24'::DATE, " + + "'time': '12:34:56'::TIME, " + + "'binary': TO_BINARY('616263', 'HEX'), " + + "'simpleClass': {'string': 'b', 'intValue': 2}" + + "}::OBJECT(" + + "string VARCHAR, " + + "b TINYINT, " + + "s SMALLINT, " + + "i INTEGER, " + + "l BIGINT, " + + "f FLOAT, " + + "d DOUBLE, " + + "bd DOUBLE, " + + "bool BOOLEAN, " + + "timestamp_ltz TIMESTAMP_LTZ, " + + "timestamp_ntz TIMESTAMP_NTZ, " + + "timestamp_tz TIMESTAMP_TZ, " + + "date DATE, " + + "time TIME, " + + "binary BINARY, " + + "simpleClass OBJECT(string VARCHAR, intValue INTEGER)" + + ")"; + private String string; private Byte b; private Short s;