From 5903ebbebfd9816e4901ce44caf0beda5a9adba8 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:18:57 -0700 Subject: [PATCH] Support user-defined and incomplete date formats (#273) (#1821) (#1830) * Support user-defined and incomplete date formats (#273) * Check custom formats for characters Signed-off-by: Guian Gumpac * Removed duplicated code Signed-off-by: Guian Gumpac * Reworked checking for exprcoretype Signed-off-by: Guian Gumpac * Changed check for time Signed-off-by: Guian Gumpac * Rework processing custom and incomplete formats and add tests. Signed-off-by: Yury-Fridlyand * Values of incomplete and incorrect formats to be returned as `TIMESTAMP` instead of `STRING`. Signed-off-by: Yury-Fridlyand * Complete fix and update tests. Signed-off-by: Yury-Fridlyand * More fixes for god of fixes. Signed-off-by: Yury-Fridlyand --------- Signed-off-by: Guian Gumpac Signed-off-by: Yury-Fridlyand Co-authored-by: Yury-Fridlyand * Refactoring. Signed-off-by: Yury-Fridlyand --------- Signed-off-by: Guian Gumpac Signed-off-by: Yury-Fridlyand Co-authored-by: Guian Gumpac (cherry picked from commit a60b222fe6333257d090a028626002498888c81f) Co-authored-by: Yury-Fridlyand --- .../opensearch/sql/sql/DateTimeFormatsIT.java | 64 ++++ .../src/test/resources/date_formats.json | 4 +- .../date_formats_index_mapping.json | 62 +++- .../data/type/OpenSearchDateType.java | 198 ++++++++--- .../value/OpenSearchExprValueFactory.java | 176 ++++------ .../data/type/OpenSearchDateTypeTest.java | 306 +++++++++------- .../value/OpenSearchExprValueFactoryTest.java | 326 ++++++++++-------- 7 files changed, 702 insertions(+), 434 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index 7cd95fb509..fc05e502c5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.Locale; +import lombok.SneakyThrows; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -56,6 +57,69 @@ public void testDateFormatsWithOr() throws IOException { rows("1984-04-12 09:07:42.000123456")); } + @Test + @SneakyThrows + public void testCustomFormats() { + String query = String.format("SELECT custom_time, custom_timestamp, custom_date_or_date," + + "custom_date_or_custom_time, custom_time_parser_check FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("custom_time", null, "time"), + schema("custom_timestamp", null, "timestamp"), + schema("custom_date_or_date", null, "date"), + schema("custom_date_or_custom_time", null, "timestamp"), + schema("custom_time_parser_check", null, "time")); + verifyDataRows(result, + rows("09:07:42", "1984-04-12 09:07:42", "1984-04-12", "1961-04-12 00:00:00", "23:44:36.321"), + rows("21:07:42", "1984-04-12 22:07:42", "1984-04-12", "1970-01-01 09:07:00", "09:01:16.542")); + } + + @Test + @SneakyThrows + public void testCustomFormats2() { + String query = String.format("SELECT custom_no_delimiter_date, custom_no_delimiter_time," + + "custom_no_delimiter_ts FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("custom_no_delimiter_date", null, "date"), + schema("custom_no_delimiter_time", null, "time"), + schema("custom_no_delimiter_ts", null, "timestamp")); + verifyDataRows(result, + rows("1984-10-20", "10:20:30", "1984-10-20 15:35:48"), + rows("1961-04-12", "09:07:00", "1961-04-12 09:07:00")); + } + + @Test + @SneakyThrows + public void testIncompleteFormats() { + String query = String.format("SELECT incomplete_1, incomplete_2, incorrect," + + "incomplete_custom_time, incomplete_custom_date FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("incomplete_1", null, "timestamp"), + schema("incomplete_2", null, "date"), + schema("incorrect", null, "timestamp"), + schema("incomplete_custom_time", null, "time"), + schema("incomplete_custom_date", null, "date")); + verifyDataRows(result, + rows("1984-01-01 00:00:00", null, null, "10:00:00", "1999-01-01"), + rows("2012-01-01 00:00:00", null, null, "20:00:00", "3021-01-01")); + } + + @Test + @SneakyThrows + public void testNumericFormats() { + String query = String.format("SELECT epoch_sec, epoch_milli" + + " FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("epoch_sec", null, "timestamp"), + schema("epoch_milli", null, "timestamp")); + verifyDataRows(result, + rows("1970-01-01 00:00:42", "1970-01-01 00:00:00.042"), + rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5")); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/integ-test/src/test/resources/date_formats.json b/integ-test/src/test/resources/date_formats.json index cc694930e9..13d46a0e8c 100644 --- a/integ-test/src/test/resources/date_formats.json +++ b/integ-test/src/test/resources/date_formats.json @@ -1,4 +1,4 @@ {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "HH:mm:ss": "09:07:42", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42"} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "1961-04-12", "custom_time_parser_check": "85476321", "incomplete_1" : 1984, "incomplete_2": null, "incomplete_custom_date": 1999, "incomplete_custom_time" : 10, "incorrect" : null, "epoch_sec" : 42, "epoch_milli" : 42, "custom_no_delimiter_date" : "19841020", "custom_no_delimiter_time" : "102030", "custom_no_delimiter_ts" : "19841020153548"} {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "HH:mm:ss": "09:07:42", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z"} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "09:07:00", "custom_time_parser_check": "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"} diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json index 938f598d0b..65811f8d9e 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json @@ -289,9 +289,9 @@ "type" : "date", "format": "yyyy-MM-dd" }, - "HH:mm:ss" : { + "custom_time" : { "type" : "date", - "format": "HH:mm:ss" + "format": "hh:mm:ss a" }, "yyyy-MM-dd_OR_epoch_millis" : { "type" : "date", @@ -300,7 +300,63 @@ "hour_minute_second_OR_t_time" : { "type" : "date", "format": "hour_minute_second||t_time" + }, + "custom_timestamp" : { + "type" : "date", + "format": "yyyy-MM-dd hh:mm:ss ---- a" + }, + "custom_date_or_date" : { + "type" : "date", + "format": "yyyy-MM-dd||date" + }, + "custom_date_or_custom_time" : { + "type" : "date", + "format" : "yyyy-MM-dd || HH:mm:ss" + }, + "custom_time_parser_check" : { + "type" : "date", + "format" : "::: k-A || A " + }, + "incomplete_1" : { + "type" : "date", + "format" : "year" + }, + "incomplete_2" : { + "type" : "date", + "format" : "E-w" + }, + "incomplete_custom_date" : { + "type" : "date", + "format" : "uuuu" + }, + "incomplete_custom_time" : { + "type" : "date", + "format" : "HH" + }, + "incorrect" : { + "type" : "date", + "format" : "'___'" + }, + "epoch_sec" : { + "type" : "date", + "format" : "epoch_second" + }, + "epoch_milli" : { + "type" : "date", + "format" : "epoch_millis" + }, + "custom_no_delimiter_date" : { + "type" : "date", + "format" : "uuuuMMdd" + }, + "custom_no_delimiter_time" : { + "type" : "date", + "format" : "HHmmss" + }, + "custom_no_delimiter_ts" : { + "type" : "date", + "format" : "uuuuMMddHHmmss" } } } -} \ No newline at end of file +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 3554a5b2b4..76947bf720 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; @@ -27,10 +28,15 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final OpenSearchDateType instance = new OpenSearchDateType(); + /** Numeric formats which support full datetime. */ + public static final List SUPPORTED_NAMED_NUMERIC_FORMATS = List.of( + FormatNames.EPOCH_MILLIS, + FormatNames.EPOCH_SECOND + ); + + /** List of named formats which support full datetime. */ public static final List SUPPORTED_NAMED_DATETIME_FORMATS = List.of( FormatNames.ISO8601, - FormatNames.EPOCH_MILLIS, - FormatNames.EPOCH_SECOND, FormatNames.BASIC_DATE_TIME, FormatNames.BASIC_DATE_TIME_NO_MILLIS, FormatNames.BASIC_ORDINAL_DATE_TIME, @@ -69,7 +75,7 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_WEEK_DATE_TIME_NO_MILLIS ); - // list of named formats that only support year/month/day + /** List of named formats that only support year/month/day. */ public static final List SUPPORTED_NAMED_DATE_FORMATS = List.of( FormatNames.BASIC_DATE, FormatNames.BASIC_ORDINAL_DATE, @@ -77,16 +83,21 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_DATE, FormatNames.YEAR_MONTH_DAY, FormatNames.STRICT_YEAR_MONTH_DAY, - FormatNames.YEAR_MONTH, - FormatNames.STRICT_YEAR_MONTH, - FormatNames.YEAR, - FormatNames.STRICT_YEAR, FormatNames.ORDINAL_DATE, FormatNames.STRICT_ORDINAL_DATE, FormatNames.WEEK_DATE, FormatNames.STRICT_WEEK_DATE, FormatNames.WEEKYEAR_WEEK_DAY, - FormatNames.STRICT_WEEKYEAR_WEEK_DAY, + FormatNames.STRICT_WEEKYEAR_WEEK_DAY + ); + + /** list of named formats which produce incomplete date, + * e.g. 1 or 2 are missing from tuple year/month/day. */ + public static final List SUPPORTED_NAMED_INCOMPLETE_DATE_FORMATS = List.of( + FormatNames.YEAR_MONTH, + FormatNames.STRICT_YEAR_MONTH, + FormatNames.YEAR, + FormatNames.STRICT_YEAR, FormatNames.WEEK_YEAR, FormatNames.WEEK_YEAR_WEEK, FormatNames.STRICT_WEEKYEAR_WEEK, @@ -94,7 +105,7 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_WEEKYEAR ); - // list of named formats that only support hour/minute/second + /** List of named formats that only support hour/minute/second. */ public static final List SUPPORTED_NAMED_TIME_FORMATS = List.of( FormatNames.BASIC_TIME, FormatNames.BASIC_TIME_NO_MILLIS, @@ -120,12 +131,17 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_T_TIME_NO_MILLIS ); + /** Formatter symbols which used to format time or date correspondingly. + * {@link java.time.format.DateTimeFormatter}. */ + private static final String CUSTOM_FORMAT_TIME_SYMBOLS = "nNASsmHkKha"; + private static final String CUSTOM_FORMAT_DATE_SYMBOLS = "FecEWwYqQgdMLDyuG"; + @EqualsAndHashCode.Exclude - String formatString; + private final List formats; private OpenSearchDateType() { super(MappingType.Date); - this.formatString = ""; + this.formats = List.of(); } private OpenSearchDateType(ExprCoreType exprCoreType) { @@ -138,102 +154,194 @@ private OpenSearchDateType(ExprType exprType) { this.exprCoreType = (ExprCoreType) exprType; } - private OpenSearchDateType(String formatStringArg) { + private OpenSearchDateType(String format) { super(MappingType.Date); - this.formatString = formatStringArg; - this.exprCoreType = getExprTypeFromFormatString(formatStringArg); + this.formats = getFormatList(format); + this.exprCoreType = getExprTypeFromFormatString(format); + } + + public boolean hasFormats() { + return !formats.isEmpty(); } /** * Retrieves and splits a user defined format string from the mapping into a list of formats. * @return A list of format names and user defined formats. */ - private List getFormatList() { - String format = strip8Prefix(formatString); - List patterns = splitCombinedPatterns(format); - return patterns; + private List getFormatList(String format) { + format = strip8Prefix(format); + return splitCombinedPatterns(format).stream().map(String::trim).collect(Collectors.toList()); } - /** * Retrieves a list of named OpenSearch formatters given by user mapping. * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. */ public List getAllNamedFormatters() { - return getFormatList().stream() + return formats.stream() .filter(formatString -> FormatNames.forName(formatString) != null) .map(DateFormatter::forPattern).collect(Collectors.toList()); } + /** + * Retrieves a list of numeric formatters that format for dates. + * @return a list of DateFormatters that can be used to parse a Date. + */ + public List getNumericNamedFormatters() { + return formats.stream() + .filter(formatString -> { + FormatNames namedFormat = FormatNames.forName(formatString); + return namedFormat != null && SUPPORTED_NAMED_NUMERIC_FORMATS.contains(namedFormat); + }) + .map(DateFormatter::forPattern).collect(Collectors.toList()); + } + + /** + * Retrieves a list of custom formats defined by the user. + * @return a list of formats as strings that can be used to parse a Date/Time/Timestamp. + */ + public List getAllCustomFormats() { + return formats.stream() + .filter(format -> FormatNames.forName(format) == null) + .map(format -> { + try { + DateFormatter.forPattern(format); + return format; + } catch (Exception ignored) { + // parsing failed + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + /** * Retrieves a list of custom formatters defined by the user. * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. */ public List getAllCustomFormatters() { - return getFormatList().stream() - .filter(formatString -> FormatNames.forName(formatString) == null) - .map(DateFormatter::forPattern).collect(Collectors.toList()); + return getAllCustomFormats().stream() + .map(DateFormatter::forPattern) + .collect(Collectors.toList()); } /** * Retrieves a list of named formatters that format for dates. - * * @return a list of DateFormatters that can be used to parse a Date. */ public List getDateNamedFormatters() { - return getFormatList().stream() + return formats.stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); - return SUPPORTED_NAMED_DATE_FORMATS.contains(namedFormat); + return namedFormat != null && SUPPORTED_NAMED_DATE_FORMATS.contains(namedFormat); }) .map(DateFormatter::forPattern).collect(Collectors.toList()); } /** * Retrieves a list of named formatters that format for Times. - * * @return a list of DateFormatters that can be used to parse a Time. */ public List getTimeNamedFormatters() { - return getFormatList().stream() + return formats.stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); - return SUPPORTED_NAMED_TIME_FORMATS.contains(namedFormat); + return namedFormat != null && SUPPORTED_NAMED_TIME_FORMATS.contains(namedFormat); }) .map(DateFormatter::forPattern).collect(Collectors.toList()); } - private ExprCoreType getExprTypeFromFormatString(String formatString) { - if (formatString.isEmpty()) { - // FOLLOW-UP: check the default formatter - and set it here instead - // of assuming that the default is always a timestamp - return TIMESTAMP; + /** + * Retrieves a list of named formatters that format for DateTimes. + * @return a list of DateFormatters that can be used to parse a DateTime. + */ + public List getDateTimeNamedFormatters() { + return formats.stream() + .filter(formatString -> { + FormatNames namedFormat = FormatNames.forName(formatString); + return namedFormat != null && SUPPORTED_NAMED_DATETIME_FORMATS.contains(namedFormat); + }) + .map(DateFormatter::forPattern).collect(Collectors.toList()); + } + + private ExprCoreType getExprTypeFromCustomFormats(List formats) { + boolean isDate = false; + boolean isTime = false; + + for (String format : formats) { + if (!isTime) { + for (char symbol : CUSTOM_FORMAT_TIME_SYMBOLS.toCharArray()) { + if (format.contains(String.valueOf(symbol))) { + isTime = true; + break; + } + } + } + if (!isDate) { + for (char symbol : CUSTOM_FORMAT_DATE_SYMBOLS.toCharArray()) { + if (format.contains(String.valueOf(symbol))) { + isDate = true; + break; + } + } + } + if (isDate && isTime) { + return TIMESTAMP; + } + } + + if (isDate) { + return DATE; + } + if (isTime) { + return TIME; } - List namedFormatters = getAllNamedFormatters(); + // Incomplete or incorrect formats: can't be converted to DATE nor TIME, for example `year` + return TIMESTAMP; + } - if (namedFormatters.isEmpty()) { + private ExprCoreType getExprTypeFromFormatString(String formatString) { + List datetimeFormatters = getDateTimeNamedFormatters(); + List numericFormatters = getNumericNamedFormatters(); + + if (formatString.isEmpty() || !datetimeFormatters.isEmpty() || !numericFormatters.isEmpty()) { return TIMESTAMP; } - if (!getAllCustomFormatters().isEmpty()) { - // FOLLOW-UP: support custom format in + List timeFormatters = getTimeNamedFormatters(); + List dateFormatters = getDateNamedFormatters(); + if (!timeFormatters.isEmpty() && !dateFormatters.isEmpty()) { return TIMESTAMP; } + List customFormatters = getAllCustomFormats(); + if (!customFormatters.isEmpty()) { + ExprCoreType customFormatType = getExprTypeFromCustomFormats(customFormatters); + ExprCoreType combinedByDefaultFormats = customFormatType; + if (!dateFormatters.isEmpty()) { + combinedByDefaultFormats = DATE; + } + if (!timeFormatters.isEmpty()) { + combinedByDefaultFormats = TIME; + } + return customFormatType == combinedByDefaultFormats ? customFormatType : TIMESTAMP; + } + // if there is nothing in the dateformatter that accepts a year/month/day, then // we can assume the type is strictly a Time object - if (namedFormatters.size() == getTimeNamedFormatters().size()) { + if (!timeFormatters.isEmpty()) { return TIME; } // if there is nothing in the dateformatter that accepts a hour/minute/second, then // we can assume the type is strictly a Date object - if (namedFormatters.size() == getDateNamedFormatters().size()) { + if (!dateFormatters.isEmpty()) { return DATE; } - // According to the user mapping, this field may contain a DATE or a TIME + // Unknown or incorrect format provided return TIMESTAMP; } @@ -280,7 +388,7 @@ public static OpenSearchDateType of() { @Override public List getParent() { - return List.of(this.exprCoreType); + return List.of(exprCoreType); } @Override @@ -290,9 +398,9 @@ public boolean shouldCast(ExprType other) { @Override protected OpenSearchDataType cloneEmpty() { - if (this.formatString.isEmpty()) { - return OpenSearchDateType.of(this.exprCoreType); + if (formats.isEmpty()) { + return OpenSearchDateType.of(exprCoreType); } - return OpenSearchDateType.of(this.formatString); + return OpenSearchDateType.of(String.join(" || ", formats)); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index abad197bd4..22a43d3444 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -45,6 +45,7 @@ import lombok.Setter; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; +import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; import org.opensearch.sql.data.model.ExprCollectionValue; @@ -60,6 +61,7 @@ import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; @@ -82,7 +84,7 @@ public class OpenSearchExprValueFactory { /** * Extend existing mapping by new data without overwrite. - * Called from aggregation only {@link AggregationQueryBuilder#buildTypeMapping}. + * Called from aggregation only {@see AggregationQueryBuilder#buildTypeMapping}. * @param typeMapping A data type mapping produced by aggregation. */ public void extendTypeMapping(Map typeMapping) { @@ -124,14 +126,10 @@ public void extendTypeMapping(Map typeMapping) { .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Boolean), (c, dt) -> ExprBooleanValue.of(c.booleanValue())) //Handles the creation of DATE, TIME & DATETIME - .put(OpenSearchDateType.of(TIME), - this::createOpenSearchDateType) - .put(OpenSearchDateType.of(DATE), - this::createOpenSearchDateType) - .put(OpenSearchDateType.of(TIMESTAMP), - this::createOpenSearchDateType) - .put(OpenSearchDateType.of(DATETIME), - this::createOpenSearchDateType) + .put(OpenSearchDateType.of(TIME), this::createOpenSearchDateType) + .put(OpenSearchDateType.of(DATE), this::createOpenSearchDateType) + .put(OpenSearchDateType.of(TIMESTAMP), this::createOpenSearchDateType) + .put(OpenSearchDateType.of(DATETIME), this::createOpenSearchDateType) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip), (c, dt) -> new OpenSearchExprIpValue(c.stringValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint), @@ -217,137 +215,81 @@ private Optional type(String field) { } /** - * Parses value with the first matching formatter as an Instant to UTF. - * - * @param value - timestamp as string - * @param dateType - field type - * @return Instant without timezone - */ - private ExprValue parseTimestampString(String value, OpenSearchDateType dateType) { - Instant parsed = null; - for (DateFormatter formatter : dateType.getAllNamedFormatters()) { - try { - TemporalAccessor accessor = formatter.parse(value); - ZonedDateTime zonedDateTime = DateFormatters.from(accessor); - // remove the Zone - parsed = zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toInstant(); - } catch (IllegalArgumentException ignored) { - // nothing to do, try another format - } - } - - // FOLLOW-UP PR: Check custom formatters too - - // if no named formatters are available, use the default - if (dateType.getAllNamedFormatters().size() == 0 - || dateType.getAllCustomFormatters().size() > 0) { - try { - parsed = DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant(); - } catch (DateTimeParseException e) { - // ignored - } - } - - if (parsed == null) { - // otherwise, throw an error that no formatters worked - throw new IllegalArgumentException( - String.format( - "Construct ExprTimestampValue from \"%s\" failed, unsupported date format.", value) - ); - } - - return new ExprTimestampValue(parsed); - } - - /** - * return the first matching formatter as a time without timezone. + * Parse value with the first matching formatter into {@link ExprValue} + * with corresponding {@link ExprCoreType}. * * @param value - time as string - * @param dateType - field data type - * @return time without timezone + * @param dataType - field data type + * @return Parsed value */ - private ExprValue parseTimeString(String value, OpenSearchDateType dateType) { - for (DateFormatter formatter : dateType.getAllNamedFormatters()) { - try { - TemporalAccessor accessor = formatter.parse(value); - ZonedDateTime zonedDateTime = DateFormatters.from(accessor); - return new ExprTimeValue( - zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalTime()); - } catch (IllegalArgumentException ignored) { - // nothing to do, try another format - } - } + private ExprValue parseDateTimeString(String value, OpenSearchDateType dataType) { + List formatters = dataType.getAllNamedFormatters(); + formatters.addAll(dataType.getAllCustomFormatters()); + ExprCoreType returnFormat = (ExprCoreType) dataType.getExprType(); - // if no named formatters are available, use the default - if (dateType.getAllNamedFormatters().size() == 0) { - try { - return new ExprTimeValue( - DateFormatters.from(STRICT_HOUR_MINUTE_SECOND_FORMATTER.parse(value)).toLocalTime()); - } catch (DateTimeParseException e) { - // ignored - } - } - throw new IllegalArgumentException("Construct ExprTimeValue from \"" + value - + "\" failed, unsupported time format."); - } - - /** - * return the first matching formatter as a date without timezone. - * - * @param value - date as string - * @param dateType - field data type - * @return date without timezone - */ - private ExprValue parseDateString(String value, OpenSearchDateType dateType) { - for (DateFormatter formatter : dateType.getAllNamedFormatters()) { + for (DateFormatter formatter : formatters) { try { TemporalAccessor accessor = formatter.parse(value); ZonedDateTime zonedDateTime = DateFormatters.from(accessor); - // return the first matching formatter as a date without timezone - return new ExprDateValue( - zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalDate()); - } catch (IllegalArgumentException ignored) { + switch (returnFormat) { + case TIME: return new ExprTimeValue( + zonedDateTime.withZoneSameLocal(UTC_ZONE_ID).toLocalTime()); + case DATE: return new ExprDateValue( + zonedDateTime.withZoneSameLocal(UTC_ZONE_ID).toLocalDate()); + default: return new ExprTimestampValue( + zonedDateTime.withZoneSameLocal(UTC_ZONE_ID).toInstant()); + } + } catch (IllegalArgumentException ignored) { // nothing to do, try another format } } - // if no named formatters are available, use the default - if (dateType.getAllNamedFormatters().size() == 0) { - try { - return new ExprDateValue( + // if no formatters are available, try the default formatter + try { + switch (returnFormat) { + case TIME: return new ExprTimeValue( + DateFormatters.from(STRICT_HOUR_MINUTE_SECOND_FORMATTER.parse(value)).toLocalTime()); + case DATE: return new ExprDateValue( DateFormatters.from(STRICT_YEAR_MONTH_DAY_FORMATTER.parse(value)).toLocalDate()); - } catch (DateTimeParseException e) { - // ignored + default: return new ExprTimestampValue( + DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant()); } + } catch (DateTimeParseException ignored) { + // ignored } - throw new IllegalArgumentException("Construct ExprDateValue from \"" + value - + "\" failed, unsupported date format."); + + throw new IllegalArgumentException(String.format( + "Construct %s from \"%s\" failed, unsupported format.", returnFormat, value)); } private ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; ExprType returnFormat = dt.getExprType(); - if (value.isNumber()) { - Instant epochMillis = Instant.ofEpochMilli(value.longValue()); - if (returnFormat == TIME) { - return new ExprTimeValue(LocalTime.from(epochMillis.atZone(UTC_ZONE_ID))); - } - if (returnFormat == DATE) { - return new ExprDateValue(LocalDate.ofInstant(epochMillis, UTC_ZONE_ID)); + if (value.isNumber()) { // isNumber + var numFormatters = dt.getNumericNamedFormatters(); + if (numFormatters.size() > 0 || !dt.hasFormats()) { + long epochMillis = 0; + if (numFormatters.contains(DateFormatter.forPattern( + FormatNames.EPOCH_SECOND.getSnakeCaseName()))) { + // no CamelCase for `EPOCH_*` formats + epochMillis = value.longValue() * 1000; + } else /* EPOCH_MILLIS */ { + epochMillis = value.longValue(); + } + Instant instant = Instant.ofEpochMilli(epochMillis); + switch ((ExprCoreType) returnFormat) { + case TIME: return new ExprTimeValue(LocalTime.from(instant.atZone(UTC_ZONE_ID))); + case DATE: return new ExprDateValue(LocalDate.ofInstant(instant, UTC_ZONE_ID)); + default: return new ExprTimestampValue(instant); + } + } else { + // custom format + return parseDateTimeString(value.stringValue(), dt); } - return new ExprTimestampValue(Instant.ofEpochMilli(value.longValue())); } - if (value.isString()) { - if (returnFormat == TIME) { - return parseTimeString(value.stringValue(), dt); - } - if (returnFormat == DATE) { - return parseDateString(value.stringValue(), dt); - } - // else timestamp/datetime - return parseTimestampString(value.stringValue(), dt); + return parseDateTimeString(value.stringValue(), dt); } return new ExprTimestampValue((Instant) value.objectValue()); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index f0add5bcd9..13393da732 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -3,34 +3,37 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.opensearch.data.type; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNotSame; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assertions.fail; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_DATETIME_FORMATS; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_DATE_FORMATS; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_INCOMPLETE_DATE_FORMATS; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_NUMERIC_FORMATS; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_TIME_FORMATS; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.isDateTypeCompatible; +import com.google.common.collect.Lists; import java.util.EnumSet; +import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.common.time.FormatNames; -import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.data.type.ExprCoreType; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class OpenSearchDateTypeTest { @@ -53,150 +56,207 @@ class OpenSearchDateTypeTest { @Test public void isCompatible() { - // timestamp types is compatible with all date-types - assertTrue(TIMESTAMP.isCompatible(defaultDateType)); - assertTrue(TIMESTAMP.isCompatible(dateDateType)); - assertTrue(TIMESTAMP.isCompatible(timeDateType)); - assertTrue(TIMESTAMP.isCompatible(datetimeDateType)); - - // datetime - assertFalse(DATETIME.isCompatible(defaultDateType)); - assertTrue(DATETIME.isCompatible(dateDateType)); - assertTrue(DATETIME.isCompatible(timeDateType)); - assertFalse(DATETIME.isCompatible(datetimeDateType)); - - // time type - assertFalse(TIME.isCompatible(defaultDateType)); - assertFalse(TIME.isCompatible(dateDateType)); - assertTrue(TIME.isCompatible(timeDateType)); - assertFalse(TIME.isCompatible(datetimeDateType)); - - // date type - assertFalse(DATE.isCompatible(defaultDateType)); - assertTrue(DATE.isCompatible(dateDateType)); - assertFalse(DATE.isCompatible(timeDateType)); - assertFalse(DATE.isCompatible(datetimeDateType)); + assertAll( + // timestamp types is compatible with all date-types + () -> assertTrue(TIMESTAMP.isCompatible(defaultDateType)), + () -> assertTrue(TIMESTAMP.isCompatible(dateDateType)), + () -> assertTrue(TIMESTAMP.isCompatible(timeDateType)), + () -> assertTrue(TIMESTAMP.isCompatible(datetimeDateType)), + + // datetime + () -> assertFalse(DATETIME.isCompatible(defaultDateType)), + () -> assertTrue(DATETIME.isCompatible(dateDateType)), + () -> assertTrue(DATETIME.isCompatible(timeDateType)), + () -> assertFalse(DATETIME.isCompatible(datetimeDateType)), + + // time type + () -> assertFalse(TIME.isCompatible(defaultDateType)), + () -> assertFalse(TIME.isCompatible(dateDateType)), + () -> assertTrue(TIME.isCompatible(timeDateType)), + () -> assertFalse(TIME.isCompatible(datetimeDateType)), + + // date type + () -> assertFalse(DATE.isCompatible(defaultDateType)), + () -> assertTrue(DATE.isCompatible(dateDateType)), + () -> assertFalse(DATE.isCompatible(timeDateType)), + () -> assertFalse(DATE.isCompatible(datetimeDateType)) + ); } // `typeName` and `legacyTypeName` return the same thing for date objects: // https://github.com/opensearch-project/sql/issues/1296 @Test public void check_typeName() { - // always use the MappingType of "DATE" - assertEquals("DATE", defaultDateType.typeName()); - assertEquals("DATE", timeDateType.typeName()); - assertEquals("DATE", dateDateType.typeName()); - assertEquals("DATE", datetimeDateType.typeName()); + assertAll( + // always use the MappingType of "DATE" + () -> assertEquals("DATE", defaultDateType.typeName()), + () -> assertEquals("DATE", timeDateType.typeName()), + () -> assertEquals("DATE", dateDateType.typeName()), + () -> assertEquals("DATE", datetimeDateType.typeName()) + ); } @Test public void check_legacyTypeName() { - // always use the legacy "DATE" type - assertEquals("DATE", defaultDateType.legacyTypeName()); - assertEquals("DATE", timeDateType.legacyTypeName()); - assertEquals("DATE", dateDateType.legacyTypeName()); - assertEquals("DATE", datetimeDateType.legacyTypeName()); + assertAll( + // always use the legacy "DATE" type + () -> assertEquals("DATE", defaultDateType.legacyTypeName()), + () -> assertEquals("DATE", timeDateType.legacyTypeName()), + () -> assertEquals("DATE", dateDateType.legacyTypeName()), + () -> assertEquals("DATE", datetimeDateType.legacyTypeName()) + ); } @Test public void check_exprTypeName() { - // exprType changes based on type (no datetime): - assertEquals(TIMESTAMP, defaultDateType.getExprType()); - assertEquals(TIME, timeDateType.getExprType()); - assertEquals(DATE, dateDateType.getExprType()); - assertEquals(TIMESTAMP, datetimeDateType.getExprType()); + assertAll( + // exprType changes based on type (no datetime): + () -> assertEquals(TIMESTAMP, defaultDateType.getExprType()), + () -> assertEquals(TIME, timeDateType.getExprType()), + () -> assertEquals(DATE, dateDateType.getExprType()), + () -> assertEquals(TIMESTAMP, datetimeDateType.getExprType()) + ); } - @Test - public void checkSupportedFormatNamesCoverage() { - EnumSet allFormatNames = EnumSet.allOf(FormatNames.class); - allFormatNames.stream().forEach(formatName -> { - assertTrue( - SUPPORTED_NAMED_DATETIME_FORMATS.contains(formatName) - || SUPPORTED_NAMED_DATE_FORMATS.contains(formatName) - || SUPPORTED_NAMED_TIME_FORMATS.contains(formatName), - formatName + " not supported"); - }); + private static Stream getAllSupportedFormats() { + return EnumSet.allOf(FormatNames.class).stream().map(Arguments::of); } - @Test - public void checkTimestampFormatNames() { - SUPPORTED_NAMED_DATETIME_FORMATS.stream().forEach( - datetimeFormat -> { - String camelCaseName = datetimeFormat.getCamelCaseName(); - if (camelCaseName != null && !camelCaseName.isEmpty()) { - OpenSearchDateType dateType = - OpenSearchDateType.of(camelCaseName); - assertTrue(dateType.getExprType() == TIMESTAMP, camelCaseName - + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); - } - - String snakeCaseName = datetimeFormat.getSnakeCaseName(); - if (snakeCaseName != null && !snakeCaseName.isEmpty()) { - OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); - assertTrue(dateType.getExprType() == TIMESTAMP, snakeCaseName - + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); - } - } - ); + @ParameterizedTest + @MethodSource("getAllSupportedFormats") + public void check_supported_format_names_coverage(FormatNames formatName) { + assertTrue(SUPPORTED_NAMED_NUMERIC_FORMATS.contains(formatName) + || SUPPORTED_NAMED_DATETIME_FORMATS.contains(formatName) + || SUPPORTED_NAMED_DATE_FORMATS.contains(formatName) + || SUPPORTED_NAMED_TIME_FORMATS.contains(formatName) + || SUPPORTED_NAMED_INCOMPLETE_DATE_FORMATS.contains(formatName), + formatName + " not supported"); + } - // check the default format case - OpenSearchDateType dateType = OpenSearchDateType.of(""); - assertTrue(dateType.getExprType() == TIMESTAMP); + private static Stream getSupportedDatetimeFormats() { + return SUPPORTED_NAMED_DATETIME_FORMATS.stream().map(Arguments::of); } - @Test - public void checkDateFormatNames() { - SUPPORTED_NAMED_DATE_FORMATS.stream().forEach( - dateFormat -> { - String camelCaseName = dateFormat.getCamelCaseName(); - if (camelCaseName != null && !camelCaseName.isEmpty()) { - OpenSearchDateType dateType = - OpenSearchDateType.of(camelCaseName); - assertTrue(dateType.getExprType() == DATE, camelCaseName - + " does not format to a DATE type, instead got " - + dateType.getExprType()); - } - - String snakeCaseName = dateFormat.getSnakeCaseName(); - if (snakeCaseName != null && !snakeCaseName.isEmpty()) { - OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); - assertTrue(dateType.getExprType() == DATE, snakeCaseName - + " does not format to a DATE type, instead got " - + dateType.getExprType()); - } - } + @ParameterizedTest + @MethodSource("getSupportedDatetimeFormats") + public void check_datetime_format_names(FormatNames datetimeFormat) { + String camelCaseName = datetimeFormat.getCamelCaseName(); + if (camelCaseName != null && !camelCaseName.isEmpty()) { + OpenSearchDateType dateType = + OpenSearchDateType.of(camelCaseName); + assertSame(dateType.getExprType(), TIMESTAMP, camelCaseName + + " does not format to a TIMESTAMP type, instead got " + dateType.getExprType()); + } + + String snakeCaseName = datetimeFormat.getSnakeCaseName(); + if (snakeCaseName != null && !snakeCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); + assertSame(dateType.getExprType(), TIMESTAMP, snakeCaseName + + " does not format to a TIMESTAMP type, instead got " + dateType.getExprType()); + } else { + fail(); + } + } + + private static Stream getSupportedDateFormats() { + return SUPPORTED_NAMED_DATE_FORMATS.stream().map(Arguments::of); + } + + @ParameterizedTest + @MethodSource("getSupportedDateFormats") + public void check_date_format_names(FormatNames dateFormat) { + String camelCaseName = dateFormat.getCamelCaseName(); + if (camelCaseName != null && !camelCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); + assertSame(dateType.getExprType(), DATE, camelCaseName + + " does not format to a DATE type, instead got " + dateType.getExprType()); + } + + String snakeCaseName = dateFormat.getSnakeCaseName(); + if (snakeCaseName != null && !snakeCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); + assertSame(dateType.getExprType(), DATE, snakeCaseName + + " does not format to a DATE type, instead got " + dateType.getExprType()); + } else { + fail(); + } + } + + private static Stream getSupportedTimeFormats() { + return SUPPORTED_NAMED_TIME_FORMATS.stream().map(Arguments::of); + } + + @ParameterizedTest + @MethodSource("getSupportedTimeFormats") + public void check_time_format_names(FormatNames timeFormat) { + String camelCaseName = timeFormat.getCamelCaseName(); + if (camelCaseName != null && !camelCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); + assertSame(dateType.getExprType(), TIME, camelCaseName + + " does not format to a TIME type, instead got " + dateType.getExprType()); + } + + String snakeCaseName = timeFormat.getSnakeCaseName(); + if (snakeCaseName != null && !snakeCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); + assertSame(dateType.getExprType(), TIME, snakeCaseName + + " does not format to a TIME type, instead got " + dateType.getExprType()); + } else { + fail(); + } + } + + private static Stream get_format_combinations_for_test() { + return Stream.of( + Arguments.of(DATE, List.of("dd.MM.yyyy", "date"), "d && custom date"), + Arguments.of(TIME, List.of("time", "HH:mm"), "t && custom time"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "time"), "t && custom date"), + Arguments.of(TIMESTAMP, List.of("date", "HH:mm"), "d && custom time"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy HH:mm", "date_time"), "dt && custom datetime"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "date_time"), "dt && custom date"), + Arguments.of(TIMESTAMP, List.of("HH:mm", "date_time"), "dt && custom time"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "epoch_second"), "custom date && num"), + Arguments.of(TIMESTAMP, List.of("HH:mm", "epoch_second"), "custom time && num"), + Arguments.of(TIMESTAMP, List.of("date_time", "epoch_second"), "dt && num"), + Arguments.of(TIMESTAMP, List.of("date", "epoch_second"), "d && num"), + Arguments.of(TIMESTAMP, List.of("time", "epoch_second"), "t && num"), + Arguments.of(TIMESTAMP, List.of(""), "no formats given"), + Arguments.of(TIMESTAMP, List.of("time", "date"), "t && d"), + Arguments.of(TIMESTAMP, List.of("epoch_second"), "numeric"), + Arguments.of(TIME, List.of("time"), "t"), + Arguments.of(DATE, List.of("date"), "d"), + Arguments.of(TIMESTAMP, List.of("date_time"), "dt"), + Arguments.of(TIMESTAMP, List.of("unknown"), "unknown/incorrect"), + Arguments.of(DATE, List.of("uuuu"), "incomplete date"), + Arguments.of(TIME, List.of("HH"), "incomplete time"), + Arguments.of(DATE, List.of("E-w"), "incomplete"), + // E - day of week, w - week of year + Arguments.of(DATE, List.of("uuuu", "E-w"), "incomplete with year"), + Arguments.of(TIMESTAMP, List.of("---"), "incorrect"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "HH:mm"), "custom date and time"), + // D - day of year, N - nano of day + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy N", "uuuu:D:HH:mm"), "custom datetime"), + Arguments.of(DATE, List.of("dd.MM.yyyy", "uuuu:D"), "custom date"), + Arguments.of(TIME, List.of("HH:mm", "N"), "custom time") ); } + @ParameterizedTest(name = "[{index}] {2}") + @MethodSource("get_format_combinations_for_test") + public void check_ExprCoreType_of_combinations_of_custom_and_predefined_formats( + ExprCoreType expected, List formats, String testName) { + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + formats = Lists.reverse(formats); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + } + @Test - public void checkTimeFormatNames() { - SUPPORTED_NAMED_TIME_FORMATS.stream().forEach( - timeFormat -> { - String camelCaseName = timeFormat.getCamelCaseName(); - if (camelCaseName != null && !camelCaseName.isEmpty()) { - OpenSearchDateType dateType = - OpenSearchDateType.of(camelCaseName); - assertTrue(dateType.getExprType() == TIME, camelCaseName - + " does not format to a TIME type, instead got " - + dateType.getExprType()); - } - - String snakeCaseName = timeFormat.getSnakeCaseName(); - if (snakeCaseName != null && !snakeCaseName.isEmpty()) { - OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); - assertTrue(dateType.getExprType() == TIME, snakeCaseName - + " does not format to a TIME type, instead got " - + dateType.getExprType()); - } - } - ); + public void dont_use_incorrect_format_as_custom() { + assertEquals(0, OpenSearchDateType.of(" ").getAllCustomFormatters().size()); } @Test - public void checkIfDateTypeCompatible() { + public void check_if_date_type_compatible() { assertTrue(isDateTypeCompatible(DATE)); assertFalse(isDateTypeCompatible(OpenSearchDataType.of( OpenSearchDataType.MappingType.Text))); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index a7e3531e8b..827606a961 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; @@ -79,12 +80,17 @@ class OpenSearchExprValueFactoryTest { .put("dateStringV", OpenSearchDateType.of("date")) .put("timeStringV", OpenSearchDateType.of("time")) .put("epochMillisV", OpenSearchDateType.of("epoch_millis")) - .put("dateOrEpochMillisV", OpenSearchDateType.of("date_time_no_millis||epoch_millis")) - .put("timeNoMillisOrTimeV", OpenSearchDateType.of("time_no_millis||time")) - .put("dateOrOrdinalDateV", OpenSearchDateType.of("date||ordinal_date")) + .put("epochSecondV", OpenSearchDateType.of("epoch_second")) + .put("timeCustomV", OpenSearchDateType.of("HHmmss")) + .put("dateCustomV", OpenSearchDateType.of("uuuuMMdd")) + .put("dateTimeCustomV", OpenSearchDateType.of("uuuuMMddHHmmss")) + .put("dateOrEpochMillisV", OpenSearchDateType.of("date_time_no_millis || epoch_millis")) + .put("timeNoMillisOrTimeV", OpenSearchDateType.of("time_no_millis || time")) + .put("dateOrOrdinalDateV", OpenSearchDateType.of("date || ordinal_date")) .put("customFormatV", OpenSearchDateType.of("yyyy-MM-dd-HH-mm-ss")) .put("customAndEpochMillisV", - OpenSearchDateType.of("yyyy-MM-dd-HH-mm-ss||epoch_millis")) + OpenSearchDateType.of("yyyy-MM-dd-HH-mm-ss || epoch_millis")) + .put("incompleteFormatV", OpenSearchDateType.of("year")) .put("boolV", OpenSearchDataType.of(BOOLEAN)) .put("structV", OpenSearchDataType.of(STRUCT)) .put("structV.id", OpenSearchDataType.of(INTEGER)) @@ -116,26 +122,32 @@ class OpenSearchExprValueFactoryTest { @Test public void constructNullValue() { - assertEquals(nullValue(), tupleValue("{\"intV\":null}").get("intV")); - assertEquals(nullValue(), constructFromObject("intV", null)); - assertTrue(new OpenSearchJsonContent(null).isNull()); + assertAll( + () -> assertEquals(nullValue(), tupleValue("{\"intV\":null}").get("intV")), + () -> assertEquals(nullValue(), constructFromObject("intV", null)), + () -> assertTrue(new OpenSearchJsonContent(null).isNull()) + ); } @Test public void iterateArrayValue() throws JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); var arrayIt = new OpenSearchJsonContent(mapper.readTree("[\"zz\",\"bb\"]")).array(); - assertTrue(arrayIt.next().stringValue().equals("zz")); - assertTrue(arrayIt.next().stringValue().equals("bb")); - assertTrue(!arrayIt.hasNext()); + assertAll( + () -> assertEquals("zz", arrayIt.next().stringValue()), + () -> assertEquals("bb", arrayIt.next().stringValue()), + () -> assertFalse(arrayIt.hasNext()) + ); } @Test public void iterateArrayValueWithOneElement() throws JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); var arrayIt = new OpenSearchJsonContent(mapper.readTree("[\"zz\"]")).array(); - assertTrue(arrayIt.next().stringValue().equals("zz")); - assertTrue(!arrayIt.hasNext()); + assertAll( + () -> assertEquals("zz", arrayIt.next().stringValue()), + () -> assertFalse(arrayIt.hasNext()) + ); } @Test @@ -145,23 +157,29 @@ public void constructNullArrayValue() { @Test public void constructByte() { - assertEquals(byteValue((byte) 1), tupleValue("{\"byteV\":1}").get("byteV")); - assertEquals(byteValue((byte) 1), constructFromObject("byteV", 1)); - assertEquals(byteValue((byte) 1), constructFromObject("byteV", "1.0")); + assertAll( + () -> assertEquals(byteValue((byte) 1), tupleValue("{\"byteV\":1}").get("byteV")), + () -> assertEquals(byteValue((byte) 1), constructFromObject("byteV", 1)), + () -> assertEquals(byteValue((byte) 1), constructFromObject("byteV", "1.0")) + ); } @Test public void constructShort() { - assertEquals(shortValue((short) 1), tupleValue("{\"shortV\":1}").get("shortV")); - assertEquals(shortValue((short) 1), constructFromObject("shortV", 1)); - assertEquals(shortValue((short) 1), constructFromObject("shortV", "1.0")); + assertAll( + () -> assertEquals(shortValue((short) 1), tupleValue("{\"shortV\":1}").get("shortV")), + () -> assertEquals(shortValue((short) 1), constructFromObject("shortV", 1)), + () -> assertEquals(shortValue((short) 1), constructFromObject("shortV", "1.0")) + ); } @Test public void constructInteger() { - assertEquals(integerValue(1), tupleValue("{\"intV\":1}").get("intV")); - assertEquals(integerValue(1), constructFromObject("intV", 1)); - assertEquals(integerValue(1), constructFromObject("intV", "1.0")); + assertAll( + () -> assertEquals(integerValue(1), tupleValue("{\"intV\":1}").get("intV")), + () -> assertEquals(integerValue(1), constructFromObject("intV", 1)), + () -> assertEquals(integerValue(1), constructFromObject("intV", "1.0")) + ); } @Test @@ -171,168 +189,181 @@ public void constructIntegerValueInStringValue() { @Test public void constructLong() { - assertEquals(longValue(1L), tupleValue("{\"longV\":1}").get("longV")); - assertEquals(longValue(1L), constructFromObject("longV", 1L)); - assertEquals(longValue(1L), constructFromObject("longV", "1.0")); + assertAll( + () -> assertEquals(longValue(1L), tupleValue("{\"longV\":1}").get("longV")), + () -> assertEquals(longValue(1L), constructFromObject("longV", 1L)), + () -> assertEquals(longValue(1L), constructFromObject("longV", "1.0")) + ); } @Test public void constructFloat() { - assertEquals(floatValue(1f), tupleValue("{\"floatV\":1.0}").get("floatV")); - assertEquals(floatValue(1f), constructFromObject("floatV", 1f)); + assertAll( + () -> assertEquals(floatValue(1f), tupleValue("{\"floatV\":1.0}").get("floatV")), + () -> assertEquals(floatValue(1f), constructFromObject("floatV", 1f)) + ); } @Test public void constructDouble() { - assertEquals(doubleValue(1d), tupleValue("{\"doubleV\":1.0}").get("doubleV")); - assertEquals(doubleValue(1d), constructFromObject("doubleV", 1d)); + assertAll( + () -> assertEquals(doubleValue(1d), tupleValue("{\"doubleV\":1.0}").get("doubleV")), + () -> assertEquals(doubleValue(1d), constructFromObject("doubleV", 1d)) + ); } @Test public void constructString() { - assertEquals(stringValue("text"), tupleValue("{\"stringV\":\"text\"}").get("stringV")); - assertEquals(stringValue("text"), constructFromObject("stringV", "text")); + assertAll( + () -> assertEquals(stringValue("text"), + tupleValue("{\"stringV\":\"text\"}").get("stringV")), + () -> assertEquals(stringValue("text"), constructFromObject("stringV", "text")) + ); } @Test public void constructBoolean() { - assertEquals(booleanValue(true), tupleValue("{\"boolV\":true}").get("boolV")); - assertEquals(booleanValue(true), constructFromObject("boolV", true)); - assertEquals(booleanValue(true), constructFromObject("boolV", "true")); - assertEquals(booleanValue(true), constructFromObject("boolV", 1)); - assertEquals(booleanValue(false), constructFromObject("boolV", 0)); + assertAll( + () -> assertEquals(booleanValue(true), tupleValue("{\"boolV\":true}").get("boolV")), + () -> assertEquals(booleanValue(true), constructFromObject("boolV", true)), + () -> assertEquals(booleanValue(true), constructFromObject("boolV", "true")), + () -> assertEquals(booleanValue(true), constructFromObject("boolV", 1)), + () -> assertEquals(booleanValue(false), constructFromObject("boolV", 0)) + ); } @Test public void constructText() { - assertEquals(new OpenSearchExprTextValue("text"), - tupleValue("{\"textV\":\"text\"}").get("textV")); - assertEquals(new OpenSearchExprTextValue("text"), - constructFromObject("textV", "text")); - - assertEquals(new OpenSearchExprTextValue("text"), - tupleValue("{\"textKeywordV\":\"text\"}").get("textKeywordV")); - assertEquals(new OpenSearchExprTextValue("text"), - constructFromObject("textKeywordV", "text")); + assertAll( + () -> assertEquals(new OpenSearchExprTextValue("text"), + tupleValue("{\"textV\":\"text\"}").get("textV")), + () -> assertEquals(new OpenSearchExprTextValue("text"), + constructFromObject("textV", "text")), + + () -> assertEquals(new OpenSearchExprTextValue("text"), + tupleValue("{\"textKeywordV\":\"text\"}").get("textKeywordV")), + () -> assertEquals(new OpenSearchExprTextValue("text"), + constructFromObject("textKeywordV", "text")) + ); } @Test public void constructDates() { ExprValue dateStringV = constructFromObject("dateStringV", "1984-04-12"); - assertEquals(new ExprDateValue("1984-04-12"), dateStringV); - - assertEquals( - new ExprDateValue(LocalDate.ofInstant(Instant.ofEpochMilli(450576000000L), - UTC_ZONE_ID)), - constructFromObject("dateV", 450576000000L)); - - assertEquals( - new ExprDateValue("1984-04-12"), - constructFromObject("dateOrOrdinalDateV", "1984-103")); - assertEquals( - new ExprDateValue("2015-01-01"), - tupleValue("{\"dateV\":\"2015-01-01\"}").get("dateV")); + assertAll( + () -> assertEquals(new ExprDateValue("1984-04-12"), dateStringV), + () -> assertEquals(new ExprDateValue( + LocalDate.ofInstant(Instant.ofEpochMilli(450576000000L), UTC_ZONE_ID)), + constructFromObject("dateV", 450576000000L)), + () -> assertEquals(new ExprDateValue("1984-04-12"), + constructFromObject("dateOrOrdinalDateV", "1984-103")), + () -> assertEquals(new ExprDateValue("2015-01-01"), + tupleValue("{\"dateV\":\"2015-01-01\"}").get("dateV")) + ); } @Test public void constructTimes() { ExprValue timeStringV = constructFromObject("timeStringV","12:10:30.000Z"); - assertTrue(timeStringV.isDateTime()); - assertTrue(timeStringV instanceof ExprTimeValue); - assertEquals(new ExprTimeValue("12:10:30"), timeStringV); - - assertEquals( - new ExprTimeValue(LocalTime.from(Instant.ofEpochMilli(1420070400001L).atZone(UTC_ZONE_ID))), - constructFromObject("timeV", 1420070400001L)); - assertEquals( - new ExprTimeValue("09:07:42.000"), - constructFromObject("timeNoMillisOrTimeV", "09:07:42.000Z")); - assertEquals( - new ExprTimeValue("09:07:42"), - tupleValue("{\"timeV\":\"09:07:42\"}").get("timeV")); + assertAll( + () -> assertTrue(timeStringV.isDateTime()), + () -> assertTrue(timeStringV instanceof ExprTimeValue), + () -> assertEquals(new ExprTimeValue("12:10:30"), timeStringV), + () -> assertEquals(new ExprTimeValue(LocalTime.from( + Instant.ofEpochMilli(1420070400001L).atZone(UTC_ZONE_ID))), + constructFromObject("timeV", 1420070400001L)), + () -> assertEquals(new ExprTimeValue("09:07:42.000"), + constructFromObject("timeNoMillisOrTimeV", "09:07:42.000Z")), + () -> assertEquals(new ExprTimeValue("09:07:42"), + tupleValue("{\"timeV\":\"09:07:42\"}").get("timeV")) + ); } @Test public void constructDatetime() { - assertEquals( - new ExprTimestampValue("2015-01-01 00:00:00"), - tupleValue("{\"timestampV\":\"2015-01-01\"}").get("timestampV")); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - tupleValue("{\"timestampV\":\"2015-01-01T12:10:30Z\"}").get("timestampV")); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - tupleValue("{\"timestampV\":\"2015-01-01T12:10:30\"}").get("timestampV")); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - tupleValue("{\"timestampV\":1420070400001}").get("timestampV")); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("timestampV", 1420070400001L)); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("timestampV", Instant.ofEpochMilli(1420070400001L))); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("epochMillisV", "1420070400001")); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("epochMillisV", 1420070400001L)); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - constructFromObject("timestampV", "2015-01-01 12:10:30")); - assertEquals( - new ExprDatetimeValue("2015-01-01 12:10:30"), - constructFromObject("datetimeV", "2015-01-01 12:10:30")); - assertEquals( - new ExprDatetimeValue("2015-01-01 12:10:30"), - constructFromObject("datetimeDefaultV", "2015-01-01 12:10:30")); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("dateOrEpochMillisV", "1420070400001")); - - // case: timestamp-formatted field, but it only gets a time: should match a time - assertEquals( - new ExprTimeValue("19:36:22"), - tupleValue("{\"timestampV\":\"19:36:22\"}").get("timestampV")); - - // case: timestamp-formatted field, but it only gets a date: should match a date - assertEquals( - new ExprDateValue("2011-03-03"), - tupleValue("{\"timestampV\":\"2011-03-03\"}").get("timestampV")); + assertAll( + () -> assertEquals( + new ExprTimestampValue("2015-01-01 00:00:00"), + tupleValue("{\"timestampV\":\"2015-01-01\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue("2015-01-01 12:10:30"), + tupleValue("{\"timestampV\":\"2015-01-01T12:10:30Z\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue("2015-01-01 12:10:30"), + tupleValue("{\"timestampV\":\"2015-01-01T12:10:30\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue("2015-01-01 12:10:30"), + tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("timestampV", 1420070400001L)), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("timestampV", Instant.ofEpochMilli(1420070400001L))), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("epochMillisV", "1420070400001")), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("epochMillisV", 1420070400001L)), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochSecond(142704001L)), + constructFromObject("epochSecondV", 142704001L)), + () -> assertEquals( + new ExprTimeValue("10:20:30"), + tupleValue("{ \"timeCustomV\" : 102030 }").get("timeCustomV")), + () -> assertEquals( + new ExprDateValue("1961-04-12"), + tupleValue("{ \"dateCustomV\" : 19610412 }").get("dateCustomV")), + () -> assertEquals( + new ExprTimestampValue("1984-05-10 20:30:40"), + tupleValue("{ \"dateTimeCustomV\" : 19840510203040 }").get("dateTimeCustomV")), + () -> assertEquals( + new ExprTimestampValue("2015-01-01 12:10:30"), + constructFromObject("timestampV", "2015-01-01 12:10:30")), + () -> assertEquals( + new ExprDatetimeValue("2015-01-01 12:10:30"), + constructFromObject("datetimeV", "2015-01-01 12:10:30")), + () -> assertEquals( + new ExprDatetimeValue("2015-01-01 12:10:30"), + constructFromObject("datetimeDefaultV", "2015-01-01 12:10:30")), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("dateOrEpochMillisV", "1420070400001")), + + // case: timestamp-formatted field, but it only gets a time: should match a time + () -> assertEquals( + new ExprTimeValue("19:36:22"), + tupleValue("{\"timestampV\":\"19:36:22\"}").get("timestampV")), + + // case: timestamp-formatted field, but it only gets a date: should match a date + () -> assertEquals( + new ExprDateValue("2011-03-03"), + tupleValue("{\"timestampV\":\"2011-03-03\"}").get("timestampV")) + ); } @Test public void constructDatetime_fromCustomFormat() { - // this is not the desirable behaviour - instead if accepts the default formatter assertEquals( new ExprDatetimeValue("2015-01-01 12:10:30"), - constructFromObject("customFormatV", "2015-01-01 12:10:30")); + constructFromObject("customFormatV", "2015-01-01-12-10-30")); - // this should pass when custom formats are supported IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> constructFromObject("customFormatV", "2015-01-01-12-10-30")); + () -> constructFromObject("customFormatV", "2015-01-01 12-10-30")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01-12-10-30\" failed, " - + "unsupported date format.", + "Construct TIMESTAMP from \"2015-01-01 12-10-30\" failed, " + + "unsupported format.", exception.getMessage()); assertEquals( new ExprDatetimeValue("2015-01-01 12:10:30"), constructFromObject("customAndEpochMillisV", "2015-01-01 12:10:30")); - // this should pass when custom formats are supported - exception = - assertThrows(IllegalArgumentException.class, - () -> constructFromObject("customAndEpochMillisV", "2015-01-01-12-10-30")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01-12-10-30\" failed, " - + "unsupported date format.", - exception.getMessage()); + new ExprDatetimeValue("2015-01-01 12:10:30"), + constructFromObject("customAndEpochMillisV", "2015-01-01-12-10-30")); } @Test @@ -341,8 +372,8 @@ public void constructDatetimeFromUnsupportedFormat_ThrowIllegalArgumentException assertThrows(IllegalArgumentException.class, () -> constructFromObject("timestampV", "2015-01-01 12:10")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01 12:10\" failed, " - + "unsupported date format.", + "Construct TIMESTAMP from \"2015-01-01 12:10\" failed, " + + "unsupported format.", exception.getMessage()); // fail with missing seconds @@ -350,8 +381,8 @@ public void constructDatetimeFromUnsupportedFormat_ThrowIllegalArgumentException assertThrows(IllegalArgumentException.class, () -> constructFromObject("dateOrEpochMillisV", "2015-01-01 12:10")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01 12:10\" failed, " - + "unsupported date format.", + "Construct TIMESTAMP from \"2015-01-01 12:10\" failed, " + + "unsupported format.", exception.getMessage()); } @@ -360,15 +391,15 @@ public void constructTimeFromUnsupportedFormat_ThrowIllegalArgumentException() { IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, () -> constructFromObject("timeV", "2015-01-01")); assertEquals( - "Construct ExprTimeValue from \"2015-01-01\" failed, " - + "unsupported time format.", + "Construct TIME from \"2015-01-01\" failed, " + + "unsupported format.", exception.getMessage()); exception = assertThrows( IllegalArgumentException.class, () -> constructFromObject("timeStringV", "10:10")); assertEquals( - "Construct ExprTimeValue from \"10:10\" failed, " - + "unsupported time format.", + "Construct TIME from \"10:10\" failed, " + + "unsupported format.", exception.getMessage()); } @@ -377,18 +408,25 @@ public void constructDateFromUnsupportedFormat_ThrowIllegalArgumentException() { IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, () -> constructFromObject("dateV", "12:10:10")); assertEquals( - "Construct ExprDateValue from \"12:10:10\" failed, " - + "unsupported date format.", + "Construct DATE from \"12:10:10\" failed, " + + "unsupported format.", exception.getMessage()); exception = assertThrows( IllegalArgumentException.class, () -> constructFromObject("dateStringV", "abc")); assertEquals( - "Construct ExprDateValue from \"abc\" failed, " - + "unsupported date format.", + "Construct DATE from \"abc\" failed, " + + "unsupported format.", exception.getMessage()); } + @Test + public void constructDateFromIncompleteFormat() { + assertEquals( + new ExprDateValue("1984-01-01"), + constructFromObject("incompleteFormatV", "1984")); + } + @Test public void constructArray() { assertEquals(