diff --git a/src/main/java/com/ly/ckibana/model/exception/UnKnowFieldException.java b/src/main/java/com/ly/ckibana/model/exception/UnKnownFieldException.java similarity index 88% rename from src/main/java/com/ly/ckibana/model/exception/UnKnowFieldException.java rename to src/main/java/com/ly/ckibana/model/exception/UnKnownFieldException.java index 2de9f0b..f444276 100644 --- a/src/main/java/com/ly/ckibana/model/exception/UnKnowFieldException.java +++ b/src/main/java/com/ly/ckibana/model/exception/UnKnownFieldException.java @@ -15,9 +15,9 @@ */ package com.ly.ckibana.model.exception; -public class UnKnowFieldException extends UiException { +public class UnKnownFieldException extends UiException { - public UnKnowFieldException(String fieldName) { + public UnKnownFieldException(String fieldName) { super(fieldName); } diff --git a/src/main/java/com/ly/ckibana/parser/MsearchParamParser.java b/src/main/java/com/ly/ckibana/parser/MsearchParamParser.java index f42b69d..9f69dab 100644 --- a/src/main/java/com/ly/ckibana/parser/MsearchParamParser.java +++ b/src/main/java/com/ly/ckibana/parser/MsearchParamParser.java @@ -24,6 +24,7 @@ import com.ly.ckibana.model.compute.indexpattern.IndexPattern; import com.ly.ckibana.model.enums.SortType; import com.ly.ckibana.model.exception.TimeNotInRangeException; +import com.ly.ckibana.model.exception.UiException; import com.ly.ckibana.model.exception.UnKnowTimeFieldException; import com.ly.ckibana.model.property.KibanaItemProperty; import com.ly.ckibana.model.property.QueryProperty; @@ -42,6 +43,7 @@ import com.ly.ckibana.util.ProxyUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import javax.annotation.Resource; @@ -72,7 +74,7 @@ public class MsearchParamParser extends ParamParser { public void checkTimeInRange(CkRequestContext ckRequestContext) { if (!isTimeInRange(ckRequestContext)) { throw new TimeNotInRangeException("查询时间跨度太大,目前支持最大查询区间为:" - + DateUtils.formatDurationWords(proxyConfigLoader.getKibanaProperty().getProxy().getMaxTimeRange())); + + DateUtils.formatDurationWords(proxyConfigLoader.getKibanaProperty().getProxy().getMaxTimeRange())); } } @@ -234,9 +236,11 @@ public void parse(RequestContext context, String defaultIndex, StringBuilder sub // 将查询任务放入线程队列 MsearchQueryTask msearchQueryTask = new MsearchQueryTask(ckRequestContext, aggResultParser, searchQuery); subCkRequests.add(msearchQueryTask); + } catch (UiException uiException) { + fastFailResponses.add(ProxyUtils.newKibanaException(HttpStatus.BAD_REQUEST, uiException.getUiShow())); } catch (Exception ex) { log.error("msearch param parse error, i:{}, uiIndex:{}, searchQuery:{}", i, uiIndex, searchQuery, ex); - fastFailResponses.add(ProxyUtils.newKibanaException(ex.getMessage())); + fastFailResponses.add(ProxyUtils.newKibanaException(HttpStatus.BAD_REQUEST, ex.getMessage())); } } diff --git a/src/main/java/com/ly/ckibana/service/CkService.java b/src/main/java/com/ly/ckibana/service/CkService.java index 088f978..bf23c28 100644 --- a/src/main/java/com/ly/ckibana/service/CkService.java +++ b/src/main/java/com/ly/ckibana/service/CkService.java @@ -24,7 +24,7 @@ import com.ly.ckibana.model.exception.DataSourceEmptyException; import com.ly.ckibana.model.exception.ResourceExceedException; import com.ly.ckibana.model.exception.TooManySimultaneousException; -import com.ly.ckibana.model.exception.UnKnowFieldException; +import com.ly.ckibana.model.exception.UnKnownFieldException; import com.ly.ckibana.model.property.CkProperty; import com.ly.ckibana.model.request.CkRequestContext; import com.ly.ckibana.model.request.ProxyConfig; @@ -264,7 +264,7 @@ private ResultSet query(ClickHouseConnection connection, String sql) throws Exce String field = ex.getCause().getMessage().split("while processing query")[0]; field = field.replace("Code: 47, e.displayText() = DB::Exception: Missing columns:", "") .replace("'", "").replace(" ", ""); - throw new UnKnowFieldException(field); + throw new UnKnownFieldException(field); } else if (ex.getErrorCode() == ClickHouseErrorCode.UNKNOWN_TABLE.code) { throw new CKNotSupportException(ex.getMessage()); } else { diff --git a/src/main/java/com/ly/ckibana/util/ParamConvertUtils.java b/src/main/java/com/ly/ckibana/util/ParamConvertUtils.java index e4eafbf..89e0a4d 100644 --- a/src/main/java/com/ly/ckibana/util/ParamConvertUtils.java +++ b/src/main/java/com/ly/ckibana/util/ParamConvertUtils.java @@ -15,7 +15,7 @@ */ package com.ly.ckibana.util; -import com.ly.ckibana.constants.Constants; +import com.ly.ckibana.model.exception.UnKnownFieldException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -40,9 +40,9 @@ public static String convertUiFieldToCkField(Map columns, String if (!columns.containsKey(result)) { result = StringUtils.trim(result); } - // 扩展动态字段查询 - if (!columns.containsKey(result) && columns.containsKey(Constants.CK_EXTENSION)) { - result = String.format("%s(%s,'%s')", Constants.CK_EXTENSION_QUERY_FUNCTION, Constants.CK_EXTENSION, result); + // 未定义字段不支持查询 + if (!columns.containsKey(result)) { + throw new UnKnownFieldException(result); } return result; } diff --git a/src/main/java/com/ly/ckibana/util/ProxyUtils.java b/src/main/java/com/ly/ckibana/util/ProxyUtils.java index dc9485a..a574c35 100644 --- a/src/main/java/com/ly/ckibana/util/ProxyUtils.java +++ b/src/main/java/com/ly/ckibana/util/ProxyUtils.java @@ -28,6 +28,7 @@ import com.ly.ckibana.model.response.Shards; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.http.HttpStatus; import java.util.Arrays; import java.util.HashMap; @@ -195,9 +196,11 @@ public static boolean isString(String type) { public static boolean isDate(String type) { return parseCkBaseType(type).startsWith("Date"); } - + /** * 构建kibana exception. + * @param error 异常说明 + * @return */ public static Response newKibanaException(String error) { Map searchError = new HashMap<>(2, 1); @@ -223,6 +226,24 @@ public static Map newKibanaExceptionV8(String error) { )); } + /** + * 构建kibana exception. + * @param httpStatus 状态码 + * @param error 异常说明 + * @return + */ + public static Response newKibanaException(HttpStatus httpStatus,String error) { + Map searchError = new HashMap<>(2, 1); + searchError.put("name", "SearchError"); + searchError.put("message", error); + Response result = new Response(); + result.setStatus(httpStatus.value()); + result.setAggregations(null); + result.setHits(null); + result.setShards(null); + result.setError(searchError); + return result; + } public static String getErrorResponse(Exception e) { return getErrorResponse(e.getMessage()); } diff --git a/src/test/java/com/ly/ckibana/CommonTest.java b/src/test/java/com/ly/ckibana/CommonTest.java index 7dec779..a0b8746 100644 --- a/src/test/java/com/ly/ckibana/CommonTest.java +++ b/src/test/java/com/ly/ckibana/CommonTest.java @@ -64,6 +64,7 @@ public class CommonTest { private static final String indexPatternJson = "{\"uiIndex\":\"table1_all\",\"cluster\":\"\",\"index\":\"table1_all\",\"database\":\"testdb\",\"timeField\":\"@timestampDateTime\",\"needSample\":false}"; //kibana代理配置信息 private static final String kibanaPropertyJson = "{\"defaultShard\":2,\"majorVersion\":6,\"proxy\":{\"ck\":{\"defaultCkDatabase\":\"testdb\",\"pass\":\"fc/3EtAe\",\"url\":\"10.177.43.183:6321\",\"user\":\"limited\"},\"es\":{\"headers\":{\"stoken\":\"b7842e1285e1d277e1730c41\"},\"host\":\"10.100.218.58:30691,10.100.218.60:30691,10.100.218.61:30691,10.100.218.62:30691,10.100.218.63:30691,10.100.218.64:30691\"},\"maxTimeRange\":86400000,\"roundAbleMinPeriod\":120000,\"whiteIndexList\":[\"table1_all\"]},\"query\":{\"sampleCountMaxThreshold\":1500000,\"useCache\":false,\"maxResultRow\":30000},\"threadPool\":{\"msearchProperty\":{\"coreSize\":100,\"queueSize\":10000}},\"yaml\":{\"name\":\"Yaml:1569371800\"}}"; + private static final String DO_TEST = "doTest"; @Resource private MsearchParamParser msearchParamParser; @Resource @@ -90,7 +91,7 @@ public void testCommon() { } /** - * 基于查询参数，转换得到sql列表，并于期望值比对。比对一致则通过，不一致则失败 + * 基于查询参数，转换得到sql列表，并与期望值比对。比对一致则通过，不一致则失败 * * @param searchQueryJson 参数 * @param needQueryHits 是否需要额外解析hits明细查询sql @@ -103,10 +104,32 @@ public void doTest(String testName, String searchQueryJson, Boolean needQueryHit List resultSqlList = convert2SqlList(searchQueryJson, needQueryHits); assertResult(testName, resultSqlList, expectedSqlList); } catch (Exception e) { - log.error("doTest", e); + log.error(DO_TEST, e); Assert.assertTrue(Boolean.FALSE); } + } + /** + * 基于查询参数，解析获取异常。并与期望异常比对。比对一致通过，不一致则失败 + * + * @param testName + * @param searchQueryJson + * @param expectException + */ + public void doTest(String testName, String searchQueryJson, Boolean needQueryHits,Exception expectException) { + try { + proxyConfigLoader.setKibanaProperty(JSONObject.parseObject(kibanaPropertyJson, KibanaProperty.class)); + convert2SqlList(searchQueryJson, needQueryHits); + log.error(DO_TEST, String.format("%s期望异常%s，实际无异常", testName, expectException)); + Assert.assertTrue(Boolean.FALSE); + } catch (Exception e) { + if (e.toString().equals(expectException.toString())) { + Assert.assertTrue(Boolean.TRUE); + } else { + log.error(DO_TEST, String.format("%s期望异常%s，实际异常为%s", testName, expectException, e)); + Assert.assertTrue(Boolean.FALSE); + } + } } /** @@ -186,7 +209,7 @@ public CkRequestContext parseRequest(String searchQueryJson) throws Exception { Map> tableColumnsCache = JSONObject.parseObject(tableColumnsCacheJson, Map.class); JSONObject searchQuery = JSONObject.parseObject(searchQueryJson); IndexPattern indexPattern = JSONObject.parseObject(indexPatternJson, IndexPattern.class); - CkRequestContext ckRequestContext = new CkRequestContext("testIp", indexPattern,proxyConfigLoader.getKibanaProperty().getQuery().getMaxResultRow()); + CkRequestContext ckRequestContext = new CkRequestContext("testIp", indexPattern, proxyConfigLoader.getKibanaProperty().getQuery().getMaxResultRow()); QueryProperty queryProperty = proxyConfigLoader.getKibanaProperty().getQuery(); CkRequestContext.SampleParam sampleParam = new CkRequestContext.SampleParam(Constants.USE_SAMPLE_COUNT_THREASHOLD, queryProperty.getSampleCountMaxThreshold()); ckRequestContext.setSampleParam(sampleParam); diff --git a/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java b/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java index fae2638..3465d17 100644 --- a/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java +++ b/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java @@ -16,6 +16,7 @@ package com.ly.ckibana.converter; import com.ly.ckibana.CommonTest; +import com.ly.ckibana.model.exception.UnKnownFieldException; import org.junit.Test; /** @@ -28,6 +29,8 @@ public class CommonDSLTest extends CommonTest { public static final String TEST_INTEGER_DSL = "INTEGER_DSL"; public static final String TEST_STRING_DSL = "STRING_DSL"; public static final String TEST_DATETIME64_DSL = "DATETIME64_DSL"; + public static final String TEST_UNKNOWN_FIELD_QUERY = "unknownFiledQuery"; + /** * 整型DSL查询测试 @@ -83,4 +86,14 @@ public void testIpv4OrIpv6DSL() { doTest(TEST_DATETIME64_DSL, query, Boolean.TRUE, expectedSqls); } + + /** + * DSL-unknownField Query + * unknowField:"value" + */ + @Test + public void testUnknownFiledQuery(){ + String query = "{\"version\":true,\"size\":500,\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"_source\":{\"excludes\":[]},\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTimeOnly\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"},{\"field\":\"@timestampDateTimeOnly\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_phrase\":{\"unknowField\":{\"query\":\"value\"}}},{\"range\":{\"@timestampDateTimeOnly\":{\"gte\":1712659236502,\"lte\":1712660136502,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"timeout\":\"300000ms\"}"; + doTest(TEST_UNKNOWN_FIELD_QUERY, query, Boolean.FALSE,new UnKnownFieldException("unknowField")); + } } diff --git a/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java b/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java index d8fc86a..5366ec5 100644 --- a/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java +++ b/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java @@ -16,6 +16,7 @@ package com.ly.ckibana.converter; import com.ly.ckibana.CommonTest; +import com.ly.ckibana.model.exception.UnKnownFieldException; import org.junit.Test; /** @@ -33,6 +34,8 @@ public class QueryStringClauseTest extends CommonTest { public static final String TEST_IP_TYPE_IPV4 = "IP_TYPE_IPV4"; public static final String TEST_IP_TYPE_IPV6 = "IP_TYPE_IPV6"; public static final String TEST_NUMBER = "NUMBER"; + public static final String TEST_UNKNOWN_FIELD_QUERY = "unknownFiledQuery"; + /** @@ -146,4 +149,14 @@ public void testNumberField() { " ]"; doTest(TEST_NUMBER, query, Boolean.FALSE, expectedSqls); } + + /** + * queryString-unknownField Query + * unknowField:"value" + */ + @Test + public void testUnknownFiledQuery(){ + String query = "{\"version\":true,\"size\":500,\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"_source\":{\"excludes\":[]},\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTimeOnly\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"},{\"field\":\"@timestampDateTimeOnly\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"unknownFiled:\\\"value\\\"\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTimeOnly\":{\"gte\":1712659440994,\"lte\":1712660340994,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"timeout\":\"300000ms\"}"; + doTest(TEST_UNKNOWN_FIELD_QUERY, query, Boolean.FALSE,new UnKnownFieldException("unknownFiled")); + } }