Merge pull request #33 from linger000/zl11357

feat:禁止未定义字段作为查询条件查询, 修复解析异常未展示到kibana Ui问题，并补充单例 #31
TongchengOpenSource · Apr 9, 2024 · 1bb5570 · 1bb5570
2 parents b8a0470 + 6d735b5
commit 1bb5570
Show file tree

Hide file tree

Showing 8 changed files with 88 additions and 14 deletions.
diff --git a/...model/exception/UnKnowFieldException.java → ...odel/exception/UnKnownFieldException.java b/...model/exception/UnKnowFieldException.java → ...odel/exception/UnKnownFieldException.java
@@ -15,9 +15,9 @@
  */
 package com.ly.ckibana.model.exception;
 
-public class UnKnowFieldException extends UiException {
+public class UnKnownFieldException extends UiException {
 
-    public UnKnowFieldException(String fieldName) {
+    public UnKnownFieldException(String fieldName) {
         super(fieldName);
     }
 

diff --git a/src/main/java/com/ly/ckibana/parser/MsearchParamParser.java b/src/main/java/com/ly/ckibana/parser/MsearchParamParser.java
@@ -24,6 +24,7 @@
 import com.ly.ckibana.model.compute.indexpattern.IndexPattern;
 import com.ly.ckibana.model.enums.SortType;
 import com.ly.ckibana.model.exception.TimeNotInRangeException;
+import com.ly.ckibana.model.exception.UiException;
 import com.ly.ckibana.model.exception.UnKnowTimeFieldException;
 import com.ly.ckibana.model.property.KibanaItemProperty;
 import com.ly.ckibana.model.property.QueryProperty;
@@ -42,6 +43,7 @@
 import com.ly.ckibana.util.ProxyUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
@@ -72,7 +74,7 @@ public class MsearchParamParser extends ParamParser {
     public void checkTimeInRange(CkRequestContext ckRequestContext) {
         if (!isTimeInRange(ckRequestContext)) {
             throw new TimeNotInRangeException("查询时间跨度太大,目前支持最大查询区间为:"
-                                              + DateUtils.formatDurationWords(proxyConfigLoader.getKibanaProperty().getProxy().getMaxTimeRange()));
+                    + DateUtils.formatDurationWords(proxyConfigLoader.getKibanaProperty().getProxy().getMaxTimeRange()));
         }
     }
 
@@ -234,9 +236,11 @@ public void parse(RequestContext context, String defaultIndex, StringBuilder sub
                 // 将查询任务放入线程队列
                 MsearchQueryTask msearchQueryTask = new MsearchQueryTask(ckRequestContext, aggResultParser, searchQuery);
                 subCkRequests.add(msearchQueryTask);
+            } catch (UiException uiException) {
+                fastFailResponses.add(ProxyUtils.newKibanaException(HttpStatus.BAD_REQUEST, uiException.getUiShow()));
             } catch (Exception ex) {
                 log.error("msearch param parse error, i:{}, uiIndex:{}, searchQuery:{}", i, uiIndex, searchQuery, ex);
-                fastFailResponses.add(ProxyUtils.newKibanaException(ex.getMessage()));
+                fastFailResponses.add(ProxyUtils.newKibanaException(HttpStatus.BAD_REQUEST, ex.getMessage()));
             }
         }
 

diff --git a/src/main/java/com/ly/ckibana/service/CkService.java b/src/main/java/com/ly/ckibana/service/CkService.java
@@ -24,7 +24,7 @@
 import com.ly.ckibana.model.exception.DataSourceEmptyException;
 import com.ly.ckibana.model.exception.ResourceExceedException;
 import com.ly.ckibana.model.exception.TooManySimultaneousException;
-import com.ly.ckibana.model.exception.UnKnowFieldException;
+import com.ly.ckibana.model.exception.UnKnownFieldException;
 import com.ly.ckibana.model.property.CkProperty;
 import com.ly.ckibana.model.request.CkRequestContext;
 import com.ly.ckibana.model.request.ProxyConfig;
@@ -264,7 +264,7 @@ private ResultSet query(ClickHouseConnection connection, String sql) throws Exce
                 String field = ex.getCause().getMessage().split("while processing query")[0];
                 field = field.replace("Code: 47, e.displayText() = DB::Exception: Missing columns:", "")
                         .replace("'", "").replace(" ", "");
-                throw new UnKnowFieldException(field);
+                throw new UnKnownFieldException(field);
             } else if (ex.getErrorCode() == ClickHouseErrorCode.UNKNOWN_TABLE.code) {
                 throw new CKNotSupportException(ex.getMessage());
             } else {

diff --git a/src/main/java/com/ly/ckibana/util/ParamConvertUtils.java b/src/main/java/com/ly/ckibana/util/ParamConvertUtils.java
@@ -15,7 +15,7 @@
  */
 package com.ly.ckibana.util;
 
-import com.ly.ckibana.constants.Constants;
+import com.ly.ckibana.model.exception.UnKnownFieldException;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 
@@ -40,9 +40,9 @@ public static String convertUiFieldToCkField(Map<String, String> columns, String
         if (!columns.containsKey(result)) {
             result = StringUtils.trim(result);
         }
-        // 扩展动态字段查询
-        if (!columns.containsKey(result) && columns.containsKey(Constants.CK_EXTENSION)) {
-            result = String.format("%s(%s,'%s')", Constants.CK_EXTENSION_QUERY_FUNCTION, Constants.CK_EXTENSION, result);
+        // 未定义字段不支持查询
+        if (!columns.containsKey(result)) {
+            throw new UnKnownFieldException(result);
         }
         return result;
     }

diff --git a/src/main/java/com/ly/ckibana/util/ProxyUtils.java b/src/main/java/com/ly/ckibana/util/ProxyUtils.java
@@ -28,6 +28,7 @@
 import com.ly.ckibana.model.response.Shards;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.http.HttpStatus;
 
 import java.util.Arrays;
 import java.util.HashMap;
@@ -195,9 +196,11 @@ public static boolean isString(String type) {
     public static boolean isDate(String type) {
         return parseCkBaseType(type).startsWith("Date");
     }
-
+    
     /**
      * 构建kibana exception.
+     * @param error 异常说明
+     * @return
      */
     public static Response newKibanaException(String error) {
         Map<String, Object> searchError = new HashMap<>(2, 1);
@@ -223,6 +226,24 @@ public static Map<String, Object> newKibanaExceptionV8(String error) {
                 ));
     }
 
+    /**
+     * 构建kibana exception.
+     * @param httpStatus 状态码
+     * @param error 异常说明
+     * @return
+     */
+    public static Response newKibanaException(HttpStatus httpStatus,String error) {
+        Map<String, Object> searchError = new HashMap<>(2, 1);
+        searchError.put("name", "SearchError");
+        searchError.put("message", error);
+        Response result = new Response();
+        result.setStatus(httpStatus.value());
+        result.setAggregations(null);
+        result.setHits(null);
+        result.setShards(null);
+        result.setError(searchError);
+        return result;
+    }
     public static String getErrorResponse(Exception e) {
         return getErrorResponse(e.getMessage());
     }

diff --git a/src/test/java/com/ly/ckibana/CommonTest.java b/src/test/java/com/ly/ckibana/CommonTest.java
@@ -64,6 +64,7 @@ public class CommonTest {
     private static final String indexPatternJson = "{\"uiIndex\":\"table1_all\",\"cluster\":\"\",\"index\":\"table1_all\",\"database\":\"testdb\",\"timeField\":\"@timestampDateTime\",\"needSample\":false}";
     //kibana代理配置信息
     private static final String kibanaPropertyJson = "{\"defaultShard\":2,\"majorVersion\":6,\"proxy\":{\"ck\":{\"defaultCkDatabase\":\"testdb\",\"pass\":\"fc/3EtAe\",\"url\":\"10.177.43.183:6321\",\"user\":\"limited\"},\"es\":{\"headers\":{\"stoken\":\"b7842e1285e1d277e1730c41\"},\"host\":\"10.100.218.58:30691,10.100.218.60:30691,10.100.218.61:30691,10.100.218.62:30691,10.100.218.63:30691,10.100.218.64:30691\"},\"maxTimeRange\":86400000,\"roundAbleMinPeriod\":120000,\"whiteIndexList\":[\"table1_all\"]},\"query\":{\"sampleCountMaxThreshold\":1500000,\"useCache\":false,\"maxResultRow\":30000},\"threadPool\":{\"msearchProperty\":{\"coreSize\":100,\"queueSize\":10000}},\"yaml\":{\"name\":\"Yaml:1569371800\"}}";
+    private static final String DO_TEST = "doTest";
     @Resource
     private MsearchParamParser msearchParamParser;
     @Resource
@@ -90,7 +91,7 @@ public void testCommon() {
     }
 
     /**
-     * 基于查询参数，转换得到sql列表，并于期望值比对。比对一致则通过，不一致则失败
+     * 基于查询参数，转换得到sql列表，并与期望值比对。比对一致则通过，不一致则失败
      *
      * @param searchQueryJson 参数
      * @param needQueryHits   是否需要额外解析hits明细查询sql
@@ -103,10 +104,32 @@ public void doTest(String testName, String searchQueryJson, Boolean needQueryHit
             List<String> resultSqlList = convert2SqlList(searchQueryJson, needQueryHits);
             assertResult(testName, resultSqlList, expectedSqlList);
         } catch (Exception e) {
-            log.error("doTest", e);
+            log.error(DO_TEST, e);
             Assert.assertTrue(Boolean.FALSE);
         }
+    }
 
+    /**
+     * 基于查询参数，解析获取异常。并与期望异常比对。比对一致通过，不一致则失败
+     *
+     * @param testName
+     * @param searchQueryJson
+     * @param expectException
+     */
+    public void doTest(String testName, String searchQueryJson, Boolean needQueryHits,Exception expectException) {
+        try {
+            proxyConfigLoader.setKibanaProperty(JSONObject.parseObject(kibanaPropertyJson, KibanaProperty.class));
+            convert2SqlList(searchQueryJson, needQueryHits);
+            log.error(DO_TEST, String.format("%s期望异常%s，实际无异常", testName, expectException));
+            Assert.assertTrue(Boolean.FALSE);
+        } catch (Exception e) {
+            if (e.toString().equals(expectException.toString())) {
+                Assert.assertTrue(Boolean.TRUE);
+            } else {
+                log.error(DO_TEST, String.format("%s期望异常%s，实际异常为%s", testName, expectException, e));
+                Assert.assertTrue(Boolean.FALSE);
+            }
+        }
     }
 
     /**
@@ -186,7 +209,7 @@ public CkRequestContext parseRequest(String searchQueryJson) throws Exception {
         Map<String, Map<String, String>> tableColumnsCache = JSONObject.parseObject(tableColumnsCacheJson, Map.class);
         JSONObject searchQuery = JSONObject.parseObject(searchQueryJson);
         IndexPattern indexPattern = JSONObject.parseObject(indexPatternJson, IndexPattern.class);
-        CkRequestContext ckRequestContext = new CkRequestContext("testIp", indexPattern,proxyConfigLoader.getKibanaProperty().getQuery().getMaxResultRow());
+        CkRequestContext ckRequestContext = new CkRequestContext("testIp", indexPattern, proxyConfigLoader.getKibanaProperty().getQuery().getMaxResultRow());
         QueryProperty queryProperty = proxyConfigLoader.getKibanaProperty().getQuery();
         CkRequestContext.SampleParam sampleParam = new CkRequestContext.SampleParam(Constants.USE_SAMPLE_COUNT_THREASHOLD, queryProperty.getSampleCountMaxThreshold());
         ckRequestContext.setSampleParam(sampleParam);

diff --git a/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java b/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java
@@ -16,6 +16,7 @@
 package com.ly.ckibana.converter;
 
 import com.ly.ckibana.CommonTest;
+import com.ly.ckibana.model.exception.UnKnownFieldException;
 import org.junit.Test;
 
 /**
@@ -28,6 +29,8 @@ public class CommonDSLTest extends CommonTest {
     public static final String TEST_INTEGER_DSL = "INTEGER_DSL";
     public static final String TEST_STRING_DSL = "STRING_DSL";
     public static final String TEST_DATETIME64_DSL = "DATETIME64_DSL";
+    public static final String TEST_UNKNOWN_FIELD_QUERY = "unknownFiledQuery";
+
 
     /**
      * 整型DSL查询测试
@@ -83,4 +86,14 @@ public void testIpv4OrIpv6DSL() {
         doTest(TEST_DATETIME64_DSL, query, Boolean.TRUE, expectedSqls);
 
     }
+
+    /**
+     * DSL-unknownField Query
+     * unknowField:"value"
+     */
+    @Test
+    public void testUnknownFiledQuery(){
+        String query = "{\"version\":true,\"size\":500,\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"_source\":{\"excludes\":[]},\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTimeOnly\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"},{\"field\":\"@timestampDateTimeOnly\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_phrase\":{\"unknowField\":{\"query\":\"value\"}}},{\"range\":{\"@timestampDateTimeOnly\":{\"gte\":1712659236502,\"lte\":1712660136502,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"timeout\":\"300000ms\"}";
+        doTest(TEST_UNKNOWN_FIELD_QUERY, query, Boolean.FALSE,new UnKnownFieldException("unknowField"));
+    }
 }
diff --git a/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java b/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java
@@ -16,6 +16,7 @@
 package com.ly.ckibana.converter;
 
 import com.ly.ckibana.CommonTest;
+import com.ly.ckibana.model.exception.UnKnownFieldException;
 import org.junit.Test;
 
 /**
@@ -33,6 +34,8 @@ public class QueryStringClauseTest extends CommonTest {
     public static final String TEST_IP_TYPE_IPV4 = "IP_TYPE_IPV4";
     public static final String TEST_IP_TYPE_IPV6 = "IP_TYPE_IPV6";
     public static final String TEST_NUMBER = "NUMBER";
+    public static final String TEST_UNKNOWN_FIELD_QUERY = "unknownFiledQuery";
+
 
 
     /**
@@ -146,4 +149,14 @@ public void testNumberField() {
                 "            ]";
         doTest(TEST_NUMBER, query, Boolean.FALSE, expectedSqls);
     }
+
+    /**
+     * queryString-unknownField Query
+     * unknowField:"value"
+     */
+    @Test
+    public void testUnknownFiledQuery(){
+        String query = "{\"version\":true,\"size\":500,\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"_source\":{\"excludes\":[]},\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTimeOnly\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"},{\"field\":\"@timestampDateTimeOnly\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"unknownFiled:\\\"value\\\"\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTimeOnly\":{\"gte\":1712659440994,\"lte\":1712660340994,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"timeout\":\"300000ms\"}";
+        doTest(TEST_UNKNOWN_FIELD_QUERY, query, Boolean.FALSE,new UnKnownFieldException("unknownFiled"));
+    }
 }