From b11cb7cbf8bd89c9745228c40c49dbafe386f16e Mon Sep 17 00:00:00 2001 From: tianyizhu Date: Mon, 28 Mar 2022 20:11:55 +0800 Subject: [PATCH 1/3] update iam search engine config tpl --- .../templates/#etc#bkiam_search_engine_config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/support-files/templates/#etc#bkiam_search_engine_config.yaml b/build/support-files/templates/#etc#bkiam_search_engine_config.yaml index 423b445..e4f710a 100644 --- a/build/support-files/templates/#etc#bkiam_search_engine_config.yaml +++ b/build/support-files/templates/#etc#bkiam_search_engine_config.yaml @@ -25,9 +25,9 @@ index: elasticsearch: indexName: iam_policy addresses: - - http://__BK_IAM_SEARCH_ENGINE_ES_HOST__:__BK_IAM_SEARCH_ENGINE_ES_PORT__ - username: "__BK_IAM_SEARCH_ENGINE_ES_USER__" - password: "__BK_IAM_SEARCH_ENGINE_ES_PASSWORD__" + - http://__BK_IAM_SEARCH_ENGINE_ES7_HOST__:__BK_IAM_SEARCH_ENGINE_ES7_PORT__ + username: "__BK_IAM_SEARCH_ENGINE_ES7_USER__" + password: "__BK_IAM_SEARCH_ENGINE_ES7_PASSWORD__" maxRetries: 3 backend: From 93fd30aa84e7599203dfaf4adc5d017a3cd81a9a Mon Sep 17 00:00:00 2001 From: tianyizhu Date: Thu, 31 Mar 2022 15:43:31 +0800 Subject: [PATCH 2/3] bugfix: create index use dynamic_templates --- VERSION | 2 +- pkg/client/es.go | 23 +++++++++++++++++++++-- pkg/engine/doc/es.go | 6 +++--- pkg/engine/doc/es_engine.go | 29 ++++++++++++----------------- release.md | 4 ++++ 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/VERSION b/VERSION index 90a27f9..af0b7dd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.5 +1.0.6 diff --git a/pkg/client/es.go b/pkg/client/es.go index fa605ef..14d730e 100644 --- a/pkg/client/es.go +++ b/pkg/client/es.go @@ -1,4 +1,4 @@ - /* +/* * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-权限中心检索引擎 * (BlueKing-IAM-Search-Engine) available. * Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. @@ -19,6 +19,7 @@ import ( "net/http" "runtime" "strconv" + "strings" "time" "github.com/cenkalti/backoff/v4" @@ -481,7 +482,25 @@ func (c *EsClient) Ping() (*esapi.Response, error) { // CreateIndex ... func (c *EsClient) CreateIndex(index string) (*esapi.Response, error) { - return c.client.Indices.Create(index) + // 动态mapping, match string 类型时索引转换为 keyword 类型 + mapping := `{ + "mappings": { + "dynamic_templates": [ + { + "strings": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ] + } + }` + + return c.client.Indices.Create(index, func(req *esapi.IndicesCreateRequest) { + req.Body = strings.NewReader(mapping) + }) } // IndexExists ... diff --git a/pkg/engine/doc/es.go b/pkg/engine/doc/es.go index 6074511..376f588 100644 --- a/pkg/engine/doc/es.go +++ b/pkg/engine/doc/es.go @@ -1,4 +1,4 @@ - /* +/* * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-权限中心检索引擎 * (BlueKing-IAM-Search-Engine) available. * Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. @@ -175,8 +175,8 @@ func (e *EsEngine) getActionCount(system, action string) (int, error) { "query": types.H{ "bool": types.H{ "filter": []interface{}{ - types.H{"term": types.H{"system.keyword": system}}, - types.H{"term": types.H{"action.id.keyword": action}}, + types.H{"term": types.H{"system": system}}, + types.H{"term": types.H{"action.id": action}}, }, }, }, diff --git a/pkg/engine/doc/es_engine.go b/pkg/engine/doc/es_engine.go index 817bfcb..be80001 100644 --- a/pkg/engine/doc/es_engine.go +++ b/pkg/engine/doc/es_engine.go @@ -1,4 +1,4 @@ - /* +/* * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-权限中心检索引擎 * (BlueKing-IAM-Search-Engine) available. * Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. @@ -32,7 +32,7 @@ func genDocQuery(req *types.SearchRequest) types.H { // if x._bk_iam_path_ starts_with /biz,1/cluster,2/ => x._bk_iam_path_ in [/biz,1/, /biz,1/cluster,2/] if key == types.BkIAMPathKey { paths := generateBkIAMPathList(value) - term := fmt.Sprintf("resource.%s.%s.%s.keyword", system, resourceNode.Type, key) + term := fmt.Sprintf("resource.%s.%s.%s", system, resourceNode.Type, key) for _, path := range paths { sqs = append(sqs, types.H{ "term": types.H{term: path}, @@ -41,12 +41,7 @@ func genDocQuery(req *types.SearchRequest) types.H { continue } - // 除了bk iam path以外的属性不能确定value类型, 判断是string再加上 keyword fieldName := fmt.Sprintf("resource.%s.%s.%s", system, resourceNode.Type, key) - _, ok := value.(string) - if ok { - fieldName = fieldName + ".keyword" - } sqs = append(sqs, types.H{ "term": types.H{fieldName: value}, }) @@ -77,15 +72,15 @@ func genDocQuery(req *types.SearchRequest) types.H { }, }, }, - types.H{"term": types.H{"system.keyword": system}}, - types.H{"term": types.H{"action.id.keyword": action}}, - types.H{"term": types.H{"type.keyword": string(types.Doc)}}, + types.H{"term": types.H{"system": system}}, + types.H{"term": types.H{"action.id": action}}, + types.H{"term": types.H{"type": string(types.Doc)}}, } // filter the subject not the req.SubjectType if req.SubjectType != types.SubjectTypeAll { must = append(must, types.H{ - "term": types.H{"subject.type.keyword": req.SubjectType}, + "term": types.H{"subject.type": req.SubjectType}, }) } @@ -154,8 +149,8 @@ func genSubjectsBoolCondition(subjects []types.Subject) types.H { sq := types.H{ "bool": types.H{ "must": []interface{}{ - types.H{"term": types.H{"subject.type.keyword": subject.Type}}, - types.H{"term": types.H{"subject.id.keyword": subject.ID}}, + types.H{"term": types.H{"subject.type": subject.Type}}, + types.H{"term": types.H{"subject.id": subject.ID}}, }, }, } @@ -189,15 +184,15 @@ func genAnyQuery(req *types.SearchRequest) types.H { }, }, }, - types.H{"term": types.H{"system.keyword": system}}, - types.H{"term": types.H{"action.id.keyword": action}}, - types.H{"term": types.H{"type.keyword": string(types.Any)}}, + types.H{"term": types.H{"system": system}}, + types.H{"term": types.H{"action.id": action}}, + types.H{"term": types.H{"type": string(types.Any)}}, } // filter the subject not the req.SubjectType if req.SubjectType != types.SubjectTypeAll { filter = append(filter, types.H{ - "term": types.H{"subject.type.keyword": req.SubjectType}, + "term": types.H{"subject.type": req.SubjectType}, }) } diff --git a/release.md b/release.md index f5073bf..2f0c62c 100644 --- a/release.md +++ b/release.md @@ -1,3 +1,7 @@ +# 1.0.6 + +- bugfix: create index use dynamic_templates + # 1.0.5 - update: change metrics name with prefix bkiam_search_engine_ From e2a3592f843fb09190f854bbf019ce9edde03726 Mon Sep 17 00:00:00 2001 From: tianyizhu Date: Thu, 31 Mar 2022 15:49:47 +0800 Subject: [PATCH 3/3] bugfix: create index use dynamic_templates --- pkg/client/es.go | 18 +----------------- pkg/indexer/index.go | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/pkg/client/es.go b/pkg/client/es.go index 14d730e..4a6e505 100644 --- a/pkg/client/es.go +++ b/pkg/client/es.go @@ -481,23 +481,7 @@ func (c *EsClient) Ping() (*esapi.Response, error) { } // CreateIndex ... -func (c *EsClient) CreateIndex(index string) (*esapi.Response, error) { - // 动态mapping, match string 类型时索引转换为 keyword 类型 - mapping := `{ - "mappings": { - "dynamic_templates": [ - { - "strings": { - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - } - ] - } - }` - +func (c *EsClient) CreateIndex(index string, mapping string) (*esapi.Response, error) { return c.client.Indices.Create(index, func(req *esapi.IndicesCreateRequest) { req.Body = strings.NewReader(mapping) }) diff --git a/pkg/indexer/index.go b/pkg/indexer/index.go index a006457..b3d87c2 100644 --- a/pkg/indexer/index.go +++ b/pkg/indexer/index.go @@ -1,4 +1,4 @@ - /* +/* * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-权限中心检索引擎 * (BlueKing-IAM-Search-Engine) available. * Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. @@ -288,7 +288,22 @@ func creatIndexIfNotExists(cfg *config.Index) error { } if resp.StatusCode == http.StatusNotFound { - _, err = esClient.CreateIndex(cfg.ElasticSearch.IndexName) + // 动态mapping, match string 类型时索引转换为 keyword 类型 + mapping := `{ + "mappings": { + "dynamic_templates": [ + { + "strings": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ] + } + }` + _, err = esClient.CreateIndex(cfg.ElasticSearch.IndexName, mapping) if err != nil { return fmt.Errorf("create index: [%s] error:%w", cfg.ElasticSearch.IndexName, err) }