diff --git a/docs/api.yaml b/docs/api.yaml index f9b3babf..b439e886 100644 --- a/docs/api.yaml +++ b/docs/api.yaml @@ -9,8 +9,34 @@ paths: summary: health check return version and revision operationId: HealthCheck consumes: + - '*/*' + produces: - application/json - text/yaml + responses: + '200': + description: '' + schema: + $ref: '#/definitions/DocHealthCheck' + '/v1/project': + get: + summary: list projects + operationId: Project + parameters: + - name: project + in: query + required: false + type: string + - name: limit + in: query + description: pagination + type: string + - name: offset + in: query + description: pagination + type: string + consumes: + - '*/*' produces: - application/json - text/yaml @@ -18,7 +44,9 @@ paths: '200': description: '' schema: - $ref: '#/definitions/DocHealthCheck' + $ref: '#/definitions/DocResponseGetProject' + '304': + description: empty body '/v1/{project}/kie/kv': get: summary: list key values by labels and key @@ -285,6 +313,16 @@ definitions: format: int64 version: type: string + DocResponseGetProject: + type: object + properties: + total: + type: integer + format: int64 + data: + type: array + items: + type: string DocResponseGetKey: type: object properties: diff --git a/pkg/common/common.go b/pkg/common/common.go index 205f2ef8..17b9b4b9 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -39,6 +39,7 @@ const ( QueryParamURLPath = "urlPath" QueryParamUserAgent = "userAgent" QueryParamOverride = "override" + QueryParameterProject = "project" ) //http headers diff --git a/pkg/model/db_schema.go b/pkg/model/db_schema.go index 0434dcdc..2b0cb3ba 100644 --- a/pkg/model/db_schema.go +++ b/pkg/model/db_schema.go @@ -90,6 +90,19 @@ type GetKVRequest struct { ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" swag:"string" validate:"uuid"` } +// ListProjectRequest contains project list request params +type ListProjectRequest struct { + Domain string `json:"domaListProjectRequestin,omitempty" yaml:"domain,omitempty" validate:"min=1,max=256,commonName"` //redundant + Project string `json:"project,omitempty" yaml:"project,omitempty"` + Offset int64 `validate:"min=0"` + Limit int64 `validate:"min=0,max=100"` +} + +//ProjectDoc is database struct to store projects +type ProjectDoc struct { + Project string `json:"project,omitempty" bson:"project,omitempty" yaml:"project,omitempty" validate:"min=1,max=256,commonName"` +} + // ListKVRequest contains kv list request params type ListKVRequest struct { Project string `json:"project,omitempty" yaml:"project,omitempty" validate:"min=1,max=256,commonName"` diff --git a/pkg/model/project.go b/pkg/model/project.go new file mode 100644 index 00000000..bec4076e --- /dev/null +++ b/pkg/model/project.go @@ -0,0 +1,7 @@ +package model + +//ProjectResponse represents the project list +type ProjectResponse struct { + Total int `json:"total"` + Data []*ProjectDoc `json:"data"` +} diff --git a/server/datasource/dao.go b/server/datasource/dao.go index 667e64fa..6c853115 100644 --- a/server/datasource/dao.go +++ b/server/datasource/dao.go @@ -63,6 +63,7 @@ type Broker interface { GetHistoryDao() HistoryDao GetTrackDao() TrackDao GetKVDao() KVDao + GetProjectDao() ProjectDao } func GetBroker() Broker { @@ -87,6 +88,11 @@ type KVDao interface { Total(ctx context.Context, project, domain string) (int64, error) } +type ProjectDao interface { + List(ctx context.Context, domain string, options ...FindOption) (*model.ProjectResponse, error) + Total(ctx context.Context, domain string) (int64, error) +} + //HistoryDao provide api of History entity type HistoryDao interface { AddHistory(ctx context.Context, kv *model.KVDoc) error diff --git a/server/datasource/etcd/init.go b/server/datasource/etcd/init.go index 8f0524ae..3fb4d6bc 100644 --- a/server/datasource/etcd/init.go +++ b/server/datasource/etcd/init.go @@ -20,6 +20,7 @@ package etcd import ( "crypto/tls" "fmt" + "github.com/apache/servicecomb-kie/server/datasource/etcd/project" "github.com/go-chassis/cari/db" dconfig "github.com/go-chassis/cari/db/config" @@ -69,6 +70,9 @@ func (*Broker) GetHistoryDao() datasource.HistoryDao { func (*Broker) GetTrackDao() datasource.TrackDao { return &track.Dao{} } +func (*Broker) GetProjectDao() datasource.ProjectDao { + return &project.Dao{} +} func init() { datasource.RegisterPlugin("etcd", NewFrom) diff --git a/server/datasource/etcd/project/project_dao.go b/server/datasource/etcd/project/project_dao.go new file mode 100644 index 00000000..7fcca399 --- /dev/null +++ b/server/datasource/etcd/project/project_dao.go @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package project + +import ( + "context" + "errors" + + "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/datasource" +) + +//Dao operate data in etcd +type Dao struct { +} + +//Total get projects total counts by domain +func (s *Dao) Total(ctx context.Context, domain string) (int64, error) { + // TODO etcd needs to be done + return 0, errors.New("can not list project,etcd not support yet") +} + +//List get projects list by domain and name +func (s *Dao) List(ctx context.Context, domain string, options ...datasource.FindOption) (*model.ProjectResponse, error) { + // TODO etcd needs to be done + return nil, errors.New("can not list project,etcd not support yet") +} diff --git a/server/datasource/mongo/init.go b/server/datasource/mongo/init.go index a104450f..d59f65be 100644 --- a/server/datasource/mongo/init.go +++ b/server/datasource/mongo/init.go @@ -21,24 +21,23 @@ import ( "context" "crypto/tls" "fmt" - - "github.com/go-chassis/cari/db" - dconfig "github.com/go-chassis/cari/db/config" - dmongo "github.com/go-chassis/cari/db/mongo" - "github.com/go-chassis/openlog" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - "go.mongodb.org/mongo-driver/x/bsonx" - "github.com/apache/servicecomb-kie/server/config" "github.com/apache/servicecomb-kie/server/datasource" "github.com/apache/servicecomb-kie/server/datasource/mongo/counter" "github.com/apache/servicecomb-kie/server/datasource/mongo/history" "github.com/apache/servicecomb-kie/server/datasource/mongo/kv" "github.com/apache/servicecomb-kie/server/datasource/mongo/model" + "github.com/apache/servicecomb-kie/server/datasource/mongo/project" "github.com/apache/servicecomb-kie/server/datasource/mongo/track" "github.com/apache/servicecomb-kie/server/datasource/tlsutil" + "github.com/go-chassis/cari/db" + dconfig "github.com/go-chassis/cari/db/config" + dmongo "github.com/go-chassis/cari/db/mongo" + "github.com/go-chassis/openlog" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/x/bsonx" ) type Broker struct { @@ -84,6 +83,9 @@ func (*Broker) GetHistoryDao() datasource.HistoryDao { func (*Broker) GetTrackDao() datasource.TrackDao { return &track.Dao{} } +func (*Broker) GetProjectDao() datasource.ProjectDao { + return &project.Dao{} +} func ensureDB() error { err := ensureRevisionCounter() diff --git a/server/datasource/mongo/project/project_dao.go b/server/datasource/mongo/project/project_dao.go new file mode 100644 index 00000000..cfc03d09 --- /dev/null +++ b/server/datasource/mongo/project/project_dao.go @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package project + +import ( + "context" + "fmt" + "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/datasource" + mmodel "github.com/apache/servicecomb-kie/server/datasource/mongo/model" + "github.com/go-chassis/cari/db/mongo" + "github.com/go-chassis/openlog" + "go.mongodb.org/mongo-driver/bson" + "strings" +) + +const ( + MsgFindKvFailed = "find kv failed, deadline exceeded" + FmtErrFindKvFailed = "can not find kv in %s" +) + +//Dao operate data in mongodb +type Dao struct { +} + +//Total get projects total counts by domain +func (s *Dao) Total(ctx context.Context, domain string) (int64, error) { + collection := mongo.GetClient().GetDB().Collection(mmodel.CollectionKV) + filter := bson.M{"domain": domain} + total, err := collection.CountDocuments(ctx, filter) + if err != nil { + openlog.Error("find total number: " + err.Error()) + return 0, err + } + return total, err +} + +//List get projects list by domain and name +func (s *Dao) List(ctx context.Context, domain string, options ...datasource.FindOption) (*model.ProjectResponse, error) { + opts := datasource.NewDefaultFindOpts() + for _, o := range options { + o(&opts) + } + data, total, err := findProjects(ctx, domain, opts) + if err != nil { + return nil, err + } + result := &model.ProjectResponse{ + Data: []*model.ProjectDoc{}, + } + for _, value := range data { + curKV := &model.ProjectDoc{Project: value.(string)} + result.Data = append(result.Data, curKV) + } + result.Total = total + return result, nil +} + +func findProjects(ctx context.Context, domain string, opts datasource.FindOptions) ([]interface{}, int, error) { + collection := mongo.GetClient().GetDB().Collection(mmodel.CollectionKV) + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + + filter := bson.M{"domain": domain} + if len(strings.TrimSpace(opts.Project)) > 0 { + filter["project"] = opts.Project + } + result, err := collection.Distinct(ctx, "project", filter) + if err != nil { + if err.Error() == context.DeadlineExceeded.Error() { + openlog.Error(MsgFindKvFailed, openlog.WithTags(openlog.Tags{ + "timeout": opts.Timeout, + })) + return nil, 0, fmt.Errorf(FmtErrFindKvFailed, opts.Timeout) + } + return nil, 0, err + } + curTotal := len(result) + offset := opts.Offset + limit := offset + opts.Limit + if offset > int64(curTotal) { + offset = int64(curTotal) + limit = int64(curTotal) + } else if limit > int64(curTotal) { + limit = int64(curTotal) + } + result = result[offset:limit] + return result, len(result), err +} diff --git a/server/datasource/options.go b/server/datasource/options.go index 2d6fee1f..524d39ad 100644 --- a/server/datasource/options.go +++ b/server/datasource/options.go @@ -67,6 +67,7 @@ type WriteOptions struct { type FindOptions struct { ExactLabels bool Status string + Project string Depth int ID string Key string @@ -86,6 +87,13 @@ type WriteOption func(*WriteOptions) //FindOption is functional option to find key value type FindOption func(*FindOptions) +//WithProject find by project +func WithProject(project string) FindOption { + return func(o *FindOptions) { + o.Project = project + } +} + // WithSync indicates that the synchronization function is on func WithSync(enabled bool) WriteOption { return func(o *WriteOptions) { diff --git a/server/resource/v1/doc_struct.go b/server/resource/v1/doc_struct.go index 8a3ddacc..b8ae963b 100644 --- a/server/resource/v1/doc_struct.go +++ b/server/resource/v1/doc_struct.go @@ -127,7 +127,7 @@ var ( ParamType: goRestful.QueryParameterKind, Desc: "pagination", } - //polling data + // DocQuerySessionIDParameters polling data DocQuerySessionIDParameters = &restful.Parameters{ DataType: "string", Name: common.QueryParamSessionID, @@ -152,6 +152,12 @@ var ( ParamType: goRestful.QueryParameterKind, Desc: "user agent of the call", } + DocQueryProjectParameters = &restful.Parameters{ + DataType: "string", + Name: common.QueryParameterProject, + ParamType: goRestful.QueryParameterKind, + Desc: "project name of the call", + } ) //swagger doc path params diff --git a/server/resource/v1/project_resource.go b/server/resource/v1/project_resource.go new file mode 100644 index 00000000..d8671563 --- /dev/null +++ b/server/resource/v1/project_resource.go @@ -0,0 +1,85 @@ +package v1 + +import ( + "github.com/apache/servicecomb-kie/pkg/common" + "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/service/kv" + goRestful "github.com/emicklei/go-restful" + "github.com/go-chassis/cari/config" + "github.com/go-chassis/foundation/validator" + "github.com/go-chassis/go-chassis/v2/server/restful" + "github.com/go-chassis/openlog" + "net/http" + "strconv" +) + +type ProjectResource struct { +} + +func (r *ProjectResource) URLPatterns() []restful.Route { + return []restful.Route{ + { + Method: http.MethodGet, + Path: "/v1/project", + ResourceFunc: r.List, + FuncDesc: "list projects", + Parameters: []*restful.Parameters{ + DocQueryProjectParameters, DocQueryLimitParameters, DocQueryOffsetParameters, + }, + Returns: []*restful.Returns{ + { + Code: http.StatusOK, + Model: model.ProjectResponse{}, + Headers: map[string]goRestful.Header{ + common.HeaderRevision: DocHeaderRevision, + }, + }, { + Code: http.StatusNotModified, + Message: "empty body", + }, + }, + Produces: []string{goRestful.MIME_JSON}, + }, + } +} + +//List response Projects list +func (r *ProjectResource) List(rctx *restful.Context) { + var err error + request := &model.ListProjectRequest{ + Project: rctx.ReadQueryParameter(common.PathParameterProject), + } + request.Domain = ReadDomain(rctx.Ctx) + offsetStr := rctx.ReadQueryParameter(common.QueryParamOffset) + limitStr := rctx.ReadQueryParameter(common.QueryParamLimit) + offset, limit, err := checkPagination(offsetStr, limitStr) + if err != nil { + WriteErrResponse(rctx, config.ErrInvalidParams, err.Error()) + return + } + if limit == 0 { + limit = 20 + } + request.Offset = offset + request.Limit = limit + err = validator.Validate(request) + if err != nil { + WriteErrResponse(rctx, config.ErrInvalidParams, err.Error()) + return + } + returnProjectsData(rctx, request) +} + +func returnProjectsData(rctx *restful.Context, request *model.ListProjectRequest) { + rev, projects, queryErr := kv.ListProjects(rctx.Ctx, request) + if queryErr != nil { + WriteErrResponse(rctx, queryErr.Code, queryErr.Message) + return + } + rctx.ReadResponseWriter().Header().Set(common.HeaderRevision, strconv.FormatInt(rev, 10)) + err := writeResponse(rctx, projects) + rctx.ReadRestfulRequest().SetAttribute(common.RespBodyContextKey, projects.Data) + if err != nil { + openlog.Error(err.Error()) + } +} diff --git a/server/resource/v1/project_resource_test.go b/server/resource/v1/project_resource_test.go new file mode 100644 index 00000000..541ff335 --- /dev/null +++ b/server/resource/v1/project_resource_test.go @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1_test + +import ( + "bytes" + "encoding/json" + "github.com/apache/servicecomb-kie/pkg/model" + _ "github.com/apache/servicecomb-kie/server/datasource/mongo" + _ "github.com/apache/servicecomb-kie/server/plugin/qms" + v1 "github.com/apache/servicecomb-kie/server/resource/v1" + _ "github.com/apache/servicecomb-kie/test" + "github.com/go-chassis/go-chassis/v2/server/restful/restfultest" + "github.com/stretchr/testify/assert" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +func TestProjectResource_Get(t *testing.T) { + t.Run("get project list", func(t *testing.T) { + kv := &model.ProjectDoc{} + j, _ := json.Marshal(kv) + r, _ := http.NewRequest("GET", "/v1/project", bytes.NewBuffer(j)) + r.Header.Set("Content-Type", "application/json") + pr := &v1.ProjectResource{} + c, _ := restfultest.New(pr, nil) + resp := httptest.NewRecorder() + c.ServeHTTP(resp, r) + body, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.Code, string(body)) + result := &model.ProjectResponse{} + err = json.Unmarshal(body, result) + assert.NoError(t, err) + assert.Equal(t, 3, len(result.Data)) + }) +} diff --git a/server/server.go b/server/server.go index c18167b9..c9b22364 100644 --- a/server/server.go +++ b/server/server.go @@ -33,6 +33,7 @@ func Run() { chassis.RegisterSchema(common.ProtocolRest, &v1.KVResource{}) chassis.RegisterSchema(common.ProtocolRest, &v1.HistoryResource{}) chassis.RegisterSchema(common.ProtocolRest, &v1.AdminResource{}) + chassis.RegisterSchema(common.ProtocolRest, &v1.ProjectResource{}) if err := chassis.Init(); err != nil { openlog.Fatal(err.Error()) } diff --git a/server/service/kv/project_svc.go b/server/service/kv/project_svc.go new file mode 100644 index 00000000..b23a15fa --- /dev/null +++ b/server/service/kv/project_svc.go @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kv + +import ( + "context" + + "github.com/apache/servicecomb-kie/pkg/common" + "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/datasource" + "github.com/go-chassis/cari/config" + "github.com/go-chassis/cari/pkg/errsvc" + "github.com/go-chassis/openlog" +) + +func ListProjects(ctx context.Context, request *model.ListProjectRequest) (int64, *model.ProjectResponse, *errsvc.Error) { + opts := []datasource.FindOption{ + datasource.WithOffset(request.Offset), + datasource.WithLimit(request.Limit), + } + if len(request.Project) > 0 { + opts = append(opts, datasource.WithProject(request.Project)) + } + rev, err := datasource.GetBroker().GetRevisionDao().GetRevision(ctx, request.Domain) + if err != nil { + return rev, nil, config.NewError(config.ErrInternal, err.Error()) + } + kv, err := listProjectsHandle(ctx, request.Domain, opts...) + if err != nil { + openlog.Error("common: " + err.Error()) + return rev, nil, config.NewError(config.ErrInternal, common.MsgDBError) + } + return rev, kv, nil +} + +func listProjectsHandle(ctx context.Context, domain string, options ...datasource.FindOption) (*model.ProjectResponse, error) { + listSema.Acquire() + defer listSema.Release() + return datasource.GetBroker().GetProjectDao().List(ctx, domain, options...) +}