From ba4b2a0f7a08486212a527cbb573533178386449 Mon Sep 17 00:00:00 2001 From: Nicolas Zin Date: Sat, 26 Aug 2023 06:04:53 -0400 Subject: [PATCH] improve the rest api server: - fix some caching logic issue - set a correct readiness endpoint - add a remote flush cache endpoint --- docs/api_docs/bundle.yaml | 13 ++ internal/goliac.go | 7 ++ internal/goliac_server.go | 21 +++- internal/sync/remote.go | 115 ++++++++++-------- swagger/flushcache.yaml | 12 ++ swagger/index.yaml | 2 + swagger_gen/restapi/embedded_spec.go | 40 ++++++ .../restapi/operations/app/get_flush_cache.go | 56 +++++++++ .../app/get_flush_cache_parameters.go | 46 +++++++ .../app/get_flush_cache_responses.go | 98 +++++++++++++++ .../app/get_flush_cache_urlbuilder.go | 87 +++++++++++++ swagger_gen/restapi/operations/goliac_api.go | 13 ++ 12 files changed, 461 insertions(+), 49 deletions(-) create mode 100644 swagger/flushcache.yaml create mode 100644 swagger_gen/restapi/operations/app/get_flush_cache.go create mode 100644 swagger_gen/restapi/operations/app/get_flush_cache_parameters.go create mode 100644 swagger_gen/restapi/operations/app/get_flush_cache_responses.go create mode 100644 swagger_gen/restapi/operations/app/get_flush_cache_urlbuilder.go diff --git a/docs/api_docs/bundle.yaml b/docs/api_docs/bundle.yaml index 5a4f5c8..8c28e31 100644 --- a/docs/api_docs/bundle.yaml +++ b/docs/api_docs/bundle.yaml @@ -53,6 +53,19 @@ paths: description: generic error response schema: $ref: '#/definitions/error' + /flushcache: + get: + tags: + - app + operationId: getFlushCache + description: Flush the Github remote cache + responses: + '200': + description: cache flushed + default: + description: generic error response + schema: + $ref: '#/definitions/error' definitions: health: type: object diff --git a/internal/goliac.go b/internal/goliac.go index 425f5f1..d34490b 100644 --- a/internal/goliac.go +++ b/internal/goliac.go @@ -34,6 +34,9 @@ type Goliac interface { // to close the clone git repository (if you called LoadAndValidateGoliacOrganization) Close() + + // flush remote cache + FlushCache() } type GoliacImpl struct { @@ -67,6 +70,10 @@ func NewGoliacImpl() (Goliac, error) { }, nil } +func (g *GoliacImpl) FlushCache() { + g.remote.FlushCache() +} + func (g *GoliacImpl) LoadAndValidateGoliacOrganization(repositoryUrl, branch string) error { var errs []error var warns []entity.Warning diff --git a/internal/goliac_server.go b/internal/goliac_server.go index 7bcf293..e4d4403 100644 --- a/internal/goliac_server.go +++ b/internal/goliac_server.go @@ -16,6 +16,7 @@ import ( "github.com/Alayacare/goliac/swagger_gen/models" "github.com/Alayacare/goliac/swagger_gen/restapi" "github.com/Alayacare/goliac/swagger_gen/restapi/operations" + "github.com/Alayacare/goliac/swagger_gen/restapi/operations/app" "github.com/Alayacare/goliac/swagger_gen/restapi/operations/health" "github.com/go-openapi/loads" "github.com/go-openapi/runtime/middleware" @@ -26,16 +27,20 @@ type GoliacServer interface { Serve() GetLiveness(health.GetLivenessParams) middleware.Responder GetReadiness(health.GetReadinessParams) middleware.Responder + GetFlushCache(app.GetFlushCacheParams) middleware.Responder } type GoliacServerImpl struct { goliac Goliac applyMutex gosync.Mutex + // when the server has finished to load the local configuration + ready bool } func NewGoliacServer(goliac Goliac) GoliacServer { return &GoliacServerImpl{ goliac: goliac, + ready: false, } } @@ -44,7 +49,17 @@ func (c *GoliacServerImpl) GetLiveness(params health.GetLivenessParams) middlewa } func (c *GoliacServerImpl) GetReadiness(params health.GetReadinessParams) middleware.Responder { - return health.NewGetLivenessOK().WithPayload(&models.Health{Status: "OK"}) + if c.ready { + return health.NewGetLivenessOK().WithPayload(&models.Health{Status: "OK"}) + } else { + message := "Not yet ready, loading local state" + return health.NewGetLivenessDefault(503).WithPayload(&models.Error{Message: &message}) + } +} + +func (c *GoliacServerImpl) GetFlushCache(app.GetFlushCacheParams) middleware.Responder { + c.goliac.FlushCache() + return app.NewGetFlushCacheOK() } func (g *GoliacServerImpl) Serve() { @@ -137,6 +152,10 @@ func (g *GoliacServerImpl) serveApply() error { if err != nil { return fmt.Errorf("failed to load and validate: %s", err) } + + // we are ready (to give local state, and to sync with remote) + g.ready = true + u, err := url.Parse(repo) if err != nil { return fmt.Errorf("failed to parse %s: %v", repo, err) diff --git a/internal/sync/remote.go b/internal/sync/remote.go index 39d48c7..1effe32 100644 --- a/internal/sync/remote.go +++ b/internal/sync/remote.go @@ -107,13 +107,10 @@ func (g *GoliacRemoteImpl) FlushCache() { func (g *GoliacRemoteImpl) RuleSets() map[string]*GithubRuleSet { if time.Now().After(g.ttlExpireRulesets) { - err := g.Load() + rulesets, err := g.loadRulesets() if err == nil { - rulesets, err := g.loadRulesets() - if err == nil { - g.rulesets = rulesets - g.ttlExpireRulesets = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) - } + g.rulesets = rulesets + g.ttlExpireRulesets = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) } } return g.rulesets @@ -164,6 +161,7 @@ func (g *GoliacRemoteImpl) Teams() map[string]*GithubTeam { } return g.teams } + func (g *GoliacRemoteImpl) Repositories() map[string]*GithubRepository { if time.Now().After(g.ttlExpireRepositories) { repositories, repositoriesByRefIds, err := g.loadRepositories() @@ -175,12 +173,21 @@ func (g *GoliacRemoteImpl) Repositories() map[string]*GithubRepository { } return g.repositories } + func (g *GoliacRemoteImpl) TeamRepositories() map[string]map[string]*GithubTeamRepo { if time.Now().After(g.ttlExpireTeamsRepos) { - teamsRepos, err := g.loadTeamReposConcurrently(config.Config.GithubConcurrentThreads) - if err == nil { - g.teamRepos = teamsRepos - g.ttlExpireTeamsRepos = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) + if config.Config.GithubConcurrentThreads <= 1 { + teamsrepos, err := g.loadTeamReposNonConcurrently() + if err == nil { + g.teamRepos = teamsrepos + g.ttlExpireTeamsRepos = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) + } + } else { + teamsrepos, err := g.loadTeamReposConcurrently(config.Config.GithubConcurrentThreads) + if err == nil { + g.teamRepos = teamsrepos + g.ttlExpireTeamsRepos = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) + } } } return g.teamRepos @@ -509,57 +516,69 @@ func (g *GoliacRemoteImpl) loadAppIds() (map[string]int, error) { } func (g *GoliacRemoteImpl) Load() error { - users, err := g.loadOrgUsers() - if err != nil { - return err - } - g.users = users - g.ttlExpireUsers = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) - - repositories, repositoriesByRefId, err := g.loadRepositories() - if err != nil { - return err - } - g.repositories = repositories - g.repositoriesByRefId = repositoriesByRefId - g.ttlExpireRepositories = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) - - teams, teamSlugByName, err := g.loadTeams() - if err != nil { - return err + if time.Now().After(g.ttlExpireUsers) { + users, err := g.loadOrgUsers() + if err != nil { + return err + } + g.users = users + g.ttlExpireUsers = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) } - g.teams = teams - g.teamSlugByName = teamSlugByName - g.ttlExpireTeams = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) - appIds, err := g.loadAppIds() - if err != nil { - return err + if time.Now().After(g.ttlExpireRepositories) { + repositories, repositoriesByRefId, err := g.loadRepositories() + if err != nil { + return err + } + g.repositories = repositories + g.repositoriesByRefId = repositoriesByRefId + g.ttlExpireRepositories = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) } - g.appIds = appIds - g.ttlExpireAppIds = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) - rulesets, err := g.loadRulesets() - if err != nil { - return err + if time.Now().After(g.ttlExpireTeams) { + teams, teamSlugByName, err := g.loadTeams() + if err != nil { + return err + } + g.teams = teams + g.teamSlugByName = teamSlugByName + g.ttlExpireTeams = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) } - g.rulesets = rulesets - g.ttlExpireRulesets = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) - if config.Config.GithubConcurrentThreads <= 1 { - teamsrepos, err := g.loadTeamReposNonConcurrently() + if time.Now().After(g.ttlExpireAppIds) { + appIds, err := g.loadAppIds() if err != nil { return err } - g.teamRepos = teamsrepos - } else { - teamsrepos, err := g.loadTeamReposConcurrently(config.Config.GithubConcurrentThreads) + g.appIds = appIds + g.ttlExpireAppIds = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) + } + + if time.Now().After(g.ttlExpireRulesets) { + rulesets, err := g.loadRulesets() if err != nil { return err } - g.teamRepos = teamsrepos + g.rulesets = rulesets + g.ttlExpireRulesets = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) + } + + if time.Now().After(g.ttlExpireTeamsRepos) { + if config.Config.GithubConcurrentThreads <= 1 { + teamsrepos, err := g.loadTeamReposNonConcurrently() + if err != nil { + return err + } + g.teamRepos = teamsrepos + } else { + teamsrepos, err := g.loadTeamReposConcurrently(config.Config.GithubConcurrentThreads) + if err != nil { + return err + } + g.teamRepos = teamsrepos + } + g.ttlExpireTeamsRepos = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) } - g.ttlExpireTeamsRepos = time.Now().Add(time.Duration(config.Config.GithubCacheTTL)) logrus.Infof("Nb remote users: %d", len(g.users)) logrus.Infof("Nb remote teams: %d", len(g.teams)) diff --git a/swagger/flushcache.yaml b/swagger/flushcache.yaml new file mode 100644 index 0000000..453a7d1 --- /dev/null +++ b/swagger/flushcache.yaml @@ -0,0 +1,12 @@ +get: + tags: + - app + operationId: getFlushCache + description: Flush the Github remote cache + responses: + 200: + description: cache flushed + default: + description: generic error response + schema: + $ref: "#/definitions/error" diff --git a/swagger/index.yaml b/swagger/index.yaml index f9fc7e2..ad9e6b8 100644 --- a/swagger/index.yaml +++ b/swagger/index.yaml @@ -28,6 +28,8 @@ paths: $ref: ./liveness.yaml /readiness: $ref: ./readiness.yaml + /flushcache: + $ref: ./flushcache.yaml definitions: diff --git a/swagger_gen/restapi/embedded_spec.go b/swagger_gen/restapi/embedded_spec.go index 76de0a4..00c6a8d 100644 --- a/swagger_gen/restapi/embedded_spec.go +++ b/swagger_gen/restapi/embedded_spec.go @@ -35,6 +35,26 @@ func init() { }, "basePath": "/api/v1", "paths": { + "/flushcache": { + "get": { + "description": "Flush the Github remote cache", + "tags": [ + "app" + ], + "operationId": "getFlushCache", + "responses": { + "200": { + "description": "cache flushed" + }, + "default": { + "description": "generic error response", + "schema": { + "$ref": "#/definitions/error" + } + } + } + } + }, "/liveness": { "get": { "description": "Check if Goliac is healthy", @@ -143,6 +163,26 @@ func init() { }, "basePath": "/api/v1", "paths": { + "/flushcache": { + "get": { + "description": "Flush the Github remote cache", + "tags": [ + "app" + ], + "operationId": "getFlushCache", + "responses": { + "200": { + "description": "cache flushed" + }, + "default": { + "description": "generic error response", + "schema": { + "$ref": "#/definitions/error" + } + } + } + } + }, "/liveness": { "get": { "description": "Check if Goliac is healthy", diff --git a/swagger_gen/restapi/operations/app/get_flush_cache.go b/swagger_gen/restapi/operations/app/get_flush_cache.go new file mode 100644 index 0000000..efbaa17 --- /dev/null +++ b/swagger_gen/restapi/operations/app/get_flush_cache.go @@ -0,0 +1,56 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package app + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime/middleware" +) + +// GetFlushCacheHandlerFunc turns a function with the right signature into a get flush cache handler +type GetFlushCacheHandlerFunc func(GetFlushCacheParams) middleware.Responder + +// Handle executing the request and returning a response +func (fn GetFlushCacheHandlerFunc) Handle(params GetFlushCacheParams) middleware.Responder { + return fn(params) +} + +// GetFlushCacheHandler interface for that can handle valid get flush cache params +type GetFlushCacheHandler interface { + Handle(GetFlushCacheParams) middleware.Responder +} + +// NewGetFlushCache creates a new http.Handler for the get flush cache operation +func NewGetFlushCache(ctx *middleware.Context, handler GetFlushCacheHandler) *GetFlushCache { + return &GetFlushCache{Context: ctx, Handler: handler} +} + +/* + GetFlushCache swagger:route GET /flushcache app getFlushCache + +Flush the Github remote cache +*/ +type GetFlushCache struct { + Context *middleware.Context + Handler GetFlushCacheHandler +} + +func (o *GetFlushCache) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + route, rCtx, _ := o.Context.RouteInfo(r) + if rCtx != nil { + *r = *rCtx + } + var Params = NewGetFlushCacheParams() + if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + + res := o.Handler.Handle(Params) // actually handle the request + o.Context.Respond(rw, r, route.Produces, route, res) + +} diff --git a/swagger_gen/restapi/operations/app/get_flush_cache_parameters.go b/swagger_gen/restapi/operations/app/get_flush_cache_parameters.go new file mode 100644 index 0000000..b3d1e94 --- /dev/null +++ b/swagger_gen/restapi/operations/app/get_flush_cache_parameters.go @@ -0,0 +1,46 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package app + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime/middleware" +) + +// NewGetFlushCacheParams creates a new GetFlushCacheParams object +// +// There are no default values defined in the spec. +func NewGetFlushCacheParams() GetFlushCacheParams { + + return GetFlushCacheParams{} +} + +// GetFlushCacheParams contains all the bound params for the get flush cache operation +// typically these are obtained from a http.Request +// +// swagger:parameters getFlushCache +type GetFlushCacheParams struct { + + // HTTP Request Object + HTTPRequest *http.Request `json:"-"` +} + +// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface +// for simple values it will use straight method calls. +// +// To ensure default values, the struct must have been initialized with NewGetFlushCacheParams() beforehand. +func (o *GetFlushCacheParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { + var res []error + + o.HTTPRequest = r + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/swagger_gen/restapi/operations/app/get_flush_cache_responses.go b/swagger_gen/restapi/operations/app/get_flush_cache_responses.go new file mode 100644 index 0000000..8d824ee --- /dev/null +++ b/swagger_gen/restapi/operations/app/get_flush_cache_responses.go @@ -0,0 +1,98 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package app + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime" + + "github.com/Alayacare/goliac/swagger_gen/models" +) + +// GetFlushCacheOKCode is the HTTP code returned for type GetFlushCacheOK +const GetFlushCacheOKCode int = 200 + +/* +GetFlushCacheOK cache flushed + +swagger:response getFlushCacheOK +*/ +type GetFlushCacheOK struct { +} + +// NewGetFlushCacheOK creates GetFlushCacheOK with default headers values +func NewGetFlushCacheOK() *GetFlushCacheOK { + + return &GetFlushCacheOK{} +} + +// WriteResponse to the client +func (o *GetFlushCacheOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses + + rw.WriteHeader(200) +} + +/* +GetFlushCacheDefault generic error response + +swagger:response getFlushCacheDefault +*/ +type GetFlushCacheDefault struct { + _statusCode int + + /* + In: Body + */ + Payload *models.Error `json:"body,omitempty"` +} + +// NewGetFlushCacheDefault creates GetFlushCacheDefault with default headers values +func NewGetFlushCacheDefault(code int) *GetFlushCacheDefault { + if code <= 0 { + code = 500 + } + + return &GetFlushCacheDefault{ + _statusCode: code, + } +} + +// WithStatusCode adds the status to the get flush cache default response +func (o *GetFlushCacheDefault) WithStatusCode(code int) *GetFlushCacheDefault { + o._statusCode = code + return o +} + +// SetStatusCode sets the status to the get flush cache default response +func (o *GetFlushCacheDefault) SetStatusCode(code int) { + o._statusCode = code +} + +// WithPayload adds the payload to the get flush cache default response +func (o *GetFlushCacheDefault) WithPayload(payload *models.Error) *GetFlushCacheDefault { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the get flush cache default response +func (o *GetFlushCacheDefault) SetPayload(payload *models.Error) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *GetFlushCacheDefault) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(o._statusCode) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} diff --git a/swagger_gen/restapi/operations/app/get_flush_cache_urlbuilder.go b/swagger_gen/restapi/operations/app/get_flush_cache_urlbuilder.go new file mode 100644 index 0000000..9330ed4 --- /dev/null +++ b/swagger_gen/restapi/operations/app/get_flush_cache_urlbuilder.go @@ -0,0 +1,87 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package app + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "errors" + "net/url" + golangswaggerpaths "path" +) + +// GetFlushCacheURL generates an URL for the get flush cache operation +type GetFlushCacheURL struct { + _basePath string +} + +// WithBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *GetFlushCacheURL) WithBasePath(bp string) *GetFlushCacheURL { + o.SetBasePath(bp) + return o +} + +// SetBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *GetFlushCacheURL) SetBasePath(bp string) { + o._basePath = bp +} + +// Build a url path and query string +func (o *GetFlushCacheURL) Build() (*url.URL, error) { + var _result url.URL + + var _path = "/flushcache" + + _basePath := o._basePath + if _basePath == "" { + _basePath = "/api/v1" + } + _result.Path = golangswaggerpaths.Join(_basePath, _path) + + return &_result, nil +} + +// Must is a helper function to panic when the url builder returns an error +func (o *GetFlushCacheURL) Must(u *url.URL, err error) *url.URL { + if err != nil { + panic(err) + } + if u == nil { + panic("url can't be nil") + } + return u +} + +// String returns the string representation of the path with query string +func (o *GetFlushCacheURL) String() string { + return o.Must(o.Build()).String() +} + +// BuildFull builds a full url with scheme, host, path and query string +func (o *GetFlushCacheURL) BuildFull(scheme, host string) (*url.URL, error) { + if scheme == "" { + return nil, errors.New("scheme is required for a full url on GetFlushCacheURL") + } + if host == "" { + return nil, errors.New("host is required for a full url on GetFlushCacheURL") + } + + base, err := o.Build() + if err != nil { + return nil, err + } + + base.Scheme = scheme + base.Host = host + return base, nil +} + +// StringFull returns the string representation of a complete url +func (o *GetFlushCacheURL) StringFull(scheme, host string) string { + return o.Must(o.BuildFull(scheme, host)).String() +} diff --git a/swagger_gen/restapi/operations/goliac_api.go b/swagger_gen/restapi/operations/goliac_api.go index 0b9a677..e66f2fa 100644 --- a/swagger_gen/restapi/operations/goliac_api.go +++ b/swagger_gen/restapi/operations/goliac_api.go @@ -19,6 +19,7 @@ import ( "github.com/go-openapi/strfmt" "github.com/go-openapi/swag" + "github.com/Alayacare/goliac/swagger_gen/restapi/operations/app" "github.com/Alayacare/goliac/swagger_gen/restapi/operations/health" ) @@ -44,6 +45,9 @@ func NewGoliacAPI(spec *loads.Document) *GoliacAPI { JSONProducer: runtime.JSONProducer(), + AppGetFlushCacheHandler: app.GetFlushCacheHandlerFunc(func(params app.GetFlushCacheParams) middleware.Responder { + return middleware.NotImplemented("operation app.GetFlushCache has not yet been implemented") + }), HealthGetLivenessHandler: health.GetLivenessHandlerFunc(func(params health.GetLivenessParams) middleware.Responder { return middleware.NotImplemented("operation health.GetLiveness has not yet been implemented") }), @@ -87,6 +91,8 @@ type GoliacAPI struct { // - application/json JSONProducer runtime.Producer + // AppGetFlushCacheHandler sets the operation handler for the get flush cache operation + AppGetFlushCacheHandler app.GetFlushCacheHandler // HealthGetLivenessHandler sets the operation handler for the get liveness operation HealthGetLivenessHandler health.GetLivenessHandler // HealthGetReadinessHandler sets the operation handler for the get readiness operation @@ -168,6 +174,9 @@ func (o *GoliacAPI) Validate() error { unregistered = append(unregistered, "JSONProducer") } + if o.AppGetFlushCacheHandler == nil { + unregistered = append(unregistered, "app.GetFlushCacheHandler") + } if o.HealthGetLivenessHandler == nil { unregistered = append(unregistered, "health.GetLivenessHandler") } @@ -262,6 +271,10 @@ func (o *GoliacAPI) initHandlerCache() { o.handlers = make(map[string]map[string]http.Handler) } + if o.handlers["GET"] == nil { + o.handlers["GET"] = make(map[string]http.Handler) + } + o.handlers["GET"]["/flushcache"] = app.NewGetFlushCache(o.context, o.AppGetFlushCacheHandler) if o.handlers["GET"] == nil { o.handlers["GET"] = make(map[string]http.Handler) }