From e6ab56c00fa70fe13d961b56519e20d80a1ef5ac Mon Sep 17 00:00:00 2001 From: "Baruch Odem (Rothkoff)" Date: Wed, 26 Jul 2023 16:19:01 +0300 Subject: [PATCH] fix: confluence anonymous access is failing after few requests (#146) Fixes #144 The error is not always reproduced, and when it is, it is reproduced for a different page every time. Since the error is `500`, means that it is a server (Confluence) error, I can't fix it and I must work around it. I added a retry mechanism and I enabled it only for our request, when it returns `500`, it will retry it 3 times. It looks like this: ``` 11:38:19 WRN confluence credentials were not provided. The scan will be made anonymously only for the public pages 11:38:20 INF Total of all 4 Spaces detected 11:38:20 INF Space - secrets have 3 pages 11:38:20 INF Space - Secrets have 1 pages 11:38:20 INF Space - Checkmarx Resources have 3 pages 11:38:29 INF Space - Checkmarx Knowledge Center have 1231 pages 11:38:37 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1375995197?expand=body.storage,version,history.previousVersion 11:38:37 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1312260170?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1188135832?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2093908162?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1829044450?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1534427435?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1383432225?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1191609716?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1636598107?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2031092718?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1998946990?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1995211335?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2100003058?expand=body.storage,version,history.previousVersion 11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1192362348?expand=body.storage,version,history.previousVersion 11:38:40 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1169195803?expand=body.storage,version,history.previousVersion 11:38:40 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1131742701?expand=body.storage,version,history.previousVersion 11:38:40 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1947631705?expand=body.storage,version,history.previousVersion 11:38:40 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1158283441?expand=body.storage,version,history.previousVersion 11:38:41 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2490630327?expand=body.storage,version,history.previousVersion 11:38:42 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2600012551?expand=body.storage,version,history.previousVersion 11:38:39 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1186267153?expand=body.storage,version,history.previousVersion exit status 1 Summary: totalitemsscanned: 1238 totalsecretsfound: 6 ``` We can see the `500` error code happens once a time and then all the requests will return with `500`. --- lib/http.go | 21 ++++++++++++++++++--- plugins/confluence.go | 6 +++--- plugins/paligo.go | 2 +- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/lib/http.go b/lib/http.go index e8c75ba0..3d3c9666 100644 --- a/lib/http.go +++ b/lib/http.go @@ -5,6 +5,8 @@ import ( "fmt" "io" "net/http" + + "github.com/rs/zerolog/log" ) type ICredentials interface { @@ -20,14 +22,19 @@ type IAuthorizationHeader interface { GetAuthorizationHeader() string } -func HttpRequest(method string, url string, autherization IAuthorizationHeader) ([]byte, *http.Response, error) { +type RetrySettings struct { + MaxRetries int + ErrorCodes []int +} + +func HttpRequest(method string, url string, authorization IAuthorizationHeader, retry RetrySettings) ([]byte, *http.Response, error) { request, err := http.NewRequest(method, url, nil) if err != nil { return nil, nil, fmt.Errorf("unexpected error creating an http request %w", err) } - if autherization.GetAuthorizationHeader() != "" { - request.Header.Set("Authorization", autherization.GetAuthorizationHeader()) + if authorization.GetAuthorizationHeader() != "" { + request.Header.Set("Authorization", authorization.GetAuthorizationHeader()) } client := &http.Client{} @@ -39,6 +46,14 @@ func HttpRequest(method string, url string, autherization IAuthorizationHeader) defer response.Body.Close() if response.StatusCode < 200 || response.StatusCode >= 300 { + if retry.MaxRetries > 0 { + for _, code := range retry.ErrorCodes { + if response.StatusCode == code { + log.Warn().Msgf("retrying http request %v", url) + return HttpRequest(method, url, authorization, RetrySettings{MaxRetries: retry.MaxRetries - 1, ErrorCodes: retry.ErrorCodes}) + } + } + } return nil, response, fmt.Errorf("error calling http url \"%v\". status code: %v", url, response) } diff --git a/plugins/confluence.go b/plugins/confluence.go index 17af6c3f..05f00fec 100644 --- a/plugins/confluence.go +++ b/plugins/confluence.go @@ -165,7 +165,7 @@ func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) { func (p *ConfluencePlugin) getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) { url := fmt.Sprintf("%s/rest/api/space?start=%d", p.URL, start) - body, _, err := lib.HttpRequest(http.MethodGet, url, p) + body, _, err := lib.HttpRequest(http.MethodGet, url, p, lib.RetrySettings{}) if err != nil { return nil, fmt.Errorf("unexpected error creating an http request %w", err) } @@ -206,7 +206,7 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePag func (p *ConfluencePlugin) getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) { url := fmt.Sprintf("%s/rest/api/space/%s/content?start=%d", p.URL, space.Key, start) - body, _, err := lib.HttpRequest(http.MethodGet, url, p) + body, _, err := lib.HttpRequest(http.MethodGet, url, p, lib.RetrySettings{}) if err != nil { return nil, fmt.Errorf("unexpected error creating an http request %w", err) @@ -253,7 +253,7 @@ func (p *ConfluencePlugin) getItem(page ConfluencePage, space ConfluenceSpaceRes url = fmt.Sprintf("%s/rest/api/content/%s?status=historical&version=%d&expand=body.storage,version,history.previousVersion", p.URL, page.ID, version) } - request, _, err := lib.HttpRequest(http.MethodGet, url, p) + request, _, err := lib.HttpRequest(http.MethodGet, url, p, lib.RetrySettings{MaxRetries: 3, ErrorCodes: []int{500}}) if err != nil { return nil, 0, fmt.Errorf("unexpected error creating an http request %w", err) } diff --git a/plugins/paligo.go b/plugins/paligo.go index 9503a80e..3f2a28d1 100644 --- a/plugins/paligo.go +++ b/plugins/paligo.go @@ -284,7 +284,7 @@ func (p *PaligoClient) request(endpoint string, lim *rate.Limiter) ([]byte, erro } url := fmt.Sprintf("https://%s.paligoapp.com/api/v2/%s", p.Instance, endpoint) - body, response, err := lib.HttpRequest("GET", url, p.auth) + body, response, err := lib.HttpRequest("GET", url, p.auth, lib.RetrySettings{}) if err != nil { if err := reserveRateLimit(response, lim, err); err != nil { return nil, err