feat: paligo plugin (#78)

- get folders and components
- handle only `component` and `folder` types
- send content to items
- avoid reaching the rate limit

Note we are handling only `folder`s to find sub-folders or `components`,
and reading only the `component` (document) content.

### Rate Limit

The rate limit **per minute** described
[here](https://paligo.net/docs/apidocs/en/index-en.html#UUID-a5b548af-9a37-d305-f5a8-11142d86fe20).
The core lib `time/rate` is limiting the rate **per second**. That's why
I'm starting with open seats for all the requests-per-minute, in case
there are fewer requests than the limit. After the first
too-many-requests error, I'm reducing the available seats to 1, so the
frequency of the requests will not reach the rate limit.

Close #75

cmd/main.go

@@ -45,6 +45,7 @@ var allPlugins = []plugins.IPlugin{
 	&plugins.DiscordPlugin{},
 	&plugins.RepositoryPlugin{},
 	&plugins.SlackPlugin{},
+	&plugins.PaligoPlugin{},
 }
 
 var channels = plugins.Channels{

go.mod

@@ -5,9 +5,11 @@ go 1.20
 require (
 	github.com/bwmarrin/discordgo v0.27.1
 	github.com/rs/zerolog v1.29.0
+	github.com/slack-go/slack v0.12.2
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.1
 	github.com/zricethezav/gitleaks/v8 v8.16.1
+	golang.org/x/time v0.1.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -35,7 +37,6 @@ require (
 	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/slack-go/slack v0.12.2 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect

go.sum

@@ -384,6 +384,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA=
+golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

lib/http.go

@@ -10,12 +10,10 @@ type ICredentials interface {
 	GetCredentials() (string, string)
 }
 
-func HttpRequest(method string, url string, credentials ICredentials) ([]byte, error) {
-	var err error
-
+func HttpRequest(method string, url string, credentials ICredentials) ([]byte, *http.Response, error) {
 	request, err := http.NewRequest(method, url, nil)
 	if err != nil {
-		return nil, fmt.Errorf("unexpected error creating an http request %w", err)
+		return nil, nil, fmt.Errorf("unexpected error creating an http request %w", err)
 	}
 
 	username, password := credentials.GetCredentials()
@@ -26,19 +24,19 @@ func HttpRequest(method string, url string, credentials ICredentials) ([]byte, e
 	client := &http.Client{}
 	response, err := client.Do(request)
 	if err != nil {
-		return nil, fmt.Errorf("unable to send http request %w", err)
+		return nil, response, fmt.Errorf("unable to send http request %w", err)
 	}
 
 	defer response.Body.Close()
 
 	if response.StatusCode < 200 || response.StatusCode >= 300 {
-		return nil, fmt.Errorf("error calling http url \"%v\". status code: %v", url, response.StatusCode)
+		return nil, response, fmt.Errorf("error calling http url \"%v\". status code: %v", url, response)
 	}
 
 	body, err := io.ReadAll(response.Body)
 	if err != nil {
-		return nil, fmt.Errorf("unexpected error reading http response body %w", err)
+		return nil, response, fmt.Errorf("unexpected error reading http response body %w", err)
 	}
 
-	return body, nil
+	return body, response, nil
 }

main.go

@@ -1,17 +1,18 @@
 package main
 
 import (
-	"github.com/checkmarx/2ms/cmd"
 	"os"
 	"os/signal"
 
+	"github.com/checkmarx/2ms/cmd"
+
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 )
 
 func main() {
 	// send all logs to stdout
-	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "15:04:05"})
 
 	// this block sets up a go routine to listen for an interrupt signal
 	// which will immediately exit gitleaks

plugins/confluence.go

@@ -173,7 +173,7 @@ func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) {
 
 func (p *ConfluencePlugin) getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) {
 	url := fmt.Sprintf("%s/rest/api/space?start=%d", p.URL, start)
-	body, err := lib.HttpRequest(http.MethodGet, url, p)
+	body, _, err := lib.HttpRequest(http.MethodGet, url, p)
 	if err != nil {
 		return nil, fmt.Errorf("unexpected error creating an http request %w", err)
 	}
@@ -214,7 +214,7 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePag
 
 func (p *ConfluencePlugin) getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) {
 	url := fmt.Sprintf("%s/rest/api/space/%s/content?start=%d", p.URL, space.Key, start)
-	body, err := lib.HttpRequest(http.MethodGet, url, p)
+	body, _, err := lib.HttpRequest(http.MethodGet, url, p)
 
 	if err != nil {
 		return nil, fmt.Errorf("unexpected error creating an http request %w", err)
@@ -264,7 +264,7 @@ func (p *ConfluencePlugin) getItem(page ConfluencePage, space ConfluenceSpaceRes
 		originalUrl = fmt.Sprintf("%s/pages/viewpage.action?pageid=%spageVersion=%d", p.URL, page.ID, version)
 	}
 
-	request, err := lib.HttpRequest(http.MethodGet, url, p)
+	request, _, err := lib.HttpRequest(http.MethodGet, url, p)
 	if err != nil {
 		return nil, 0, fmt.Errorf("unexpected error creating an http request %w", err)
 	}