fix: compliance evaluation search cmd, searches greater than last 7 d…

…ays (#977)

* fix: compliance evaluation search cmd, searches greater than last 7 days

api/compliance_evaluations.go

@@ -18,6 +18,8 @@
 
 package api
 
+import "time"
+
 type ComplianceEvaluationService struct {
 	client *Client
 }
@@ -45,10 +47,22 @@ const AwsComplianceEvaluationDataset complianceEvaluationDataset = "AwsComplianc
 //  )
 //   lacework.V2.ComplianceEvaluation.Search(&awsComplianceEvaluationSearchResponse, filters)
 //
-func (svc *ComplianceEvaluationService) Search(response interface{}, filters ComplianceEvaluationSearch) error {
+func (svc *ComplianceEvaluationService) Search(response interface{}, filters SearchableFilter) error {
 	return svc.client.RequestEncoderDecoder("POST", apiV2ComplianceEvaluationsSearch, filters, response)
 }
 
+func (c *ComplianceEvaluationSearch) GetTimeFilter() *TimeFilter {
+	return c.TimeFilter
+}
+
+func (c *ComplianceEvaluationSearch) SetStartTime(t *time.Time) {
+	c.TimeFilter.StartTime = t
+}
+
+func (c *ComplianceEvaluationSearch) SetEndTime(t *time.Time) {
+	c.TimeFilter.EndTime = t
+}
+
 type ComplianceEvaluationSearch struct {
 	SearchFilter
 	Dataset complianceEvaluationDataset `json:"dataset"`

api/compliance_evaluations_aws.go

@@ -25,6 +25,10 @@ type ComplianceEvaluationAwsResponse struct {
 	Paging V2Pagination              `json:"paging"`
 }
 
+func (r ComplianceEvaluationAwsResponse) GetDataLength() int {
+	return len(r.Data)
+}
+
 func (r ComplianceEvaluationAwsResponse) PageInfo() *V2Pagination {
 	return &r.Paging
 }

api/inventory.go

@@ -18,6 +18,8 @@
 
 package api
 
+import "time"
+
 type InventoryService struct {
 	client *Client
 }
@@ -47,7 +49,7 @@ const AwsInventoryDataset inventoryDataset = "AwsCompliance"
 //  )
 //   lacework.V2.Inventory.Search(&awsInventorySearchResponse, filters)
 //
-func (svc *InventoryService) Search(response interface{}, filters InventorySearch) error {
+func (svc *InventoryService) Search(response interface{}, filters SearchableFilter) error {
 	return svc.client.RequestEncoderDecoder("POST", apiV2InventorySearch, filters, response)
 }
 
@@ -56,3 +58,15 @@ type InventorySearch struct {
 	Csp     inventoryType    `json:"csp"`
 	Dataset inventoryDataset `json:"dataset"`
 }
+
+func (i InventorySearch) GetTimeFilter() *TimeFilter {
+	return i.TimeFilter
+}
+
+func (i InventorySearch) SetStartTime(time *time.Time) {
+	i.TimeFilter.StartTime = time
+}
+
+func (i InventorySearch) SetEndTime(time *time.Time) {
+	i.TimeFilter.EndTime = time
+}

api/inventory_aws.go

@@ -23,6 +23,10 @@ type InventoryAwsResponse struct {
 	Paging V2Pagination   `json:"paging"`
 }
 
+func (r InventoryAwsResponse) GetDataLength() int {
+	return len(r.Data)
+}
+
 func (r InventoryAwsResponse) PageInfo() *V2Pagination {
 	return &r.Paging
 }

api/v2_search_filters.go

@@ -18,7 +18,11 @@
 
 package api
 
-import "time"
+import (
+	"errors"
+	"math"
+	"time"
+)
 
 // SearchFilter is the representation of an advanced search payload
 // for retrieving information out of the Lacework APIv2 Server
@@ -54,3 +58,60 @@ type TimeFilter struct {
 	StartTime *time.Time `json:"startTime,omitempty"`
 	EndTime   *time.Time `json:"endTime,omitempty"`
 }
+
+type SearchResponse interface {
+	GetDataLength() int
+}
+
+type SearchableFilter interface {
+	GetTimeFilter() *TimeFilter
+	SetStartTime(*time.Time)
+	SetEndTime(*time.Time)
+}
+
+// V2ApiMaxSearchHistoryDays defines the maximum number of days in the past api v2 allows to be searched
+const V2ApiMaxSearchHistoryDays = 92
+
+// V2ApiMaxSearchWindowDays defines the maximum number of days in a single request api v2 allows to be searched
+const V2ApiMaxSearchWindowDays = 7
+
+type search func(response interface{}, filters SearchableFilter) error
+
+// WindowedSearchFirst performs a new search of a specific time frame size,
+// until response data is found or the max searchable days is reached
+func WindowedSearchFirst(fn search, size int, max int, response SearchResponse, filter SearchableFilter) error {
+	if size > max {
+		return errors.New("window size cannot be greater than max history")
+	}
+
+	// if start and end time are the same, adjust the windows
+	timeDifference := int(math.RoundToEven(filter.GetTimeFilter().EndTime.Sub(*filter.GetTimeFilter().StartTime).Hours() / 24))
+
+	if timeDifference == 0 {
+		newStart := filter.GetTimeFilter().StartTime.AddDate(0, 0, -size)
+		filter.SetStartTime(&newStart)
+	}
+
+	for i := timeDifference; i < max; i += size {
+		err := fn(&response, filter)
+		if err != nil {
+			return err
+		}
+		if response.GetDataLength() != 0 {
+			return nil
+		}
+
+		// adjust window
+		newStart := filter.GetTimeFilter().StartTime.AddDate(0, 0, -size)
+		newEnd := filter.GetTimeFilter().EndTime.AddDate(0, 0, -size)
+
+		// ensure we do not go over the max allowed searchable days
+		rem := (max - i) % size
+		if rem > 0 {
+			newStart = filter.GetTimeFilter().StartTime.AddDate(0, 0, -rem)
+		}
+		filter.SetStartTime(&newStart)
+		filter.SetEndTime(&newEnd)
+	}
+	return nil
+}

api/v2_search_filters_test.go

@@ -0,0 +1,214 @@
+package api_test
+
+import (
+	"encoding/json"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/lacework/go-sdk/api"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestWindowedSearch test basic functionality of windowedSearch
+// windowed search takes a function of type search and filter that implements
+// searchable filter
+func TestWindowedSearch(t *testing.T) {
+	var (
+		now          = time.Now().UTC()
+		before       = now.AddDate(0, 0, -7) // last 7 days
+		testResponse mockSearchResponse
+		filter       = mockSearchFilter{
+			SearchFilter: api.SearchFilter{
+				Filters: []api.Filter{{
+					Expression: "eq",
+					Field:      "urn",
+					Value:      "text",
+				}},
+				TimeFilter: &api.TimeFilter{
+					StartTime: &before,
+					EndTime:   &now,
+				},
+			},
+			Value: "MOCK",
+		}
+	)
+	searchCounter = 0
+	err := api.WindowedSearchFirst(mockSearch, api.V2ApiMaxSearchWindowDays, api.V2ApiMaxSearchHistoryDays, &testResponse, &filter)
+	assert.NoError(t, err)
+	assert.Equal(t, searchCounter, 3)
+	assert.NotEmpty(t, testResponse.Data)
+	assert.Equal(t, testResponse.Data[0].ID, "MOCK_1")
+	assert.Equal(t, testResponse.Data[0].Value, "EXAMPLE_VALUE_1")
+
+	// startTime from current Date should be less than max history
+	timeDifference := int(math.RoundToEven(time.Now().Sub(*filter.GetTimeFilter().StartTime).Hours() / 24))
+	assert.Less(t, timeDifference, api.V2ApiMaxSearchHistoryDays)
+}
+
+// TestWindowedSearchMaxHistory test max history is not exceeded
+func TestWindowedSearchMaxHistory(t *testing.T) {
+	var (
+		// set max history and window size. searchCounter should not exceed 1
+		maxWindow    = 5
+		maxHistory   = 10
+		now          = time.Now().UTC()
+		before       = now.AddDate(0, 0, -5) // last 5 days
+		testResponse mockSearchResponse
+		filter       = mockSearchFilter{
+			SearchFilter: api.SearchFilter{
+				TimeFilter: &api.TimeFilter{
+					StartTime: &before,
+					EndTime:   &now,
+				},
+			},
+			Value: "MOCK",
+		}
+	)
+
+	searchCounter = 0
+	err := api.WindowedSearchFirst(mockSearch, maxWindow, maxHistory, &testResponse, &filter)
+	assert.NoError(t, err)
+
+	// search counter should equal 1. No data is found in search
+	assert.Equal(t, searchCounter, 1)
+	assert.Empty(t, testResponse.Data)
+
+	// startTime from current date should be less or equal to max history
+	timeDifference := int(math.RoundToEven(time.Now().Sub(*filter.GetTimeFilter().StartTime).Hours() / 24))
+	assert.Equal(t, timeDifference, maxHistory)
+}
+
+// TestWindowedSearchMaxHistory test max window is greater than history
+// returns error
+func TestWindowedSearchHistory(t *testing.T) {
+	var (
+		// set max history and window size. Window cannot exceed history size
+		maxWindow    = 10
+		maxHistory   = 5
+		now          = time.Now().UTC()
+		before       = now.AddDate(0, 0, -10) // last 10 days
+		testResponse mockSearchResponse
+		filter       = mockSearchFilter{
+			SearchFilter: api.SearchFilter{
+				TimeFilter: &api.TimeFilter{
+					StartTime: &before,
+					EndTime:   &now,
+				},
+			},
+			Value: "MOCK",
+		}
+	)
+
+	err := api.WindowedSearchFirst(mockSearch, maxWindow, maxHistory, &testResponse, &filter)
+	assert.ErrorContains(t, err, "window size cannot be greater than max history")
+}
+
+// TestWindowedSearchMaxHistory test history is not divisible by window that final search only searches remainder
+// the final search only searches remainder
+func TestWindowedSearchHistoryRemainder(t *testing.T) {
+	var (
+		// set max history and window size. searchCounter should not exceed 1
+		maxWindow    = 3
+		maxHistory   = 5
+		now          = time.Now().UTC()
+		before       = now.AddDate(0, 0, -3) // last 3 days
+		testResponse mockSearchResponse
+		filter       = mockSearchFilter{
+			SearchFilter: api.SearchFilter{
+				TimeFilter: &api.TimeFilter{
+					StartTime: &before,
+					EndTime:   &now,
+				},
+			},
+			Value: "MOCK",
+		}
+	)
+
+	searchCounter = 0
+	err := api.WindowedSearchFirst(mockSearch, maxWindow, maxHistory, &testResponse, &filter)
+	assert.NoError(t, err)
+
+	// search counter should equal 1. No data is found in search
+	assert.Equal(t, searchCounter, 1)
+	assert.Empty(t, testResponse.Data)
+
+	// the time difference on final search should adjust as to not exceed the max history
+	finalSearchEndTime := filter.GetTimeFilter().EndTime
+	finalSearchStartTime := filter.GetTimeFilter().StartTime
+	timeDifference := int(math.RoundToEven(finalSearchEndTime.Sub(*finalSearchStartTime).Hours() / 24))
+	assert.Equal(t, 2, timeDifference)
+
+	// startTime from current date should be less than max history
+	maxTimeDifference := int(math.RoundToEven(filter.GetTimeFilter().StartTime.Sub(time.Now()).Hours() / 24))
+	assert.Less(t, maxTimeDifference, maxHistory)
+}
+
+func mockSearch(response interface{}, filters api.SearchableFilter) error {
+	// simulate the search item not being found in first windows
+	mockResponse := mockSearchResponseJson
+	if searchCounter < 3 {
+		mockResponse = mockSearchResponseEmptyJson
+		searchCounter++
+	}
+
+	err := json.Unmarshal([]byte(mockResponse), &response)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+var searchCounter int
+
+type mockSearchResponse struct {
+	Data []mockSearchResponseData `json:"data"`
+}
+
+type mockSearchResponseData struct {
+	ID    string `json:"id"`
+	Value string `json:"value"`
+}
+
+func (m mockSearchResponse) GetDataLength() int {
+	return len(m.Data)
+}
+
+type mockSearchFilter struct {
+	api.SearchFilter
+	Value string `json:"val"`
+}
+
+func (m mockSearchFilter) GetTimeFilter() *api.TimeFilter {
+	return m.TimeFilter
+
+}
+
+func (m mockSearchFilter) SetStartTime(t *time.Time) {
+	m.TimeFilter.StartTime = t
+}
+
+func (m mockSearchFilter) SetEndTime(t *time.Time) {
+	m.TimeFilter.EndTime = t
+}
+
+var mockSearchResponseJson = `
+{
+  "data": [
+    {
+      "id": "MOCK_1",
+      "value": "EXAMPLE_VALUE_1" 
+    },
+    {
+      "id": "MOCK_2",
+      "value": "EXAMPLE_VALUE_2" 
+    }
+  ]
+}
+`
+
+var mockSearchResponseEmptyJson = `
+{
+  "data": []
+}
+`

cli/cmd/compliance_aws.go

@@ -448,7 +448,7 @@ The output from status with the --json flag can be used in the body of PATCH api
 					Csp:     api.AwsInventoryType,
 				}
 			)
-			err := cli.LwApi.V2.Inventory.Search(&awsInventorySearchResponse, filter)
+			err := api.WindowedSearchFirst(cli.LwApi.V2.Inventory.Search, api.V2ApiMaxSearchWindowDays, api.V2ApiMaxSearchHistoryDays, &awsInventorySearchResponse, &filter)
 			cli.StopProgress()
 
 			if len(awsInventorySearchResponse.Data) == 0 {
@@ -479,7 +479,8 @@ The output from status with the --json flag can be used in the body of PATCH api
 					Dataset: api.AwsComplianceEvaluationDataset,
 				}
 			)
-			err = cli.LwApi.V2.ComplianceEvaluations.Search(&awsComplianceEvaluationSearchResponse, complianceFilter)
+
+			err = api.WindowedSearchFirst(cli.LwApi.V2.ComplianceEvaluations.Search, api.V2ApiMaxSearchWindowDays, api.V2ApiMaxSearchHistoryDays, &awsComplianceEvaluationSearchResponse, &complianceFilter)
 			cli.StopProgress()
 			if err != nil {
 				return err