feat(api): v2PageMetadata to log page information

This new type is used to compute the total pages and the page number when reading pages using the `client.NextPage()` function Inside the Pageable interface, there are a few new functions that are automatically implemented when attaching the `v2PageMetadata` type into any Pageable struct, so attaching that struct is a requirement. Signed-off-by: Salim Afiune Maya <[email protected]>
lacework · Feb 1, 2023 · fd5338b · fd5338b
1 parent d3cb69a
commit fd5338b
Show file tree

Hide file tree

Showing 12 changed files with 94 additions and 38 deletions.
diff --git a/api/agent_info.go b/api/agent_info.go
@@ -35,6 +35,8 @@ func (svc *AgentInfoService) Search(response interface{}, filters SearchFilter)
 type AgentInfoResponse struct {
 	Data   []AgentInfo  `json:"data"`
 	Paging V2Pagination `json:"paging"`
+
+	v2PageMetadata `json:"-"`
 }
 
 // Fulfill Pageable interface (look at api/v2.go)
@@ -43,6 +45,7 @@ func (r AgentInfoResponse) PageInfo() *V2Pagination {
 }
 func (r *AgentInfoResponse) ResetPaging() {
 	r.Paging = V2Pagination{}
+	r.Data = nil
 }
 
 type AgentInfo struct {

diff --git a/api/alerts.go b/api/alerts.go
@@ -92,6 +92,8 @@ func (a Alerts) SortBySeverity() {
 type AlertsResponse struct {
 	Data   Alerts       `json:"data"`
 	Paging V2Pagination `json:"paging"`
+
+	v2PageMetadata `json:"-"`
 }
 
 // Fulfill Pageable interface (look at api/v2.go)
@@ -100,6 +102,7 @@ func (r AlertsResponse) PageInfo() *V2Pagination {
 }
 func (r *AlertsResponse) ResetPaging() {
 	r.Paging = V2Pagination{}
+	r.Data = nil
 }
 
 func (svc *AlertsService) List() (response AlertsResponse, err error) {
@@ -127,8 +130,8 @@ func (svc *AlertsService) ListAll() (response AlertsResponse, err error) {
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
 
@@ -172,7 +175,7 @@ func (svc *AlertsService) ListAllByTime(start, end time.Time) (
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
diff --git a/api/alerts_search.go b/api/alerts_search.go
@@ -62,7 +62,7 @@ func (svc *AlertsService) SearchAll(filter SearchFilter) (
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
diff --git a/api/compliance_evaluations_aws.go b/api/compliance_evaluations_aws.go
@@ -34,6 +34,7 @@ func (r ComplianceEvaluationAwsResponse) PageInfo() *V2Pagination {
 }
 func (r *ComplianceEvaluationAwsResponse) ResetPaging() {
 	r.Paging = V2Pagination{}
+	r.Data = nil
 }
 
 type ComplianceEvaluationAws struct {

diff --git a/api/entities_containers.go b/api/entities_containers.go
@@ -59,20 +59,27 @@ func (svc *EntitiesService) ListAllContainers() (response ContainersEntityRespon
 	for {
 		all = append(all, response.Data...)
 
-		pageOk, err = svc.client.NextPage(&response)
+		newResponse := ContainersEntityResponse{
+			Paging: response.Paging,
+		}
+		pageOk, err = svc.client.NextPage(&newResponse)
 		if err == nil && pageOk {
+			response = newResponse
 			continue
 		}
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
 
-// ListAllContainersWithFilters iterates over all pages to return all active container information at once based on a user defined filter
-func (svc *EntitiesService) ListAllContainersWithFilters(filters SearchFilter) (response ContainersEntityResponse, err error) {
+// ListAllContainersWithFilters iterates over all pages to return all active container
+// information at once based on a user defined filter
+func (svc *EntitiesService) ListAllContainersWithFilters(filters SearchFilter) (
+	response ContainersEntityResponse, err error,
+) {
 	response, err = svc.ListContainersWithFilters(filters)
 	if err != nil {
 		return
@@ -93,14 +100,16 @@ func (svc *EntitiesService) ListAllContainersWithFilters(filters SearchFilter) (
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
 
 type ContainersEntityResponse struct {
 	Data   []ContainerEntity `json:"data"`
 	Paging V2Pagination      `json:"paging"`
+
+	v2PageMetadata `json:"-"`
 }
 
 // Fulfill Pageable interface (look at api/v2.go)
@@ -109,6 +118,7 @@ func (r ContainersEntityResponse) PageInfo() *V2Pagination {
 }
 func (r *ContainersEntityResponse) ResetPaging() {
 	r.Paging = V2Pagination{}
+	r.Data = nil
 }
 
 // Total returns the total number of active containers

diff --git a/api/entities_images.go b/api/entities_images.go
@@ -62,8 +62,8 @@ func (svc *EntitiesService) ListAllImages() (response ImagesEntityResponse, err
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
 
@@ -89,14 +89,16 @@ func (svc *EntitiesService) ListAllImagesWithFilters(filters SearchFilter) (resp
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
 
 type ImagesEntityResponse struct {
 	Data   []ImageEntity `json:"data"`
 	Paging V2Pagination  `json:"paging"`
+
+	v2PageMetadata `json:"-"`
 }
 
 // Fulfill Pageable interface (look at api/v2.go)
@@ -105,6 +107,7 @@ func (r ImagesEntityResponse) PageInfo() *V2Pagination {
 }
 func (r *ImagesEntityResponse) ResetPaging() {
 	r.Paging = V2Pagination{}
+	r.Data = nil
 }
 
 type ImageEntity struct {

diff --git a/api/entities_machine_details.go b/api/entities_machine_details.go
@@ -64,8 +64,8 @@ func (svc *EntitiesService) ListAllMachineDetails() (response MachineDetailsEnti
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
 
@@ -91,14 +91,16 @@ func (svc *EntitiesService) ListAllMachineDetailsWithFilters(filters SearchFilte
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
 
 type MachineDetailsEntityResponse struct {
 	Data   []MachineDetailEntity `json:"data"`
 	Paging V2Pagination          `json:"paging"`
+
+	v2PageMetadata `json:"-"`
 }
 
 // Fulfill Pageable interface (look at api/v2.go)
@@ -107,6 +109,7 @@ func (r MachineDetailsEntityResponse) PageInfo() *V2Pagination {
 }
 func (r *MachineDetailsEntityResponse) ResetPaging() {
 	r.Paging = V2Pagination{}
+	r.Data = nil
 }
 
 type MachineDetailEntity struct {

diff --git a/api/entities_users.go b/api/entities_users.go
@@ -56,14 +56,16 @@ func (svc *EntitiesService) ListAllUsers() (response UsersEntityResponse, err er
 		break
 	}
 
-	response.Data = all
 	response.ResetPaging()
+	response.Data = all
 	return
 }
 
 type UsersEntityResponse struct {
 	Data   []UserEntity `json:"data"`
 	Paging V2Pagination `json:"paging"`
+
+	v2PageMetadata `json:"-"`
 }
 
 // Fulfill Pagination interface (look at api/v2.go)
@@ -72,6 +74,7 @@ func (r UsersEntityResponse) PageInfo() *V2Pagination {
 }
 func (r *UsersEntityResponse) ResetPaging() {
 	r.Paging = V2Pagination{}
+	r.Data = nil
 }
 
 type UserEntity struct {

diff --git a/api/inventory_aws.go b/api/inventory_aws.go
@@ -32,6 +32,7 @@ func (r InventoryAwsResponse) PageInfo() *V2Pagination {
 }
 func (r *InventoryAwsResponse) ResetPaging() {
 	r.Paging = V2Pagination{}
+	r.Data = nil
 }
 
 type InventoryAws struct {

diff --git a/api/v2.go b/api/v2.go
@@ -140,11 +140,39 @@ type V2Pagination struct {
 	} `json:"urls"`
 }
 
+// v2PageMetadata is used to compute the total pages and the page number
+// when reading pages using the client.NextPage() function
+type v2PageMetadata struct {
+	totalPages int
+	pageNumber int
+}
+
+func (m v2PageMetadata) PageNumber() int {
+	return m.pageNumber
+}
+func (m v2PageMetadata) TotalPages() int {
+	return m.totalPages
+}
+func (m *v2PageMetadata) SetTotalPages(total int) {
+	m.totalPages = total
+}
+func (m *v2PageMetadata) PageRead() {
+	m.pageNumber++
+}
+
 // Pageable is the interface that structs should implement to become
 // pageable and be able to use the client.NextPage() function
 type Pageable interface {
 	PageInfo() *V2Pagination
 	ResetPaging()
+
+	// all these functions are automatically implemented when attaching
+	// the v2PageMetadata type into any Pageable struct, so attaching that
+	// struct is a requirement
+	PageRead()
+	SetTotalPages(int)
+	TotalPages() int
+	PageNumber() int
 }
 
 // NextPage
@@ -188,14 +216,24 @@ func (c *Client) NextPage(p Pageable) (bool, error) {
 		return false, nil
 	}
 
-	c.log.Info("pagination", zap.Int("rows", pagination.Rows),
-		zap.Int("total_rows", pagination.TotalRows),
-		zap.String("next_page", pagination.Urls.NextPage),
-	)
 	if pagination.Urls.NextPage == "" {
 		return false, nil
 	}
 
+	if p.PageNumber() == 0 {
+		// first page, initialize pagination metadata
+		p.SetTotalPages(pagination.TotalRows / pagination.Rows)
+		p.PageRead()
+	}
+
+	c.log.Info("pagination",
+		zap.Int("page_number", p.PageNumber()),
+		zap.Int("total_pages", p.TotalPages()),
+		zap.Int("rows", pagination.Rows),
+		zap.Int("total_rows", pagination.TotalRows),
+		zap.String("next_page", pagination.Urls.NextPage),
+	)
+
 	pageURL, err := url.Parse(pagination.Urls.NextPage)
 	if err != nil {
 		return false, errors.Wrap(err, "unable to part next page url")
@@ -209,5 +247,6 @@ func (c *Client) NextPage(p Pageable) (bool, error) {
 	p.ResetPaging()
 	c.log.Info("pagination reset")
 	err = c.RequestDecoder("GET", path, nil, p)
+	p.PageRead()
 	return true, err
 }