From fd5338b69ac3ea22a611cc57799e4a139c545e8a Mon Sep 17 00:00:00 2001 From: Salim Afiune Maya Date: Tue, 31 Jan 2023 22:27:47 -0600 Subject: [PATCH] feat(api): v2PageMetadata to log page information This new type is used to compute the total pages and the page number when reading pages using the `client.NextPage()` function Inside the Pageable interface, there are a few new functions that are automatically implemented when attaching the `v2PageMetadata` type into any Pageable struct, so attaching that struct is a requirement. Signed-off-by: Salim Afiune Maya --- api/agent_info.go | 3 ++ api/alerts.go | 7 +++-- api/alerts_search.go | 2 +- api/compliance_evaluations_aws.go | 1 + api/entities_containers.go | 20 +++++++++---- api/entities_images.go | 7 +++-- api/entities_machine_details.go | 7 +++-- api/entities_users.go | 5 +++- api/inventory_aws.go | 1 + api/v2.go | 47 ++++++++++++++++++++++++++++--- api/v2_vulnerabilities.go | 30 +++++++------------- cli/cmd/agent_list.go | 2 +- 12 files changed, 94 insertions(+), 38 deletions(-) diff --git a/api/agent_info.go b/api/agent_info.go index 36e335281..6b69370fc 100644 --- a/api/agent_info.go +++ b/api/agent_info.go @@ -35,6 +35,8 @@ func (svc *AgentInfoService) Search(response interface{}, filters SearchFilter) type AgentInfoResponse struct { Data []AgentInfo `json:"data"` Paging V2Pagination `json:"paging"` + + v2PageMetadata `json:"-"` } // Fulfill Pageable interface (look at api/v2.go) @@ -43,6 +45,7 @@ func (r AgentInfoResponse) PageInfo() *V2Pagination { } func (r *AgentInfoResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } type AgentInfo struct { diff --git a/api/alerts.go b/api/alerts.go index 14334acd4..3181b4acd 100644 --- a/api/alerts.go +++ b/api/alerts.go @@ -92,6 +92,8 @@ func (a Alerts) SortBySeverity() { type AlertsResponse struct { Data Alerts `json:"data"` Paging V2Pagination `json:"paging"` + + v2PageMetadata `json:"-"` } // Fulfill Pageable interface (look at api/v2.go) @@ -100,6 +102,7 @@ func (r AlertsResponse) PageInfo() *V2Pagination { } func (r *AlertsResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } func (svc *AlertsService) List() (response AlertsResponse, err error) { @@ -127,8 +130,8 @@ func (svc *AlertsService) ListAll() (response AlertsResponse, err error) { break } - response.Data = all response.ResetPaging() + response.Data = all return } @@ -172,7 +175,7 @@ func (svc *AlertsService) ListAllByTime(start, end time.Time) ( break } - response.Data = all response.ResetPaging() + response.Data = all return } diff --git a/api/alerts_search.go b/api/alerts_search.go index d36ec4526..6c95691aa 100644 --- a/api/alerts_search.go +++ b/api/alerts_search.go @@ -62,7 +62,7 @@ func (svc *AlertsService) SearchAll(filter SearchFilter) ( break } - response.Data = all response.ResetPaging() + response.Data = all return } diff --git a/api/compliance_evaluations_aws.go b/api/compliance_evaluations_aws.go index d57894cf6..f3106f2c1 100644 --- a/api/compliance_evaluations_aws.go +++ b/api/compliance_evaluations_aws.go @@ -34,6 +34,7 @@ func (r ComplianceEvaluationAwsResponse) PageInfo() *V2Pagination { } func (r *ComplianceEvaluationAwsResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } type ComplianceEvaluationAws struct { diff --git a/api/entities_containers.go b/api/entities_containers.go index df0124fec..d52b63a8b 100644 --- a/api/entities_containers.go +++ b/api/entities_containers.go @@ -59,20 +59,27 @@ func (svc *EntitiesService) ListAllContainers() (response ContainersEntityRespon for { all = append(all, response.Data...) - pageOk, err = svc.client.NextPage(&response) + newResponse := ContainersEntityResponse{ + Paging: response.Paging, + } + pageOk, err = svc.client.NextPage(&newResponse) if err == nil && pageOk { + response = newResponse continue } break } - response.Data = all response.ResetPaging() + response.Data = all return } -// ListAllContainersWithFilters iterates over all pages to return all active container information at once based on a user defined filter -func (svc *EntitiesService) ListAllContainersWithFilters(filters SearchFilter) (response ContainersEntityResponse, err error) { +// ListAllContainersWithFilters iterates over all pages to return all active container +// information at once based on a user defined filter +func (svc *EntitiesService) ListAllContainersWithFilters(filters SearchFilter) ( + response ContainersEntityResponse, err error, +) { response, err = svc.ListContainersWithFilters(filters) if err != nil { return @@ -93,14 +100,16 @@ func (svc *EntitiesService) ListAllContainersWithFilters(filters SearchFilter) ( break } - response.Data = all response.ResetPaging() + response.Data = all return } type ContainersEntityResponse struct { Data []ContainerEntity `json:"data"` Paging V2Pagination `json:"paging"` + + v2PageMetadata `json:"-"` } // Fulfill Pageable interface (look at api/v2.go) @@ -109,6 +118,7 @@ func (r ContainersEntityResponse) PageInfo() *V2Pagination { } func (r *ContainersEntityResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } // Total returns the total number of active containers diff --git a/api/entities_images.go b/api/entities_images.go index cf501705b..4efef9d78 100644 --- a/api/entities_images.go +++ b/api/entities_images.go @@ -62,8 +62,8 @@ func (svc *EntitiesService) ListAllImages() (response ImagesEntityResponse, err break } - response.Data = all response.ResetPaging() + response.Data = all return } @@ -89,14 +89,16 @@ func (svc *EntitiesService) ListAllImagesWithFilters(filters SearchFilter) (resp break } - response.Data = all response.ResetPaging() + response.Data = all return } type ImagesEntityResponse struct { Data []ImageEntity `json:"data"` Paging V2Pagination `json:"paging"` + + v2PageMetadata `json:"-"` } // Fulfill Pageable interface (look at api/v2.go) @@ -105,6 +107,7 @@ func (r ImagesEntityResponse) PageInfo() *V2Pagination { } func (r *ImagesEntityResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } type ImageEntity struct { diff --git a/api/entities_machine_details.go b/api/entities_machine_details.go index d035d00cc..f771f774d 100644 --- a/api/entities_machine_details.go +++ b/api/entities_machine_details.go @@ -64,8 +64,8 @@ func (svc *EntitiesService) ListAllMachineDetails() (response MachineDetailsEnti break } - response.Data = all response.ResetPaging() + response.Data = all return } @@ -91,14 +91,16 @@ func (svc *EntitiesService) ListAllMachineDetailsWithFilters(filters SearchFilte break } - response.Data = all response.ResetPaging() + response.Data = all return } type MachineDetailsEntityResponse struct { Data []MachineDetailEntity `json:"data"` Paging V2Pagination `json:"paging"` + + v2PageMetadata `json:"-"` } // Fulfill Pageable interface (look at api/v2.go) @@ -107,6 +109,7 @@ func (r MachineDetailsEntityResponse) PageInfo() *V2Pagination { } func (r *MachineDetailsEntityResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } type MachineDetailEntity struct { diff --git a/api/entities_users.go b/api/entities_users.go index 504d2b1d4..155d2c062 100644 --- a/api/entities_users.go +++ b/api/entities_users.go @@ -56,14 +56,16 @@ func (svc *EntitiesService) ListAllUsers() (response UsersEntityResponse, err er break } - response.Data = all response.ResetPaging() + response.Data = all return } type UsersEntityResponse struct { Data []UserEntity `json:"data"` Paging V2Pagination `json:"paging"` + + v2PageMetadata `json:"-"` } // Fulfill Pagination interface (look at api/v2.go) @@ -72,6 +74,7 @@ func (r UsersEntityResponse) PageInfo() *V2Pagination { } func (r *UsersEntityResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } type UserEntity struct { diff --git a/api/inventory_aws.go b/api/inventory_aws.go index b0280a00b..1440124cd 100644 --- a/api/inventory_aws.go +++ b/api/inventory_aws.go @@ -32,6 +32,7 @@ func (r InventoryAwsResponse) PageInfo() *V2Pagination { } func (r *InventoryAwsResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } type InventoryAws struct { diff --git a/api/v2.go b/api/v2.go index e6efa0916..971e0a008 100644 --- a/api/v2.go +++ b/api/v2.go @@ -140,11 +140,39 @@ type V2Pagination struct { } `json:"urls"` } +// v2PageMetadata is used to compute the total pages and the page number +// when reading pages using the client.NextPage() function +type v2PageMetadata struct { + totalPages int + pageNumber int +} + +func (m v2PageMetadata) PageNumber() int { + return m.pageNumber +} +func (m v2PageMetadata) TotalPages() int { + return m.totalPages +} +func (m *v2PageMetadata) SetTotalPages(total int) { + m.totalPages = total +} +func (m *v2PageMetadata) PageRead() { + m.pageNumber++ +} + // Pageable is the interface that structs should implement to become // pageable and be able to use the client.NextPage() function type Pageable interface { PageInfo() *V2Pagination ResetPaging() + + // all these functions are automatically implemented when attaching + // the v2PageMetadata type into any Pageable struct, so attaching that + // struct is a requirement + PageRead() + SetTotalPages(int) + TotalPages() int + PageNumber() int } // NextPage @@ -188,14 +216,24 @@ func (c *Client) NextPage(p Pageable) (bool, error) { return false, nil } - c.log.Info("pagination", zap.Int("rows", pagination.Rows), - zap.Int("total_rows", pagination.TotalRows), - zap.String("next_page", pagination.Urls.NextPage), - ) if pagination.Urls.NextPage == "" { return false, nil } + if p.PageNumber() == 0 { + // first page, initialize pagination metadata + p.SetTotalPages(pagination.TotalRows / pagination.Rows) + p.PageRead() + } + + c.log.Info("pagination", + zap.Int("page_number", p.PageNumber()), + zap.Int("total_pages", p.TotalPages()), + zap.Int("rows", pagination.Rows), + zap.Int("total_rows", pagination.TotalRows), + zap.String("next_page", pagination.Urls.NextPage), + ) + pageURL, err := url.Parse(pagination.Urls.NextPage) if err != nil { return false, errors.Wrap(err, "unable to part next page url") @@ -209,5 +247,6 @@ func (c *Client) NextPage(p Pageable) (bool, error) { p.ResetPaging() c.log.Info("pagination reset") err = c.RequestDecoder("GET", path, nil, p) + p.PageRead() return true, err } diff --git a/api/v2_vulnerabilities.go b/api/v2_vulnerabilities.go index f7f283c48..84a5bb430 100644 --- a/api/v2_vulnerabilities.go +++ b/api/v2_vulnerabilities.go @@ -92,19 +92,15 @@ func (svc *v2ContainerVulnerabilityService) SearchAllPages(filters SearchFilter) for { all = append(all, response.Data...) - newResponse := VulnerabilitiesContainersResponse{ - Paging: response.Paging, - } - pageOk, err = svc.client.NextPage(&newResponse) + pageOk, err = svc.client.NextPage(&response) if err == nil && pageOk { - response = newResponse continue } break } - response.Data = all response.ResetPaging() + response.Data = all return } @@ -146,10 +142,6 @@ func (res *VulnerabilitiesContainersScanStatusResponse) CheckStatus() string { return res.Data.Status } - if res.Data.Status != "" { - return res.Data.Status - } - return "Unknown" } @@ -166,16 +158,14 @@ func (res *VulnerabilitiesContainerScanResponse) CheckStatus() string { return res.Data.Status } - if res.Data.Status != "" { - return res.Data.Status - } - return "Unknown" } type VulnerabilitiesContainersResponse struct { Data []VulnerabilityContainer `json:"data"` Paging V2Pagination `json:"paging"` + + v2PageMetadata `json:"-"` } func (r VulnerabilitiesContainersResponse) HighestSeverity() string { @@ -238,6 +228,7 @@ func (r VulnerabilitiesContainersResponse) PageInfo() *V2Pagination { } func (r *VulnerabilitiesContainersResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } func (r VulnerabilitiesContainersResponse) CriticalVulnerabilities() int32 { @@ -428,25 +419,23 @@ func (svc *v2HostVulnerabilityService) SearchAllPages(filters SearchFilter) ( for { all = append(all, response.Data...) - newResponse := VulnerabilitiesHostResponse{ - Paging: response.Paging, - } - pageOk, err = svc.client.NextPage(&newResponse) + pageOk, err = svc.client.NextPage(&response) if err == nil && pageOk { - response = newResponse continue } break } - response.Data = all response.ResetPaging() + response.Data = all return } type VulnerabilitiesHostResponse struct { Data []VulnerabilityHost `json:"data"` Paging V2Pagination `json:"paging"` + + v2PageMetadata `json:"-"` } // Fulfill Pagination interface (look at api/v2.go) @@ -455,6 +444,7 @@ func (r VulnerabilitiesHostResponse) PageInfo() *V2Pagination { } func (r *VulnerabilitiesHostResponse) ResetPaging() { r.Paging = V2Pagination{} + r.Data = nil } type VulnerabilityHost struct { diff --git a/cli/cmd/agent_list.go b/cli/cmd/agent_list.go index 2f7a99701..218301267 100644 --- a/cli/cmd/agent_list.go +++ b/cli/cmd/agent_list.go @@ -149,8 +149,8 @@ func listAgents(_ *cobra.Command, _ []string) error { } break } - response.Data = agents response.ResetPaging() + response.Data = agents } cli.StartProgress("Crunching agent data...")