TC-1801 Paginated queries

Signed-off-by: mrizzi <[email protected]>

pkg/assembler/backends/ent/backend/backend.go

@@ -29,7 +29,8 @@ import (
 
 const (
 	// Batch size for ingesting in bulk. Increasing this could results in "PostgreSQL only supports 65535 parameters" error
-	MaxBatchSize = 5000
+	MaxBatchSize       = 5000
+	MaxWhereParameters = 65534
 )
 
 var Errorf = gqlerror.Errorf

pkg/assembler/backends/ent/backend/sbom.go

@@ -23,6 +23,7 @@ import (
 
 	"entgo.io/contrib/entgql"
 	"entgo.io/ent/dialect/sql"
+	"github.com/99designs/gqlgen/graphql"
 	"github.com/google/uuid"
 	"github.com/guacsec/guac/internal/testing/ptrfrom"
 	"github.com/guacsec/guac/pkg/assembler/backends/ent"
@@ -35,6 +36,7 @@ import (
 	"github.com/guacsec/guac/pkg/assembler/backends/helper"
 	"github.com/guacsec/guac/pkg/assembler/graphql/model"
 	"github.com/pkg/errors"
+	"github.com/vektah/gqlparser/v2/ast"
 	"github.com/vektah/gqlparser/v2/gqlerror"
 )
 
@@ -294,16 +296,25 @@ func (b *EntBackend) HasSBOM(ctx context.Context, spec *model.HasSBOMSpec) ([]*m
 		spec = &model.HasSBOMSpec{}
 	}
 
-	sbomQuery := b.client.BillOfMaterials.Query().
-		Where(hasSBOMQuery(*spec))
-
-	records, err := getSBOMObjectWithIncludes(sbomQuery).
-		All(ctx)
+	// this is kind of trick because Ent pagination expects some fields to be provided in the GraphQL request
+	// but, in this case, it's rather a conversion from the non-paginated request, i.e. HasSBOM,
+	// to a paginated request and so some fields must be added to the context for the paginated approach to work,
+	// in fact it checks for the ent.EdgesField existence otherwise no query is executed.
+	fc := graphql.GetFieldContext(ctx)
+	var node = ast.Field{Alias: ent.NodeField}
+	for _, selection := range fc.Field.Selections {
+		node.SelectionSet = append(node.SelectionSet, selection)
+	}
+	fc.Field.Selections = append(fc.Field.Selections, &ast.Field{Alias: ent.EdgesField, SelectionSet: ast.SelectionSet{&node}})
+	hasSBOMList, err := b.HasSBOMList(ctx, *spec, nil, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, funcName)
 	}
-
-	return collect(records, toModelHasSBOM), nil
+	var result []*model.HasSbom
+	for _, edge := range hasSBOMList.Edges {
+		result = append(result, edge.Node)
+	}
+	return result, nil
 }
 
 func hasSBOMQuery(spec model.HasSBOMSpec) predicate.BillOfMaterials {

pkg/assembler/backends/ent/backend/search.go

@@ -414,61 +414,66 @@ func (b *EntBackend) findVulnerabilities(ctx context.Context, hasSBOMSpec *model
 		return nil, gqlerror.Errorf("error querying for IncludedDependencies with SBOM URI %v due to : %v", sbomURI, err)
 	}
 	var dependenciesPackagesUUIDs uuid.UUIDs
-	for _, dependency := range dependencies {
-		dependenciesPackagesUUIDs = append(dependenciesPackagesUUIDs, dependency.DependentPackageVersionID)
+	for _, dep := range dependencies {
+		dependenciesPackagesUUIDs = append(dependenciesPackagesUUIDs, dep.DependentPackageVersionID)
 	}
 
-	// retrieve CertifyVex
-	certifyVexQuery := b.client.CertifyVex.Query().
-		Where(
-			certifyvex.StatusNEQ(model.VexStatusNotAffected.String()),
-			certifyvex.PackageIDIn(dependenciesPackagesUUIDs...),
-		).
-		WithVulnerability().
-		WithPackage(withPackageVersionTree()).
-		Order(ent.Desc(vulnerabilityid.FieldID))
+	batches := chunk(dependenciesPackagesUUIDs, MaxWhereParameters)
+	var result []model.CertifyVulnOrCertifyVEXStatement
+	for _, pkgs := range batches {
+		// retrieve CertifyVex
+		certifyVexQuery := b.client.CertifyVex.Query().
+			Where(
+				certifyvex.StatusNEQ(model.VexStatusNotAffected.String()),
+				certifyvex.PackageIDIn(pkgs...),
+			).
+			WithVulnerability().
+			WithPackage(withPackageVersionTree()).
+			Order(ent.Desc(vulnerabilityid.FieldID))
 
-	if offset != nil {
-		certifyVexQuery.Offset(*offset)
-	}
+		if offset != nil {
+			certifyVexQuery.Offset(*offset)
+		}
 
-	if limit != nil {
-		certifyVexQuery.Limit(*limit)
-	}
+		if limit != nil {
+			certifyVexQuery.Limit(*limit)
+		}
 
-	certifyVexes, err := certifyVexQuery.All(ctx)
-	if err != nil {
-		return nil, gqlerror.Errorf("error querying for CertifyVex with SBOM URI %v due to : %v", sbomURI, err)
-	}
+		certifyVexes, err := certifyVexQuery.All(ctx)
+		if err != nil {
+			return nil, gqlerror.Errorf("error querying for CertifyVex with SBOM URI %v due to : %v", sbomURI, err)
+		}
 
-	var result []model.CertifyVulnOrCertifyVEXStatement
-	for _, certifyVex := range certifyVexes {
-		result = append(result, toModelCertifyVEXStatement(certifyVex))
+		for _, certifyVex := range certifyVexes {
+			result = append(result, toModelCertifyVEXStatement(certifyVex))
+		}
 	}
 
-	// retrieve CertifyVuln
-	certifyVulnQuery := b.client.CertifyVuln.Query().
-		Where(
-			certifyvuln.PackageIDIn(dependenciesPackagesUUIDs...),
-		).
-		WithVulnerability().
-		WithPackage(withPackageVersionTree()).
-		Order(ent.Desc(vulnerabilityid.FieldID))
+	for _, pkgs := range batches {
+		// retrieve CertifyVuln
+		certifyVulnQuery := b.client.CertifyVuln.Query().
+			Where(
+				certifyvuln.PackageIDIn(pkgs...),
+			).
+			WithVulnerability().
+			WithPackage(withPackageVersionTree()).
+			Order(ent.Desc(vulnerabilityid.FieldID))
 
-	if offset != nil {
-		certifyVulnQuery.Offset(*offset)
-	}
+		if offset != nil {
+			certifyVulnQuery.Offset(*offset)
+		}
 
-	if limit != nil {
-		certifyVulnQuery.Limit(*limit)
-	}
+		if limit != nil {
+			certifyVulnQuery.Limit(*limit)
+		}
 
-	certifyVulns, err := certifyVulnQuery.All(ctx)
-	if err != nil {
-		return nil, gqlerror.Errorf("error querying for CertifyVuln with SBOM URI %v due to : %v", sbomURI, err)
-	}
-	for _, certifyVuln := range certifyVulns {
-		result = append(result, toModelCertifyVulnerability(certifyVuln))
+		certifyVulns, err := certifyVulnQuery.All(ctx)
+		if err != nil {
+			return nil, gqlerror.Errorf("error querying for CertifyVuln with SBOM URI %v due to : %v", sbomURI, err)
+		}
+		for _, certifyVuln := range certifyVulns {
+			result = append(result, toModelCertifyVulnerability(certifyVuln))
+		}
 	}
 
 	return &result, nil

pkg/assembler/backends/ent/gql_pagination_helper.go

@@ -0,0 +1,22 @@
+//
+// Copyright 2023 The GUAC Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ent
+
+// exporting some pagination related fields to be used directly in backend code
+const (
+	EdgesField = edgesField
+	NodeField  = nodeField
+)