Add support for consuming whole array results in matrix

This commit adds support for referencing whole array results produced by another PipelineTask as a matrix parameter value. This feature request is necessary before promoting Matrix to beta. For more details please see issue #6110. This closes issue #6602. Co-authored-by: Yongxuan Zhang <[email protected]>
tektoncd · May 4, 2023 · 4f1cdbe · 4f1cdbe
1 parent 3d6ec63
commit 4f1cdbe
Show file tree

Hide file tree

Showing 15 changed files with 501 additions and 91 deletions.
diff --git a/docs/matrix.md b/docs/matrix.md
@@ -303,7 +303,7 @@ See the end-to-end example in [`PipelineRun` with `Matrix` and `Results`][pr-wit
 
 #### Results in Matrix.Params
 
-`Matrix.Params` supports string replacements from `Results` of type String, Array or Object.
+`Matrix.Params` supports whole array replacements and string replacements from `Results` of type String, Array or Object
 
 ```yaml
 tasks:
@@ -314,16 +314,9 @@ tasks:
   matrix:
     params:
     - name: values
-      value:
-      - $(tasks.task-1.results.foo) # string replacement from string result
-      - $(tasks.task-2.results.bar[0]) # string replacement from array result
-      - $(tasks.task-3.results.rad.key) # string replacement from object result
+      value: $(tasks.task-4.results.whole-array[*])
 ```
 
-For further information, see the example in [`PipelineRun` with `Matrix` and `Results`][pr-with-matrix-and-results].
-
-We plan to add support passing whole Array `Results` into the `Matrix` (#5925):
-
 ```yaml
 tasks:
 ...
@@ -333,9 +326,16 @@ tasks:
   matrix:
     params:
     - name: values
-      value: $(tasks.task-4.results.foo) # array
+      value:
+        - $(tasks.task-1.results.a-string-result)
+        - $(tasks.task-2.results.an-array-result[0])
+        - $(tasks.task-3.results.an-object-result.key)
 ```
 
+
+For further information, see the example in [`PipelineRun` with `Matrix` and `Results`][pr-with-matrix-and-results].
+
+
 #### Results in Matrix.Include.Params
 
 `Matrix.Include.Params` supports string replacements from `Results` of type String, Array or Object.

diff --git a/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml b/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml
@@ -21,7 +21,7 @@ kind: PipelineRun
 metadata:
   generateName: matrixed-pr-
 spec:
-  serviceAccountName: 'default'
+  serviceAccountName: "default"
   pipelineSpec:
     tasks:
       - name: get-platforms
@@ -56,10 +56,7 @@ spec:
         matrix:
           params:
             - name: platform
-              value:
-                - $(tasks.get-platforms.results.platforms[0])
-                - $(tasks.get-platforms.results.platforms[1])
-                - $(tasks.get-platforms.results.platforms[2])
+              value: $(tasks.get-platforms.results.platforms[*])
             - name: browser
               value:
                 - $(tasks.get-browsers-and-url.results.browsers[0])

diff --git a/pkg/apis/pipeline/v1/matrix_types.go b/pkg/apis/pipeline/v1/matrix_types.go
@@ -348,24 +348,3 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params []Param) (errs *a
 	}
 	return errs
 }
-
-// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references
-// to entire arrays. This is temporary until whole array replacements for matrix parameters are supported.
-// See issue #6056 for more details
-func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) {
-	if m.HasParams() {
-		for i, param := range m.Params {
-			val := param.Value.StringVal
-			expressions, ok := GetVarSubstitutionExpressionsForParam(param)
-			if ok {
-				if LooksLikeContainsResultRefs(expressions) {
-					_, stringIdx := ParseResultName(val)
-					if stringIdx == "*" {
-						errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i))
-					}
-				}
-			}
-		}
-	}
-	return errs
-}
diff --git a/pkg/apis/pipeline/v1/pipeline_types_test.go b/pkg/apis/pipeline/v1/pipeline_types_test.go
@@ -711,19 +711,6 @@ func TestPipelineTask_ValidateMatrix(t *testing.T) {
 					Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}},
 				}}},
 		},
-	}, {
-		name: "parameters in matrix contain whole array results references",
-		pt: &PipelineTask{
-			Name: "task",
-			Matrix: &Matrix{
-				Params: Params{{
-					Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"},
-				}}},
-		},
-		wantErrs: &apis.FieldError{
-			Message: "matrix parameters cannot contain whole array result references",
-			Paths:   []string{"matrix.params[0]"},
-		},
 	}, {
 		name: "count of combinations of parameters in the matrix exceeds the maximum",
 		pt: &PipelineTask{

diff --git a/pkg/apis/pipeline/v1/pipeline_validation.go b/pkg/apis/pipeline/v1/pipeline_validation.go
@@ -155,7 +155,6 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr
 		// when the enable-api-fields feature gate is anything but "alpha".
 		errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields))
 		errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx))
-		errs = errs.Also(pt.Matrix.validateNoWholeArrayResults())
 		errs = errs.Also(pt.Matrix.validateUniqueParams())
 	}
 	errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params))

diff --git a/pkg/apis/pipeline/v1/pipeline_validation_test.go b/pkg/apis/pipeline/v1/pipeline_validation_test.go
@@ -3095,6 +3095,16 @@ func Test_validateMatrix(t *testing.T) {
 					Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.bar-task.results.b-result)"}},
 				}}},
 		}},
+	}, {
+		name: "parameters in matrix contain whole array results references",
+		tasks: PipelineTaskList{{
+			Name:    "a-task",
+			TaskRef: &TaskRef{Name: "a-task"},
+			Matrix: &Matrix{
+				Params: Params{{
+					Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}},
+				}}},
+		}},
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

diff --git a/pkg/apis/pipeline/v1beta1/matrix_types.go b/pkg/apis/pipeline/v1beta1/matrix_types.go
@@ -17,7 +17,6 @@ import (
 	"context"
 	"fmt"
 	"sort"
-	"strings"
 
 	"github.com/tektoncd/pipeline/pkg/apis/config"
 	"golang.org/x/exp/maps"
@@ -349,21 +348,3 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params Params) (errs *ap
 	}
 	return errs
 }
-
-// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references
-// to entire arrays. This is temporary until whole array replacements for matrix paraemeters are supported.
-// See issue #6056 for more details
-func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) {
-	if m.HasParams() {
-		for i, param := range m.Params {
-			val := param.Value.StringVal
-			expressions, ok := GetVarSubstitutionExpressionsForParam(param)
-			if ok {
-				if LooksLikeContainsResultRefs(expressions) && strings.Contains(val, "[*]") {
-					errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i))
-				}
-			}
-		}
-	}
-	return errs
-}
diff --git a/pkg/apis/pipeline/v1beta1/pipeline_types_test.go b/pkg/apis/pipeline/v1beta1/pipeline_types_test.go
@@ -684,19 +684,6 @@ func TestPipelineTask_validateMatrix(t *testing.T) {
 					Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}},
 				}}},
 		},
-	}, {
-		name: "parameters in matrix contain whole array results references",
-		pt: &PipelineTask{
-			Name: "task",
-			Matrix: &Matrix{
-				Params: Params{{
-					Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"},
-				}}},
-		},
-		wantErrs: &apis.FieldError{
-			Message: "matrix parameters cannot contain whole array result references",
-			Paths:   []string{"matrix.params[0]"},
-		},
 	}, {
 		name: "count of combinations of parameters in the matrix exceeds the maximum",
 		pt: &PipelineTask{

diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation.go b/pkg/apis/pipeline/v1beta1/pipeline_validation.go
@@ -167,7 +167,6 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr
 		// when the enable-api-fields feature gate is anything but "alpha".
 		errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields))
 		errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx))
-		errs = errs.Also(pt.Matrix.validateNoWholeArrayResults())
 		errs = errs.Also(pt.Matrix.validateUniqueParams())
 	}
 	errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params))

diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go
@@ -3139,6 +3139,16 @@ func Test_validateMatrix(t *testing.T) {
 					Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.bar-task.results.b-result)"}},
 				}}},
 		}},
+	}, {
+		name: "parameters in matrix contain whole array results references",
+		tasks: PipelineTaskList{{
+			Name:    "a-task",
+			TaskRef: &TaskRef{Name: "a-task"},
+			Matrix: &Matrix{
+				Params: Params{{
+					Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}},
+				}}},
+		}},
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go
@@ -774,6 +774,12 @@ func (c *Reconciler) runNextSchedulableTask(ctx context.Context, pr *v1beta1.Pip
 
 		switch {
 		case rpt.IsCustomTask() && rpt.PipelineTask.IsMatrixed():
+			// RunObjectNames Names have not yet been populated for a matrix that has whole array references
+			// Since we have already applied Task Results, we can now get the names of the RunObjects
+			// after array result substitution before fanning out the Matrix and creating RunObjects
+			if len(rpt.RunObjectName) == 0 {
+				rpt.RunObjectNames = resources.GetNamesOfRuns(pr.Status.ChildReferences, rpt.PipelineTask.Name, pr.Name, rpt.PipelineTask.Matrix.CountCombinations())
+			}
 			rpt.RunObjects, err = c.createRunObjects(ctx, rpt, pr)
 			if err != nil {
 				recorder.Eventf(pr, corev1.EventTypeWarning, "RunsCreationFailed", "Failed to create Runs %q: %v", rpt.RunObjectNames, err)
@@ -788,6 +794,12 @@ func (c *Reconciler) runNextSchedulableTask(ctx context.Context, pr *v1beta1.Pip
 				return err
 			}
 		case rpt.PipelineTask.IsMatrixed():
+			// TaskRun Names have not yet been populated for a matrix that has whole array references
+			// Since we have already applied Task Results, we can now get the names of the TaskRuns
+			// after array result substitution before fanning out the Matrix and creating TaskRuns
+			if len(rpt.TaskRunNames) == 0 {
+				rpt.TaskRunNames = resources.GetNamesOfTaskRuns(pr.Status.ChildReferences, rpt.PipelineTask.Name, pr.Name, rpt.PipelineTask.Matrix.CountCombinations())
+			}
 			rpt.TaskRuns, err = c.createTaskRuns(ctx, rpt, pr)
 			if err != nil {
 				recorder.Eventf(pr, corev1.EventTypeWarning, "TaskRunsCreationFailed", "Failed to create TaskRuns %q: %v", rpt.TaskRunNames, err)