Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for consuming whole array results in matrix #6603

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions docs/matrix.md
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ See the end-to-end example in [`PipelineRun` with `Matrix` and `Results`][pr-wit

#### Results in Matrix.Params

`Matrix.Params` supports string replacements from `Results` of type String, Array or Object.
`Matrix.Params` supports whole array replacements and string replacements from `Results` of type String, Array or Object

```yaml
tasks:
Expand All @@ -314,16 +314,9 @@ tasks:
matrix:
params:
- name: values
value:
- $(tasks.task-1.results.foo) # string replacement from string result
- $(tasks.task-2.results.bar[0]) # string replacement from array result
- $(tasks.task-3.results.rad.key) # string replacement from object result
value: $(tasks.task-4.results.whole-array[*])
```

For further information, see the example in [`PipelineRun` with `Matrix` and `Results`][pr-with-matrix-and-results].

We plan to add support passing whole Array `Results` into the `Matrix` (#5925):

```yaml
tasks:
...
Expand All @@ -333,9 +326,16 @@ tasks:
matrix:
params:
- name: values
value: $(tasks.task-4.results.foo) # array
value:
- $(tasks.task-1.results.a-string-result)
- $(tasks.task-2.results.an-array-result[0])
- $(tasks.task-3.results.an-object-result.key)
```


For further information, see the example in [`PipelineRun` with `Matrix` and `Results`][pr-with-matrix-and-results].


#### Results in Matrix.Include.Params

`Matrix.Include.Params` supports string replacements from `Results` of type String, Array or Object.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ kind: PipelineRun
metadata:
generateName: matrixed-pr-
spec:
serviceAccountName: 'default'
serviceAccountName: "default"
pipelineSpec:
tasks:
- name: get-platforms
Expand Down Expand Up @@ -56,10 +56,7 @@ spec:
matrix:
params:
- name: platform
value:
- $(tasks.get-platforms.results.platforms[0])
- $(tasks.get-platforms.results.platforms[1])
- $(tasks.get-platforms.results.platforms[2])
value: $(tasks.get-platforms.results.platforms[*])
- name: browser
value:
- $(tasks.get-browsers-and-url.results.browsers[0])
Expand Down
21 changes: 0 additions & 21 deletions pkg/apis/pipeline/v1/matrix_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -348,24 +348,3 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params []Param) (errs *a
}
return errs
}

// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references
// to entire arrays. This is temporary until whole array replacements for matrix parameters are supported.
// See issue #6056 for more details
func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) {
if m.HasParams() {
for i, param := range m.Params {
val := param.Value.StringVal
expressions, ok := GetVarSubstitutionExpressionsForParam(param)
if ok {
if LooksLikeContainsResultRefs(expressions) {
_, stringIdx := ParseResultName(val)
if stringIdx == "*" {
errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i))
}
}
}
}
}
return errs
}
13 changes: 0 additions & 13 deletions pkg/apis/pipeline/v1/pipeline_types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -696,19 +696,6 @@ func TestPipelineTask_ValidateMatrix(t *testing.T) {
Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}},
}}},
},
}, {
name: "parameters in matrix contain whole array results references",
pt: &PipelineTask{
Name: "task",
Matrix: &Matrix{
Params: Params{{
Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"},
}}},
},
wantErrs: &apis.FieldError{
Message: "matrix parameters cannot contain whole array result references",
Paths: []string{"matrix.params[0]"},
},
}, {
name: "count of combinations of parameters in the matrix exceeds the maximum",
pt: &PipelineTask{
Expand Down
1 change: 0 additions & 1 deletion pkg/apis/pipeline/v1/pipeline_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,6 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr
// when the enable-api-fields feature gate is anything but "alpha".
errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields))
errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx))
errs = errs.Also(pt.Matrix.validateNoWholeArrayResults())
errs = errs.Also(pt.Matrix.validateUniqueParams())
}
errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params))
Expand Down
10 changes: 10 additions & 0 deletions pkg/apis/pipeline/v1/pipeline_validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3150,6 +3150,16 @@ func Test_validateMatrix(t *testing.T) {
Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.bar-task.results.b-result)"}},
}}},
}},
}, {
name: "parameters in matrix contain whole array results references",
tasks: PipelineTaskList{{
Name: "a-task",
TaskRef: &TaskRef{Name: "a-task"},
Matrix: &Matrix{
Params: Params{{
Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}},
}}},
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
19 changes: 0 additions & 19 deletions pkg/apis/pipeline/v1beta1/matrix_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"context"
"fmt"
"sort"
"strings"

"github.com/tektoncd/pipeline/pkg/apis/config"
"golang.org/x/exp/maps"
Expand Down Expand Up @@ -349,21 +348,3 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params Params) (errs *ap
}
return errs
}

// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references
// to entire arrays. This is temporary until whole array replacements for matrix paraemeters are supported.
// See issue #6056 for more details
func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) {
if m.HasParams() {
for i, param := range m.Params {
val := param.Value.StringVal
expressions, ok := GetVarSubstitutionExpressionsForParam(param)
if ok {
if LooksLikeContainsResultRefs(expressions) && strings.Contains(val, "[*]") {
errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i))
}
}
}
}
return errs
}
13 changes: 0 additions & 13 deletions pkg/apis/pipeline/v1beta1/pipeline_types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -680,19 +680,6 @@ func TestPipelineTask_validateMatrix(t *testing.T) {
Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}},
}}},
},
}, {
name: "parameters in matrix contain whole array results references",
pt: &PipelineTask{
Name: "task",
Matrix: &Matrix{
Params: Params{{
Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"},
}}},
},
wantErrs: &apis.FieldError{
Message: "matrix parameters cannot contain whole array result references",
Paths: []string{"matrix.params[0]"},
},
}, {
name: "count of combinations of parameters in the matrix exceeds the maximum",
pt: &PipelineTask{
Expand Down
1 change: 0 additions & 1 deletion pkg/apis/pipeline/v1beta1/pipeline_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,6 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr
// when the enable-api-fields feature gate is anything but "alpha".
errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields))
errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx))
errs = errs.Also(pt.Matrix.validateNoWholeArrayResults())
errs = errs.Also(pt.Matrix.validateUniqueParams())
}
errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params))
Expand Down
10 changes: 10 additions & 0 deletions pkg/apis/pipeline/v1beta1/pipeline_validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3193,6 +3193,16 @@ func Test_validateMatrix(t *testing.T) {
Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.bar-task.results.b-result)"}},
}}},
}},
}, {
name: "parameters in matrix contain whole array results references",
tasks: PipelineTaskList{{
Name: "a-task",
TaskRef: &TaskRef{Name: "a-task"},
Matrix: &Matrix{
Params: Params{{
Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}},
}}},
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
2 changes: 0 additions & 2 deletions pkg/reconciler/pipelinerun/pipelinerun.go
Original file line number Diff line number Diff line change
Expand Up @@ -812,7 +812,6 @@ func (c *Reconciler) createTaskRuns(ctx context.Context, rpt *resources.Resolved
if rpt.PipelineTask.IsMatrixed() {
matrixCombinations = rpt.PipelineTask.Matrix.FanOut()
}

for i, taskRunName := range rpt.TaskRunNames {
var params v1.Params
if len(matrixCombinations) > i {
Expand Down Expand Up @@ -894,7 +893,6 @@ func (c *Reconciler) createCustomRuns(ctx context.Context, rpt *resources.Resolv
if rpt.PipelineTask.IsMatrixed() {
matrixCombinations = rpt.PipelineTask.Matrix.FanOut()
}

for i, customRunName := range rpt.CustomRunNames {
var params v1.Params
if len(matrixCombinations) > i {
Expand Down
Loading