From 1eff8708ac9ae800f4f4fd35693a84464b51bb86 Mon Sep 17 00:00:00 2001 From: Jerop Date: Fri, 3 Jun 2022 10:09:05 -0400 Subject: [PATCH] TEP-0090: Add Matrix Package [TEP-0090][tep-0090] proposed executing a `PipelineTask` in parallel `TaskRuns` and `Runs` with substitutions from combinations of `Parameters` in a `Matrix`. In this change, we add the matrix package which generates the combinations of the `Parameters` in a `Matrix`. [tep-0090]: https://github.com/tektoncd/community/blob/main/teps/0090-matrix.md --- pkg/matrix/matrix.go | 56 ++++++++ pkg/matrix/matrix_test.go | 152 ++++++++++++++++++++++ pkg/matrix/matrix_types.go | 36 ++++++ pkg/matrix/matrix_types_test.go | 219 ++++++++++++++++++++++++++++++++ 4 files changed, 463 insertions(+) create mode 100644 pkg/matrix/matrix.go create mode 100644 pkg/matrix/matrix_test.go create mode 100644 pkg/matrix/matrix_types.go create mode 100644 pkg/matrix/matrix_types_test.go diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go new file mode 100644 index 00000000000..7afaff4b375 --- /dev/null +++ b/pkg/matrix/matrix.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The Tekton Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package matrix + +import ( + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + "strconv" +) + +// FanOut produces combinations of Parameters of type String from a slice of Parameters of type Array. +func FanOut(params []v1beta1.Param) Combinations { + var combinations []*Combination + for _, parameter := range params { + combinations = distributeParameter(parameter, combinations) + } + return combinations +} + +func distributeParameter(param v1beta1.Param, existingCombinations []*Combination) []*Combination { + var expandedCombinations []*Combination + if len(existingCombinations) == 0 { + for i, value := range param.Value.ArrayVal { + expandedCombinations = append(expandedCombinations, createCombination(i, param.Name, value, []v1beta1.Param{})) + } + } else { + count := 0 + for _, value := range param.Value.ArrayVal { + for _, perm := range existingCombinations { + expandedCombinations = append(expandedCombinations, createCombination(count, param.Name, value, perm.Params)) + count += 1 + } + } + } + return expandedCombinations +} + +func createCombination(i int, name string, value string, parameters []v1beta1.Param) *Combination { + return &Combination{ + MatrixId: strconv.Itoa(i), + Params: append(parameters, v1beta1.Param{ + Name: name, + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: value}, + }), + } +} diff --git a/pkg/matrix/matrix_test.go b/pkg/matrix/matrix_test.go new file mode 100644 index 00000000000..c817aec1f4f --- /dev/null +++ b/pkg/matrix/matrix_test.go @@ -0,0 +1,152 @@ +/* +Copyright 2022 The Tekton Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package matrix + +import ( + "github.com/google/go-cmp/cmp" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + "testing" +) + +func Test_FanOut(t *testing.T) { + tests := []struct { + name string + matrix []v1beta1.Param + wantCombinations Combinations + }{{ + name: "single array in matrix", + matrix: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}}, + }}, + wantCombinations: Combinations{{ + MatrixId: "0", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }}, + }, { + MatrixId: "1", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }}, + }, { + MatrixId: "2", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }}, + }}, + }, { + name: "multiple arrays in matrix", + matrix: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, + }}, + wantCombinations: Combinations{{ + MatrixId: "0", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + }, { + MatrixId: "1", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + }, { + MatrixId: "2", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + }, { + MatrixId: "3", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + }, { + MatrixId: "4", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + }, { + MatrixId: "5", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + }, { + MatrixId: "6", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + }, { + MatrixId: "7", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + }, { + MatrixId: "8", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotCombinations := FanOut(tt.matrix) + if d := cmp.Diff(tt.wantCombinations, gotCombinations); d != "" { + t.Errorf("Combinations of Parameters did not match the expected Combinations: %s", d) + } + }) + } +} diff --git a/pkg/matrix/matrix_types.go b/pkg/matrix/matrix_types.go new file mode 100644 index 00000000000..56b019db5b0 --- /dev/null +++ b/pkg/matrix/matrix_types.go @@ -0,0 +1,36 @@ +/* + Copyright 2022 The Tekton Authors + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package matrix + +import "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + +type Combinations []*Combination + +// Combination is a specific combination of Parameters from a Matrix. +type Combination struct { + // Identification of a combination from Parameters in a Matrix. + MatrixId string + + // A specific combination of Parameters in a Matrix. + Params []v1beta1.Param +} + +// ToMap converts a list of Combinations to a map where the key is the matrixId and the values are Parameters. +func (combinations Combinations) ToMap() map[string][]v1beta1.Param { + m := map[string][]v1beta1.Param{} + for _, combination := range combinations { + m[combination.MatrixId] = combination.Params + } + return m +} diff --git a/pkg/matrix/matrix_types_test.go b/pkg/matrix/matrix_types_test.go new file mode 100644 index 00000000000..ed3cedb88d1 --- /dev/null +++ b/pkg/matrix/matrix_types_test.go @@ -0,0 +1,219 @@ +/* +Copyright 2022 The Tekton Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package matrix + +import ( + "github.com/google/go-cmp/cmp" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + "testing" +) + +func Test_ToMap(t *testing.T) { + tests := []struct { + name string + combinations Combinations + want map[string][]v1beta1.Param + }{{ + name: "one array in matrix", + combinations: Combinations{{ + MatrixId: "0", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }}, + }, { + MatrixId: "1", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }}, + }, { + MatrixId: "2", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }}, + }}, + want: map[string][]v1beta1.Param{ + "0": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }}, + "1": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }}, + "2": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }}, + }, + }, { + name: "multiple arrays in matrix", + combinations: Combinations{{ + MatrixId: "0", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + }, { + MatrixId: "1", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + }, { + MatrixId: "2", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + }, { + MatrixId: "3", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + }, { + MatrixId: "4", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + }, { + MatrixId: "5", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + }, { + MatrixId: "6", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + }, { + MatrixId: "7", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + }, { + MatrixId: "8", + Params: []v1beta1.Param{{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + }}, + want: map[string][]v1beta1.Param{ + "0": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + "1": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + "2": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "chrome"}, + }}, + "3": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + "4": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + "5": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "safari"}, + }}, + "6": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "linux"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + "7": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "mac"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + "8": {{ + Name: "platform", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "windows"}, + }, { + Name: "browser", + Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeString, StringVal: "firefox"}, + }}, + }, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if d := cmp.Diff(tt.want, tt.combinations.ToMap()); d != "" { + t.Errorf("Map of Combinations of Parameters did not match the expected Map %s", d) + } + }) + } +}