[CI][GCI/3] Add variations attribute to create tests in multiple clus…

…ter environment (#33718)

This diff adds a 'variations' attribute to release test definitions, and update the parser to interpret this attribute.

This attribute is used when one wants to define several flavors for a test definition. Each flavor defines a set of test parameters, such as its environment, cluster compute, etc. A test will then be created for each flavor.

Signed-off-by: Cuong Nguyen <[email protected]>

release/ray_release/tests/test_config.py

@@ -1,7 +1,6 @@
 import os
 import sys
 import yaml
-import copy
 import pytest
 
 from ray_release.config import (
@@ -44,8 +43,15 @@
     }
 )
 
-def test_definition_parser():
-    test_definitions = yaml.safe_load('''
+
+def test_parse_test_definition():
+    """
+    Unit test for the ray_release.config.parse_test_definition function. In particular,
+    we check that the code correctly parse a test definition that have the 'variations'
+    field.
+    """
+    test_definitions = yaml.safe_load(
+        """
         - name: sample_test
           working_dir: sample_dir
           frequency: nightly
@@ -62,7 +68,10 @@ def test_definition_parser():
               cluster:
                 cluster_env: env_gce.yaml
                 cluster_compute: compute_gce.yaml
-    ''')
+    """
+    )
+    # Check that parsing returns two tests, one for each variation (aws and gce). Check
+    # that both tests are valid, and their fields are populated correctly
     tests = parse_test_definition(test_definitions)
     aws_test = tests[0]
     gce_test = tests[1]
@@ -72,13 +81,14 @@ def test_definition_parser():
     assert aws_test["name"] == "sample_test.aws"
     assert gce_test["cluster"]["cluster_compute"] == "compute_gce.yaml"
     invalid_test_definition = test_definitions[0]
-    invalid_test_definition['variations'] = []
+    # Intentionally make the test definition invalid by create an empty 'variations'
+    # field. Check that the parser throws exception at runtime
+    invalid_test_definition["variations"] = []
     with pytest.raises(ReleaseTestConfigError):
         parse_test_definition([invalid_test_definition])
-    invalid_test_definition['variations'] = [
-        {'__suffix__': 'aws'},
-        {}
-    ]
+    # Intentionally make the test definition invalid by making one 'variation' entry
+    # missing the __suffix__ entry. Check that the parser throws exception at runtime
+    invalid_test_definition["variations"] = [{"__suffix__": "aws"}, {}]
     with pytest.raises(ReleaseTestConfigError):
         parse_test_definition([invalid_test_definition])