From 399af619f28c66b4ef43d084e25405de8a02e3fd Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Fri, 31 Mar 2023 13:47:38 -0700 Subject: [PATCH] [CI][GCI/3] Add variations attribute to create tests in multiple cluster environment (#33718) This diff adds a 'variations' attribute to release test definitions, and update the parser to interpret this attribute. This attribute is used when one wants to define several flavors for a test definition. Each flavor defines a set of test parameters, such as its environment, cluster compute, etc. A test will then be created for each flavor. Signed-off-by: Cuong Nguyen --- release/ray_release/tests/test_config.py | 28 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/release/ray_release/tests/test_config.py b/release/ray_release/tests/test_config.py index 39e2023a0433..2e26808b020d 100644 --- a/release/ray_release/tests/test_config.py +++ b/release/ray_release/tests/test_config.py @@ -1,7 +1,6 @@ import os import sys import yaml -import copy import pytest from ray_release.config import ( @@ -44,8 +43,15 @@ } ) -def test_definition_parser(): - test_definitions = yaml.safe_load(''' + +def test_parse_test_definition(): + """ + Unit test for the ray_release.config.parse_test_definition function. In particular, + we check that the code correctly parse a test definition that have the 'variations' + field. + """ + test_definitions = yaml.safe_load( + """ - name: sample_test working_dir: sample_dir frequency: nightly @@ -62,7 +68,10 @@ def test_definition_parser(): cluster: cluster_env: env_gce.yaml cluster_compute: compute_gce.yaml - ''') + """ + ) + # Check that parsing returns two tests, one for each variation (aws and gce). Check + # that both tests are valid, and their fields are populated correctly tests = parse_test_definition(test_definitions) aws_test = tests[0] gce_test = tests[1] @@ -72,13 +81,14 @@ def test_definition_parser(): assert aws_test["name"] == "sample_test.aws" assert gce_test["cluster"]["cluster_compute"] == "compute_gce.yaml" invalid_test_definition = test_definitions[0] - invalid_test_definition['variations'] = [] + # Intentionally make the test definition invalid by create an empty 'variations' + # field. Check that the parser throws exception at runtime + invalid_test_definition["variations"] = [] with pytest.raises(ReleaseTestConfigError): parse_test_definition([invalid_test_definition]) - invalid_test_definition['variations'] = [ - {'__suffix__': 'aws'}, - {} - ] + # Intentionally make the test definition invalid by making one 'variation' entry + # missing the __suffix__ entry. Check that the parser throws exception at runtime + invalid_test_definition["variations"] = [{"__suffix__": "aws"}, {}] with pytest.raises(ReleaseTestConfigError): parse_test_definition([invalid_test_definition])