From 238b2edab5f4dfb5c40f84ae11b4018727643365 Mon Sep 17 00:00:00 2001 From: Chi-Sheng Liu Date: Thu, 7 Nov 2024 10:41:55 +0800 Subject: [PATCH] [Feat][Sample-yaml] Deprecated python sample yaml test cleanup Closes: ray-project/kuberay#2506 Signed-off-by: Chi-Sheng Liu --- .buildkite/test-sample-yamls.yml | 40 ------------ tests/test_sample_raycluster_yamls.py | 92 --------------------------- tests/test_sample_rayjob_yamls.py | 54 ---------------- 3 files changed, 186 deletions(-) delete mode 100644 tests/test_sample_raycluster_yamls.py delete mode 100644 tests/test_sample_rayjob_yamls.py diff --git a/.buildkite/test-sample-yamls.yml b/.buildkite/test-sample-yamls.yml index 331e8cabf2..685c4bb9b4 100644 --- a/.buildkite/test-sample-yamls.yml +++ b/.buildkite/test-sample-yamls.yml @@ -1,46 +1,6 @@ #ci:group=:yaml: Sample YAML tests # TODO: Remove deprecated tests after migrating to Go tests -- label: '(Deprecated) Test RayCluster Sample YAMLs (nightly operator)' - instance_size: large - image: golang:1.22 - commands: - - ./.buildkite/setup-env.sh - # Build nightly KubeRay operator image - - pushd ray-operator - - IMG=kuberay/operator:nightly make docker-image - - popd - # Use nightly KubeRay operator image - - source .venv/bin/activate && BUILDKITE_ENV=true OPERATOR_IMAGE=kuberay/operator:nightly python3 tests/test_sample_raycluster_yamls.py - -- label: '(Deprecated) Test RayCluster Sample YAMLs (latest release)' - instance_size: large - image: golang:1.22 - commands: - - ./.buildkite/setup-env.sh - # Use KubeRay operator image from the latest release - - source .venv/bin/activate && BUILDKITE_ENV=true OPERATOR_IMAGE=quay.io/kuberay/operator:v1.2.2 python3 tests/test_sample_raycluster_yamls.py - -- label: '(Deprecated) Test RayJob Sample YAMLs (nightly operator)' - instance_size: large - image: golang:1.22 - commands: - - ./.buildkite/setup-env.sh - # Build nightly KubeRay operator image - - pushd ray-operator - - IMG=kuberay/operator:nightly make docker-image - - popd - # Use nightly KubeRay operator image - - source .venv/bin/activate && BUILDKITE_ENV=true OPERATOR_IMAGE=kuberay/operator:nightly python3 tests/test_sample_rayjob_yamls.py - -- label: '(Deprecated) Test RayJob Sample YAMLs (latest release)' - instance_size: large - image: golang:1.22 - commands: - - ./.buildkite/setup-env.sh - # Use KubeRay operator image from the latest release - - source .venv/bin/activate && BUILDKITE_ENV=true OPERATOR_IMAGE=quay.io/kuberay/operator:v1.2.2 python3 tests/test_sample_rayjob_yamls.py - - label: '(Deprecated) Test RayService Sample YAMLs (nightly operator)' instance_size: large image: golang:1.22 diff --git a/tests/test_sample_raycluster_yamls.py b/tests/test_sample_raycluster_yamls.py deleted file mode 100644 index 0589a55fd0..0000000000 --- a/tests/test_sample_raycluster_yamls.py +++ /dev/null @@ -1,92 +0,0 @@ -''' Test sample RayCluster YAML files to catch invalid and outdated ones. ''' -import logging -import unittest -import os -import git -import yaml -import argparse - -from framework.prototype import ( - RuleSet, - GeneralTestCase, - RayClusterAddCREvent, - HeadPodNameRule, - EasyJobRule, - HeadSvcRule, -) - -from framework.utils import ( - CONST -) - -logger = logging.getLogger(__name__) - -def parse_args(): - parser = argparse.ArgumentParser(description='Run tests for specified YAML files.') - parser.add_argument('--yaml-files', nargs='*', help='Use the filename under path `ray-operator/config/samples` to specify which YAML files should be tested.') - return parser.parse_args() - -if __name__ == '__main__': - NAMESPACE = 'default' - SAMPLE_PATH = CONST.REPO_ROOT.joinpath("ray-operator/config/samples/") - - sample_yaml_files = [] - - # Paths of untracked files, specified as strings, relative to KubeRay - # git root directory. - untracked_files = set( - git.Repo(CONST.REPO_ROOT).untracked_files - ) - - args = parse_args() - - for file in os.scandir(SAMPLE_PATH): - if not file.is_file(): - continue - # For local development, skip untracked files. - if os.path.relpath(file.path, CONST.REPO_ROOT) in untracked_files: - continue - # Skip files that don't match the specified YAML files - if args.yaml_files and file.name not in args.yaml_files: - continue - - with open(file, encoding="utf-8") as cr_yaml: - for k8s_object in yaml.safe_load_all(cr_yaml): - if k8s_object['kind'] == 'RayCluster': - sample_yaml_files.append( - {'path': file.path, 'name': file.name, 'cr': k8s_object} - ) - break - - skip_tests = { - 'ray-cluster.complete.large.yaml': 'Skip this test because it requires a lot of resources.', - 'ray-cluster.autoscaler.large.yaml': - 'Skip this test because it requires a lot of resources.', - 'ray-cluster.tpu-v4-singlehost.yaml': 'Skip this test because it requires TPU resources.', - 'ray-cluster.tpu-v4-multihost.yaml' : 'Skip this test because it requires TPU resources', - 'ray-cluster.tpu-v6e-singlehost.yaml' : 'Skip this test because it requires TPU resources', - 'ray-cluster.tpu-v6e-16-multihost.yaml' : 'Skip this test because it requires TPU resources', - 'ray-cluster.tpu-v6e-256-multihost.yaml' : 'Skip this test because it requires TPU resources', - 'ray-cluster.gke-bucket.yaml': 'Skip this test because it requires GKE and k8s service accounts.', - 'ray-service.high-availability-locust.yaml': 'Skip this test because the RayCluster here is only used for testing RayService.', - } - - rs = RuleSet([HeadPodNameRule(), EasyJobRule(), HeadSvcRule()]) - - # Build a test plan - logger.info("Build a test plan ...") - test_cases = unittest.TestSuite() - for index, new_cr in enumerate(sample_yaml_files): - if new_cr['name'] in skip_tests: - logger.info('[SKIP TEST %d] %s: %s', index, new_cr['name'], skip_tests[new_cr['name']]) - continue - logger.info('[TEST %d]: %s', index, new_cr['name']) - addEvent = RayClusterAddCREvent(new_cr['cr'], [rs], 90, NAMESPACE, new_cr['path']) - test_cases.addTest(GeneralTestCase('runtest', addEvent)) - - # Execute all tests - runner = unittest.TextTestRunner() - test_result = runner.run(test_cases) - - # Without this line, the exit code will always be 0. - assert test_result.wasSuccessful() diff --git a/tests/test_sample_rayjob_yamls.py b/tests/test_sample_rayjob_yamls.py deleted file mode 100644 index 4b52a7c36f..0000000000 --- a/tests/test_sample_rayjob_yamls.py +++ /dev/null @@ -1,54 +0,0 @@ -''' Test sample RayJob YAML files to catch invalid and outdated ones. ''' -import unittest -import os -import logging -import yaml - -from framework.prototype import ( - RuleSet, - GeneralTestCase, - RayJobAddCREvent, - EasyJobRule, - ShutdownJobRule, -) - -from framework.utils import ( - CONST -) - -logger = logging.getLogger(__name__) - -if __name__ == '__main__': - NAMESPACE = 'default' - SAMPLE_PATH = CONST.REPO_ROOT.joinpath("ray-operator/config/samples/") - YAMLs = ['ray-job.sample.yaml', 'ray-job.shutdown.yaml', 'ray-job.custom-head-svc.yaml', 'ray-job.resources.yaml'] - - sample_yaml_files = [] - for filename in YAMLs: - filepath = SAMPLE_PATH.joinpath(filename) - with open(filepath, encoding="utf-8") as cr_yaml: - for k8s_object in yaml.safe_load_all(cr_yaml): - if k8s_object['kind'] == 'RayJob': - sample_yaml_files.append( - {'path': filepath, 'name': filename, 'cr': k8s_object} - ) - break - # NOTE: The Ray Job "SUCCEEDED" status is checked in the `RayJobAddCREvent` itself. - # (The event is not considered "converged" until the job has succeeded.) The EasyJobRule - # is only used to additionally check that the Ray Cluster remains alive and functional. - rs = RuleSet([EasyJobRule(), ShutdownJobRule()]) - - # Build a test plan - logger.info("Building a test plan ...") - test_cases = unittest.TestSuite() - for index, new_cr in enumerate(sample_yaml_files): - logger.info('[TEST %d]: %s', index, new_cr['name']) - addEvent = RayJobAddCREvent(new_cr['cr'], [rs], 300, NAMESPACE, new_cr['path']) - test_cases.addTest(GeneralTestCase('runtest', addEvent)) - - # Execute all testsCRs - runner = unittest.TextTestRunner() - test_result = runner.run(test_cases) - - # Without this line, the exit code will always be 0. - assert test_result.wasSuccessful()