Set RAY_USAGE_STATS_EXTRA_TAGS for release tests (#26366)

- Record the test name for the usage stats.
- Change the cluster name to indicate if it's smoke test or not.

python/ray/_private/usage/usage_constants.py

@@ -34,5 +34,3 @@
 LIBRARY_USAGE_PREFIX = "library_usage_"
 
 USAGE_STATS_NAMESPACE = "usage_stats"
-
-EXTRA_USAGE_TAGS = "RAY_EXTRA_USAGE_TAGS"

python/ray/_private/usage/usage_lib.py

@@ -135,7 +135,7 @@ class UsageStatsToReport:
     #: The sequence number of the report.
     seq_number: int
     #: The extra tags to report when specified by an
-    #  environment variable EXTRA_USAGE_TAGS
+    #  environment variable RAY_USAGE_STATS_EXTRA_TAGS
     extra_usage_tags: Optional[Dict[str, str]]
     #: The number of alive nodes when the report is generated.
     total_num_nodes: Optional[int]
@@ -491,7 +491,7 @@ def _parse_extra_usage_tags() -> Dict[str, str]:
     Returns:
         Dictionary of key value pair parsed.
     """
-    extra_tags = os.getenv("RAY_EXTRA_USAGE_TAGS", None)
+    extra_tags = os.getenv("RAY_USAGE_STATS_EXTRA_TAGS", None)
     if not extra_tags:
         return None
 

python/ray/tests/test_usage_stats.py

@@ -102,28 +102,28 @@ def reset_lib_usage():
 def test_parse_extra_usage_tags(monkeypatch):
     with monkeypatch.context() as m:
         # Test a normal case.
-        m.setenv("RAY_EXTRA_USAGE_TAGS", "key=val;key2=val2")
+        m.setenv("RAY_USAGE_STATS_EXTRA_TAGS", "key=val;key2=val2")
         result = ray_usage_lib._parse_extra_usage_tags()
         assert result["key"] == "val"
         assert result["key2"] == "val2"
 
-        m.setenv("RAY_EXTRA_USAGE_TAGS", "key=val;key2=val2;")
+        m.setenv("RAY_USAGE_STATS_EXTRA_TAGS", "key=val;key2=val2;")
         result = ray_usage_lib._parse_extra_usage_tags()
         assert result["key"] == "val"
         assert result["key2"] == "val2"
 
         # Test that the env var is not given.
-        m.delenv("RAY_EXTRA_USAGE_TAGS")
+        m.delenv("RAY_USAGE_STATS_EXTRA_TAGS")
         result = ray_usage_lib._parse_extra_usage_tags()
         assert result is None
 
         # Test the parsing failure.
-        m.setenv("RAY_EXTRA_USAGE_TAGS", "key=val,key2=val2")
+        m.setenv("RAY_USAGE_STATS_EXTRA_TAGS", "key=val,key2=val2")
         result = ray_usage_lib._parse_extra_usage_tags()
         assert result is None
 
         # Test differnt types of parsing failures.
-        m.setenv("RAY_EXTRA_USAGE_TAGS", "key=v=al,key2=val2")
+        m.setenv("RAY_USAGE_STATS_EXTRA_TAGS", "key=v=al,key2=val2")
         result = ray_usage_lib._parse_extra_usage_tags()
         assert result is None
 
@@ -1070,7 +1070,7 @@ def test_usage_stats_tags(monkeypatch, ray_start_cluster, reset_lib_usage):
         m.setenv("RAY_USAGE_STATS_ENABLED", "1")
         m.setenv("RAY_USAGE_STATS_REPORT_URL", "http://127.0.0.1:8000/usage")
         m.setenv("RAY_USAGE_STATS_REPORT_INTERVAL_S", "1")
-        m.setenv("RAY_EXTRA_USAGE_TAGS", "key=val;key2=val2")
+        m.setenv("RAY_USAGE_STATS_EXTRA_TAGS", "key=val;key2=val2")
         cluster = ray_start_cluster
         cluster.add_node(num_cpus=3)
         cluster.add_node(num_cpus=3)

release/ray_release/cluster_manager/cluster_manager.py

@@ -12,15 +12,22 @@
 
 class ClusterManager(abc.ABC):
     def __init__(
-        self, test_name: str, project_id: str, sdk: Optional["AnyscaleSDK"] = None
+        self,
+        test_name: str,
+        project_id: str,
+        sdk: Optional["AnyscaleSDK"] = None,
+        smoke_test: bool = False,
     ):
         self.sdk = sdk or get_anyscale_sdk()
 
         self.test_name = test_name
+        self.smoke_test = smoke_test
         self.project_id = project_id
         self.project_name = get_project_name(self.project_id, self.sdk)
 
-        self.cluster_name = f"{test_name}_{int(time.time())}"
+        self.cluster_name = (
+            f"{test_name}{'-smoke-test' if smoke_test else ''}_{int(time.time())}"
+        )
         self.cluster_id = None
 
         self.cluster_env = None
@@ -44,6 +51,9 @@ def set_cluster_env(self, cluster_env: Dict[str, Any]):
         self.cluster_env["env_vars"]["RAY_bootstrap_with_gcs"] = "1"
         self.cluster_env["env_vars"]["RAY_USAGE_STATS_ENABLED"] = "1"
         self.cluster_env["env_vars"]["RAY_USAGE_STATS_SOURCE"] = "nightly-tests"
+        self.cluster_env["env_vars"][
+            "RAY_USAGE_STATS_EXTRA_TAGS"
+        ] = f"test_name={self.test_name};smoke_test={self.smoke_test}"
 
         self.cluster_env_name = (
             f"{self.project_name}_{self.project_id[4:8]}"

release/ray_release/glue.py

@@ -128,7 +128,9 @@ def run_release_test(
 
     # Instantiate managers and command runner
     try:
-        cluster_manager = cluster_manager_cls(test["name"], anyscale_project)
+        cluster_manager = cluster_manager_cls(
+            test["name"], anyscale_project, smoke_test=smoke_test
+        )
         file_manager = file_manager_cls(cluster_manager=cluster_manager)
         command_runner = command_runner_cls(cluster_manager, file_manager, working_dir)
     except Exception as e:

release/ray_release/tests/test_cluster_manager.py

@@ -100,6 +100,38 @@ def setUp(self) -> None:
         )
         self.sdk.reset()
 
+    def testClusterName(self):
+        sdk = MockSDK()
+        sdk.returns["get_project"] = APIDict(result=APIDict(name="release_unit_tests"))
+        cluster_manager = self.cls(
+            test_name="test", project_id=UNIT_TEST_PROJECT_ID, smoke_test=False, sdk=sdk
+        )
+        self.assertRegex(cluster_manager.cluster_name, r"^test_\d+$")
+        cluster_manager = self.cls(
+            test_name="test", project_id=UNIT_TEST_PROJECT_ID, smoke_test=True, sdk=sdk
+        )
+        self.assertRegex(cluster_manager.cluster_name, r"^test-smoke-test_\d+$")
+
+    def testSetClusterEnv(self):
+        sdk = MockSDK()
+        sdk.returns["get_project"] = APIDict(result=APIDict(name="release_unit_tests"))
+        cluster_manager = self.cls(
+            test_name="test", project_id=UNIT_TEST_PROJECT_ID, smoke_test=False, sdk=sdk
+        )
+        cluster_manager.set_cluster_env({})
+        self.assertEqual(
+            cluster_manager.cluster_env["env_vars"]["RAY_USAGE_STATS_EXTRA_TAGS"],
+            "test_name=test;smoke_test=False",
+        )
+        cluster_manager = self.cls(
+            test_name="Test", project_id=UNIT_TEST_PROJECT_ID, smoke_test=True, sdk=sdk
+        )
+        cluster_manager.set_cluster_env({})
+        self.assertEqual(
+            cluster_manager.cluster_env["env_vars"]["RAY_USAGE_STATS_EXTRA_TAGS"],
+            "test_name=Test;smoke_test=True",
+        )
+
     @patch("time.sleep", lambda *a, **kw: None)
     def testFindCreateClusterComputeExisting(self):
         # Find existing compute and succeed

release/ray_release/tests/test_glue.py

@@ -108,9 +108,15 @@ def setUp(self) -> None:
         this_file_manager_return = self.file_manager_return
 
         class MockClusterManager(MockReturn, FullClusterManager):
-            def __init__(self, test_name: str, project_id: str, sdk=None):
+            def __init__(
+                self,
+                test_name: str,
+                project_id: str,
+                sdk=None,
+                smoke_test: bool = False,
+            ):
                 super(MockClusterManager, self).__init__(
-                    test_name, project_id, this_sdk
+                    test_name, project_id, this_sdk, smoke_test=smoke_test
                 )
                 self.return_dict = this_cluster_manager_return