Add support to run benchmark against min distribution (#3684)

Signed-off-by: Rishabh Singh <[email protected]>

src/run_benchmark_test.py

@@ -6,7 +6,9 @@
 # compatible open source license.
 
 import sys
+from typing import Union
 
+from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
 from system import console
 from test_workflow.benchmark_test.benchmark_args import BenchmarkArgs
@@ -20,7 +22,8 @@ def main() -> int:
     """
     benchmark_args = BenchmarkArgs()
     console.configure(level=benchmark_args.logging_level)
-    manifest = BundleManifest.from_file(benchmark_args.bundle_manifest)
+    manifest: Union[BundleManifest, BuildManifest] = BundleManifest.from_file(benchmark_args.bundle_manifest) if not benchmark_args.min_distribution else \
+        BuildManifest.from_file(benchmark_args.bundle_manifest)
     BenchmarkTestRunners.from_args(benchmark_args, manifest).run()
     return 0
 

src/test_workflow/benchmark_test/benchmark_test_cluster.py

@@ -11,18 +11,19 @@
 import os
 import subprocess
 from contextlib import contextmanager
-from typing import Any, Generator
+from typing import Any, Generator, Union
 
 import requests
 from requests.auth import HTTPBasicAuth
 from retry.api import retry_call  # type: ignore
 
+from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
 from test_workflow.benchmark_test.benchmark_args import BenchmarkArgs
 
 
 class BenchmarkTestCluster:
-    manifest: BundleManifest
+    manifest: Union[BundleManifest, BuildManifest]
     work_dir: str
     current_workspace: str
     args: BenchmarkArgs
@@ -39,7 +40,7 @@ class BenchmarkTestCluster:
 
     def __init__(
             self,
-            bundle_manifest: BundleManifest,
+            bundle_manifest: Union[BundleManifest, BuildManifest],
             config: dict,
             args: BenchmarkArgs,
             current_workspace: str
@@ -121,7 +122,9 @@ def setup_cdk_params(self, config: dict) -> dict:
         else:
             suffix = self.manifest.build.id + '-' + self.manifest.build.architecture
         return {
-            "distributionUrl": self.manifest.build.location,
+            "distributionUrl": self.manifest.build.location if isinstance(self.manifest, BundleManifest) else
+            f"https://artifacts.opensearch.org/snapshots/core/opensearch/{self.manifest.build.version}/opensearch-min-"
+            f"{self.manifest.build.version}-linux-{self.manifest.build.architecture}-latest.tar.gz",
             "vpcId": config["Constants"]["VpcId"],
             "account": config["Constants"]["AccountId"],
             "region": config["Constants"]["Region"],

src/test_workflow/benchmark_test/benchmark_test_runner.py

@@ -7,18 +7,20 @@
 
 import abc
 import os
+from typing import Union
 
+from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
 from test_workflow.benchmark_test.benchmark_args import BenchmarkArgs
 
 
 class BenchmarkTestRunner(abc.ABC):
     args: BenchmarkArgs
-    test_manifest: BundleManifest
+    test_manifest: Union[BundleManifest, BuildManifest]
     security: bool
     tests_dir: str
 
-    def __init__(self, args: BenchmarkArgs, test_manifest: BundleManifest) -> None:
+    def __init__(self, args: BenchmarkArgs, test_manifest: Union[BundleManifest, BuildManifest]) -> None:
         self.args = args
         self.test_manifest = test_manifest
 

src/test_workflow/benchmark_test/benchmark_test_runner_opensearch.py

@@ -7,11 +7,13 @@
 
 import logging
 import os
+from typing import Union
 
 import yaml
 from retry.api import retry_call  # type: ignore
 
 from git.git_repository import GitRepository
+from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
 from system.temporary_directory import TemporaryDirectory
 from system.working_directory import WorkingDirectory
@@ -25,7 +27,7 @@ class BenchmarkTestRunnerOpenSearch(BenchmarkTestRunner):
     """
       Runner to execute the performance tests for opensearch.
     """
-    def __init__(self, args: BenchmarkArgs, test_manifest: BundleManifest) -> None:
+    def __init__(self, args: BenchmarkArgs, test_manifest: Union[BundleManifest, BuildManifest]) -> None:
         super().__init__(args, test_manifest)
         logging.info("Running opensearch tests")
 

src/test_workflow/benchmark_test/benchmark_test_runner_opensearch_plugins.py

@@ -4,8 +4,9 @@
 # The OpenSearch Contributors require contributions made to
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
+from typing import Union
 
-
+from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
 from test_workflow.benchmark_test.benchmark_args import BenchmarkArgs
 from test_workflow.benchmark_test.benchmark_test_runner import BenchmarkTestRunner
@@ -18,7 +19,7 @@ class BenchmarkTestRunnerOpenSearchPlugins(BenchmarkTestRunner):
     """
       Runner to execute the performance tests for opensearch plugins. The plugins need to define the test suite
     """
-    def __init__(self, args: BenchmarkArgs, test_manifest: BundleManifest) -> None:
+    def __init__(self, args: BenchmarkArgs, test_manifest: Union[BundleManifest, BuildManifest]) -> None:
         super().__init__(args, test_manifest)
 
     def get_plugin_repo_url(self) -> str:

src/test_workflow/benchmark_test/benchmark_test_runners.py

@@ -4,7 +4,9 @@
 # The OpenSearch Contributors require contributions made to
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
+from typing import Union
 
+from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
 from test_workflow.benchmark_test.benchmark_args import BenchmarkArgs
 from test_workflow.benchmark_test.benchmark_test_runner import BenchmarkTestRunner
@@ -19,5 +21,5 @@ class BenchmarkTestRunners:
     }
 
     @classmethod
-    def from_args(cls, args: BenchmarkArgs, test_manifest: BundleManifest) -> BenchmarkTestRunner:
+    def from_args(cls, args: BenchmarkArgs, test_manifest: Union[BundleManifest, BuildManifest]) -> BenchmarkTestRunner:
         return cls.RUNNERS.get(args.component, BenchmarkTestRunnerOpenSearchPlugins)(args, test_manifest)

tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/data/min_distribution_manifest.yml

@@ -0,0 +1,44 @@
+---
+schema-version: '1.2'
+build:
+  name: OpenSearch
+  version: 2.9.0-SNAPSHOT
+  platform: linux
+  architecture: arm64
+  distribution: tar
+  id: '8042'
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: 2.x
+    commit_id: 2c013429a028291b6136e3df4b0eb3d5759eaa06
+    artifacts:
+      maven:
+        - maven/org/opensearch/rest-api-spec/maven-metadata.xml
+        - maven/org/opensearch/rest-api-spec/maven-metadata.xml.sha1
+        - maven/org/opensearch/rest-api-spec/maven-metadata.xml.md5
+      dist:
+        - dist/opensearch-min-2.9.0-SNAPSHOT-linux-arm64.tar.gz
+      core-plugins:
+        - core-plugins/analysis-icu-2.9.0-SNAPSHOT.zip
+        - core-plugins/analysis-kuromoji-2.9.0-SNAPSHOT.zip
+        - core-plugins/analysis-nori-2.9.0-SNAPSHOT.zip
+        - core-plugins/analysis-phonetic-2.9.0-SNAPSHOT.zip
+        - core-plugins/analysis-smartcn-2.9.0-SNAPSHOT.zip
+        - core-plugins/analysis-stempel-2.9.0-SNAPSHOT.zip
+        - core-plugins/analysis-ukrainian-2.9.0-SNAPSHOT.zip
+        - core-plugins/discovery-azure-classic-2.9.0-SNAPSHOT.zip
+        - core-plugins/discovery-ec2-2.9.0-SNAPSHOT.zip
+        - core-plugins/discovery-gce-2.9.0-SNAPSHOT.zip
+        - core-plugins/identity-shiro-2.9.0-SNAPSHOT.zip
+        - core-plugins/ingest-attachment-2.9.0-SNAPSHOT.zip
+        - core-plugins/mapper-annotated-text-2.9.0-SNAPSHOT.zip
+        - core-plugins/mapper-murmur3-2.9.0-SNAPSHOT.zip
+        - core-plugins/mapper-size-2.9.0-SNAPSHOT.zip
+        - core-plugins/repository-azure-2.9.0-SNAPSHOT.zip
+        - core-plugins/repository-gcs-2.9.0-SNAPSHOT.zip
+        - core-plugins/repository-hdfs-2.9.0-SNAPSHOT.zip
+        - core-plugins/repository-s3-2.9.0-SNAPSHOT.zip
+        - core-plugins/store-smb-2.9.0-SNAPSHOT.zip
+        - core-plugins/transport-nio-2.9.0-SNAPSHOT.zip
+    version: 2.9.0.0-SNAPSHOT

tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_cluster.py

@@ -50,6 +50,8 @@ def test_create_single_node_secure(self, mock_wait_for_processing: Optional[Mock
         self.assertTrue("securityDisabled=false" in self.benchmark_test_cluster.params)
         self.assertTrue("singleNodeCluster=true" in self.benchmark_test_cluster.params)
         self.assertTrue("isInternal=true" in self.benchmark_test_cluster.params)
+        self.assertTrue("distributionUrl=https://artifacts.opensearch.org/bundles/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/opensearch-1.0.0-linux-x64.tar.gz" in self.benchmark_test_cluster.params)
+        self.assertTrue(isinstance(self.manifest, BundleManifest))
         with patch("subprocess.check_call") as mock_check_call:
             self.benchmark_test_cluster.terminate()
             self.assertEqual(mock_check_call.call_count, 1)

tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_cluster_min.py

@@ -0,0 +1,49 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import os
+import unittest
+from typing import Optional
+from unittest.mock import MagicMock, Mock, patch
+
+from manifests.build_manifest import BuildManifest
+from test_workflow.benchmark_test.benchmark_test_cluster import BenchmarkTestCluster
+
+
+class TestBenchmarkTestClusterMin(unittest.TestCase):
+    DATA = os.path.join(os.path.dirname(__file__), "data")
+    MIN_MANIFEST = os.path.join(DATA, "min_distribution_manifest.yml")
+
+    def setUp(self, args: Optional[Mock] = None) -> None:
+        self.args = Mock()
+
+        self.args.workload = "nyc_taxis"
+        self.args.stack_suffix = "test-suffix"
+        self.args.insecure = True
+        self.args.single_node = False
+        self.args.min_distribution = True
+        self.manifest = BuildManifest.from_path(self.MIN_MANIFEST)
+        self.stack_name = "stack"
+        self.security = True
+        self.config = {"Constants": {"SecurityGroupId": "sg-00000000", "VpcId": "vpc-12345", "AccountId": "12345678",
+                                     "Region": "us-west-2", "Role": "role-arn", "serverAccessType": "prefixList", "restrictServerAccessTo": "pl-1234",
+                                     "isInternal": "true"}}
+        self.benchmark_test_cluster = BenchmarkTestCluster(bundle_manifest=self.manifest, config=self.config, args=self.args, current_workspace="current_workspace")
+
+    @patch("test_workflow.benchmark_test.benchmark_test_cluster.BenchmarkTestCluster.wait_for_processing")
+    def test_create_min_cluster(self, mock_wait_for_processing: Optional[Mock]) -> None:
+        mock_file = MagicMock(side_effect=[{"opensearch-infra-stack-test-suffix-8042-arm64": {"loadbalancerurl": "www.example.com"}}])
+        with patch("subprocess.check_call") as mock_check_call:
+            with patch("builtins.open", MagicMock()):
+                with patch("json.load", mock_file):
+                    self.benchmark_test_cluster.start()
+                    self.assertEqual(mock_check_call.call_count, 1)
+        self.assertTrue(isinstance(self.manifest, BuildManifest))
+        self.assertTrue("securityDisabled=true" in self.benchmark_test_cluster.params)
+        self.assertTrue("minDistribution=true" in self.benchmark_test_cluster.params)
+        self.assertTrue("distributionUrl=https://artifacts.opensearch.org/snapshots/core/opensearch/2.9.0-SNAPSHOT/"
+                        "opensearch-min-2.9.0-SNAPSHOT-linux-arm64-latest.tar.gz" in self.benchmark_test_cluster.params)