Nop Ensembler Config (#192)

* UI changes for nop ensembler config

* Make handling of router / version status consistent

* SDK changes for default route

* Correct the default route id in unit tests

* Add tests for the nop ensembler config

* Update sample code and doc

* Add PR comments

Co-authored-by: Krithika Sundararajan <[email protected]>

docs/how-to/create-a-router/configure-ensembler.md

@@ -9,7 +9,9 @@ Turing currently supports ensemblers in the same fashion as the enrichers. The e
 Currently, there are 4 options available - no ensembler, a standard ensembler, Docker and an external ensembler.
 
 ## No Ensembler
-The router will return a response from the default route, specified in Configure Routes. This option is available only when **no experiment engine** is configured in Configure Experiment Engine.
+The router will return a response from the route configured to act as the final response. This option is available only when **no experiment engine** is configured in Configure Experiment Engine.
+
+![](../../.gitbook/assets/nop_ensembler_config.png)
 
 ## Docker
 Turing will deploy specified image as a post-processor and will send the original request, responses from all routes, and the treatment configuration (if a Experiment Engine is selected, in Configure Experiment Engine), for ensembling. To configure a Docker ensembler, there are 3 sections to be filled.

sdk/README.md

@@ -19,14 +19,14 @@ Check out [samples](./samples) for examples on how to use Turing SDK.
 
 #### Prerequisites
 
-* Python >= 3.8
+* Python 3.7.*
 * openapi-generator >= 5.1.0 (`brew install openapi-generator`)
 
 ### Make commands
 
 * Setup development environment
 ```shell
-make dev
+make setup
 ```
 
 * (Re-)generate openapi client
@@ -36,5 +36,5 @@ make gen-client
 
 * Run unit tests
 ```shell
-make test-unit
+make test
 ```

sdk/samples/router/general.py

@@ -188,10 +188,10 @@ def main(turing_api: str, project: str):
     )
 
     # Create an ensembler for the router
-    # Note: Ensembling for Turing Routers is done through Standard, Docker or Pyfunc ensemblers, and its configuration
-    # is managed by the `RouterEnsemblerConfig` class. Three helper classes (child classes of `RouterEnsemblerConfig`)
-    # have been created to assist you in constructing these objects - `StandardRouterEnsemblerConfig`,
-    # `DockerRouterEnsemblerConfig` and `PyfuncRouterEnsemblerConfig`.
+    # Note: Ensembling for Turing Routers is done through Nop, Standard, Docker or Pyfunc ensemblers, and its configuration
+    # is managed by the `RouterEnsemblerConfig` class. Helper classes (child classes of `RouterEnsemblerConfig`)
+    # have been created to assist you in constructing these objects - `NopRouterEnsemblerConfig`,
+    # `StandardRouterEnsemblerConfig`, `DockerRouterEnsemblerConfig` and `PyfuncRouterEnsemblerConfig`.
     ensembler = DockerRouterEnsemblerConfig(
         image="ealen/echo-server:0.5.1",
         resource_request=ResourceRequest(

sdk/tests/conftest.py

@@ -356,7 +356,6 @@ def ensembler(request, generic_ensembler_standard_config, generic_ensembler_dock
         updated_at=datetime.now() + timedelta(seconds=10)
     )
 
-
 @pytest.fixture
 def generic_standard_router_ensembler_config(generic_ensembler_standard_config):
     return turing.generated.models.RouterEnsemblerConfig(
@@ -454,8 +453,8 @@ def generic_router_version(
         error="NONE",
         image="test.io/just-a-test/turing-router:0.0.0-build.0",
         routes=[generic_route for _ in range(2)],
-        default_route="http://models.internal/default",
-        default_route_id="control",
+        default_route=generic_route.endpoint,
+        default_route_id=generic_route.id,
         rules=[generic_traffic_rule for _ in range(2)],
         experiment_engine=experiment_config,
         resource_request=generic_resource_request,
@@ -485,7 +484,7 @@ def generic_router_config():
             )
         ],
         rules=None,
-        default_route_id="test",
+        default_route_id="model-a",
         experiment_engine=ExperimentConfig(
             type="test-exp",
             config={

sdk/tests/router/config/router_config_test.py

@@ -1,5 +1,5 @@
 import pytest
-from turing.router.config.route import Route, DuplicateRouteException
+from turing.router.config.route import Route, DuplicateRouteException, InvalidRouteException
 
 
 @pytest.mark.parametrize(
@@ -18,3 +18,17 @@ def test_set_router_config_with_invalid_routes(actual, new_routes, expected, req
     actual.routes = new_routes
     with pytest.raises(expected):
         actual.to_open_api()
+
+@pytest.mark.parametrize(
+    "actual,invalid_route_id,expected", [
+        pytest.param(
+            "generic_router_config",
+            "test-route-not-exists",
+            InvalidRouteException
+        )
+    ])
+def test_set_router_config_with_invalid_default_route(actual, invalid_route_id, expected, request):
+    actual = request.getfixturevalue(actual)
+    actual.default_route_id = invalid_route_id
+    with pytest.raises(expected):
+        actual.to_open_api()

sdk/tests/router/config/router_ensembler_config_test.py

@@ -3,13 +3,15 @@
 from turing.generated.exceptions import ApiValueError
 from turing.router.config.common.env_var import EnvVar
 from turing.router.config.resource_request import ResourceRequest
+from turing.router.config.route import InvalidRouteException
 from turing.router.config.router_ensembler_config import (RouterEnsemblerConfig,
+                                                          EnsemblerNopConfig,
+                                                          NopRouterEnsemblerConfig,
                                                           PyfuncRouterEnsemblerConfig,
                                                           DockerRouterEnsemblerConfig,
                                                           StandardRouterEnsemblerConfig,
                                                           InvalidExperimentMappingException)
 
-
 @pytest.mark.parametrize(
     "id,type,standard_config,docker_config,expected", [
         pytest.param(
@@ -437,3 +439,59 @@ def test_set_standard_router_ensembler_config_with_valid_experiment_mappings(
     )
     actual.experiment_mappings = new_experiment_mappings
     assert actual.to_open_api() == request.getfixturevalue(expected)
+
+
+@pytest.mark.parametrize(
+    "final_response_route_id,nop_config,expected", [
+        pytest.param(
+            "test-route",
+            EnsemblerNopConfig(final_response_route_id="test-route"),
+            None
+        )
+    ])
+def test_create_nop_router_ensembler_config(
+    final_response_route_id,
+    nop_config,
+    expected):
+    ensembler = NopRouterEnsemblerConfig(final_response_route_id=final_response_route_id)
+    assert ensembler.nop_config == nop_config
+    assert ensembler.to_open_api() == expected
+
+@pytest.mark.parametrize(
+    "router_config,ensembler_config", [
+        pytest.param(
+            "generic_router_config",
+           NopRouterEnsemblerConfig(final_response_route_id="model-b"),
+        )
+    ])
+def test_copy_nop_ensembler_default_route(
+    router_config,
+    ensembler_config,
+    request):
+    router = request.getfixturevalue(router_config)
+    # Check precondition
+    assert router.default_route_id != ensembler_config.final_response_route_id
+
+    router.ensembler = ensembler_config
+    actual = router.to_open_api()
+    router.default_route_id = ensembler_config.final_response_route_id
+    expected = router.to_open_api()
+    assert actual == expected
+
+@pytest.mark.parametrize(
+    "router_config,ensembler_config,expected", [
+        pytest.param(
+            "generic_router_config",
+           NopRouterEnsemblerConfig(final_response_route_id="test-route-not-exists"),
+           InvalidRouteException,
+        )
+    ])
+def test_create_nop_router_ensembler_config_with_invalid_route(
+    router_config,
+    ensembler_config,
+    expected,
+    request):
+    router = request.getfixturevalue(router_config)
+    router.ensembler = ensembler_config
+    with pytest.raises(expected):
+       router.to_open_api()

sdk/tests/testdata/api_responses/create_router_0000.json

@@ -31,7 +31,7 @@
         "timeout": "100ms"
       }
     ],
-    "default_route_id": "test",
+    "default_route_id": "model-a",
     "experiment_engine": {
       "type": "test-exp",
       "config": {

sdk/turing/router/config/route.py

@@ -73,6 +73,8 @@ def to_open_api(self) -> OpenApiModel:
 class InvalidUrlException(Exception):
     pass
 
-
 class DuplicateRouteException(Exception):
     pass
+
+class InvalidRouteException(Exception):
+    pass

sdk/turing/router/config/router_config.py

@@ -10,7 +10,7 @@
 from turing.router.config.resource_request import ResourceRequest
 from turing.router.config.log_config import LogConfig, ResultLoggerType
 from turing.router.config.enricher import Enricher
-from turing.router.config.router_ensembler_config import RouterEnsemblerConfig
+from turing.router.config.router_ensembler_config import RouterEnsemblerConfig, NopRouterEnsemblerConfig
 from turing.router.config.experiment_config import ExperimentConfig
 
 
@@ -73,7 +73,8 @@ def __init__(self,
         self.timeout = timeout
         self.log_config = log_config
         self.enricher = enricher
-        self.ensembler = ensembler
+        # Init nop ensembler config if ensembler is not set
+        self.ensembler = ensembler or NopRouterEnsemblerConfig(final_response_route_id=default_route_id)
 
     @property
     def environment_name(self) -> str:
@@ -207,6 +208,9 @@ def ensembler(self, ensembler: Union[RouterEnsemblerConfig, Dict]):
     def to_open_api(self) -> OpenApiModel:
         kwargs = {}
         self._verify_no_duplicate_routes()
+
+        # Get default route id before processing the ensembler
+        kwargs['default_route_id'] = self._get_default_route_id()
 
         if self.rules is not None:
             kwargs['rules'] = [rule.to_open_api() for rule in self.rules]
@@ -216,20 +220,40 @@ def to_open_api(self) -> OpenApiModel:
             kwargs['enricher'] = self.enricher.to_open_api()
         if self.ensembler is not None:
             kwargs['ensembler'] = self.ensembler.to_open_api()
+            if kwargs['ensembler'] is None:
+                # The Turing API does not handle an ensembler type "nop" - it must be left unset.
+                del kwargs['ensembler']
 
         return turing.generated.models.RouterConfig(
             environment_name=self.environment_name,
             name=self.name,
             config=turing.generated.models.RouterVersionConfig(
                 routes=[route.to_open_api() for route in self.routes],
-                default_route_id=self.default_route_id,
                 experiment_engine=self.experiment_engine.to_open_api(),
                 timeout=self.timeout,
                 log_config=self.log_config.to_open_api(),
                 **kwargs
             )
         )
 
+    def _get_default_route_id(self):
+        default_route_id = self.default_route_id
+        # If nop config is set, use the final_response_route_id as the default
+        if (self.ensembler.type == "nop" and
+            self.ensembler.nop_config is not None and
+            self.ensembler.nop_config.final_response_route_id is not None):
+            default_route_id = self.ensembler.nop_config.final_response_route_id
+        self._verify_default_route_exists(default_route_id)
+        return default_route_id
+
+    def _verify_default_route_exists(self, default_route_id: str):
+        for route in self.routes:
+            if route.id == default_route_id:
+                return
+        raise turing.router.config.route.InvalidRouteException(
+                f"Default route id {default_route_id} is not registered in the routes."
+            )
+
     def _verify_no_duplicate_routes(self):
         route_id_counter = Counter(route.id for route in self.routes)
         most_common_route_id, max_frequency = route_id_counter.most_common(n=1)[0]

sdk/turing/router/config/router_ensembler_config.py

@@ -1,37 +1,57 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 import turing.generated.models
 from typing import List, Dict, Union
 from turing.generated.model_utils import OpenApiModel
 from turing.router.config.resource_request import ResourceRequest
 from turing.router.config.common.env_var import EnvVar
 
+@dataclass
+class EnsemblerNopConfig:
+    final_response_route_id: str
+
+    _final_response_route_id: str = field(init=False, repr=False)
+
+    @property
+    def final_response_route_id(self) -> str:
+        return self._final_response_route_id
+
+    @final_response_route_id.setter
+    def final_response_route_id(self, final_response_route_id: str):
+        self._final_response_route_id = final_response_route_id
+
+    def to_open_api(self) -> OpenApiModel:
+        return None
+
 
 @dataclass
 class RouterEnsemblerConfig:
     """
     Class to create a new RouterEnsemblerConfig
 
-    :param type: type of the ensembler; must be one of {'standard', 'docker'}
+    :param type: type of the ensembler; must be one of {'nop', 'standard', 'docker', 'pyfunc'}
     :param id: id of the ensembler
     :param standard_config: EnsemblerStandardConfig instance containing mappings between routes and treatments
     :param docker_config: EnsemblerDockerConfig instance containing configs for the docker ensembler
     """
     type: str
     id: int = None
+    nop_config: EnsemblerNopConfig = None
     standard_config: turing.generated.models.EnsemblerStandardConfig = None
     docker_config: turing.generated.models.EnsemblerDockerConfig = None
     pyfunc_config: turing.generated.models.EnsemblerPyfuncConfig = None
 
     def __init__(self,
                  type: str,
                  id: int = None,
+                 nop_config: EnsemblerNopConfig = None,
                  standard_config: turing.generated.models.EnsemblerStandardConfig = None,
                  docker_config: turing.generated.models.EnsemblerDockerConfig = None,
                  pyfunc_config: turing.generated.models.EnsemblerPyfuncConfig = None,
                  **kwargs):
         self.id = id
         self.type = type
+        self.nop_config = nop_config
         self.standard_config = standard_config
         self.docker_config = docker_config
         self.pyfunc_config = pyfunc_config
@@ -50,7 +70,7 @@ def type(self) -> str:
 
     @type.setter
     def type(self, type: str):
-        assert type in {"standard", "docker", "pyfunc"}
+        assert type in {"nop", "standard", "docker", "pyfunc"}
         self._type = type
 
     @property
@@ -109,6 +129,21 @@ def pyfunc_config(self, pyfunc_config: turing.generated.models.EnsemblerPyfuncCo
         else:
             self._pyfunc_config = pyfunc_config
 
+    @property
+    def nop_config(self) -> EnsemblerNopConfig:
+        return self._nop_config
+
+    @nop_config.setter
+    def nop_config(self, nop_config: EnsemblerNopConfig):
+        if isinstance(nop_config, EnsemblerNopConfig):
+            self._nop_config = nop_config
+        elif isinstance(nop_config, dict):
+            self._nop_config = turing.generated.models.EnsemblerPyfuncConfig(
+                **nop_config
+            )
+        else:
+            self._nop_config = nop_config
+
     def to_open_api(self) -> OpenApiModel:
         kwargs = {}
 
@@ -350,6 +385,32 @@ def to_open_api(self) -> OpenApiModel:
         )
         return super().to_open_api()
 
+@dataclass
+class NopRouterEnsemblerConfig(RouterEnsemblerConfig):
+    def __init__(self,
+                 final_response_route_id: str):
+        """
+        Method to create a new Nop ensembler
+
+        :param final_response_route_id: The route id of the route to be returned as the final response
+        """
+        self.final_response_route_id = final_response_route_id
+        super().__init__(type="nop",
+            nop_config = EnsemblerNopConfig(final_response_route_id=self.final_response_route_id))
+
+    @property
+    def final_response_route_id(self) -> str:
+        return self._final_response_route_id
+
+    @final_response_route_id.setter
+    def final_response_route_id(self, final_response_route_id: str):
+        self._final_response_route_id = final_response_route_id
+
+    def to_open_api(self) -> OpenApiModel:
+        # Nop config is not passed down to the API. The final_response_route_id property
+        # will be parsed in the router config and copied over as appropriate.
+        return None
+
 
 class InvalidExperimentMappingException(Exception):
     pass