Extend drift detect server to expose metrics

components/alibi-detect-server/adserver/cd_model.py

@@ -5,6 +5,55 @@
 from .numpy_encoder import NumpyEncoder
 from alibi_detect.utils.saving import load_detector, Data
 from adserver.base import AlibiDetectModel
+from seldon_core.user_model import SeldonResponse
+
+
+def _drift_to_metrics(drift):
+    metrics = []
+
+    batch_score = drift.get("batch_score")
+    if batch_score:
+        metrics.append(
+            {
+                "key": "seldon_metric_drift_batch_score",
+                "value": batch_score,
+                "type": "GAUGE",
+            }
+        )
+
+    feature_score = drift.get("feature_score")
+    if feature_score:
+        metrics.append(
+            {
+                "key": "seldon_metric_drift_feature_score",
+                "value": feature_score,
+                "type": "COUNTER",
+            }
+        )
+
+    is_drift = drift.get("is_drift")
+    if is_drift:
+        metrics.append(
+            {
+                "key": "seldon_metric_drift_is_drift",
+                "value": is_drift,
+                "type": "COUNTER",
+            }
+        )
+
+    p_val = drift.get("p_val")
+    if p_val and isinstance(p_val, list):
+        for i, p in enumerate(p_val):
+            metrics.append(
+                {
+                    "key": "seldon_metric_drift_p_val",
+                    "value": p,
+                    "type": "COUNTER",
+                    "tags": {"p_val_index": str(i)},
+                }
+            )
+
+    return metrics
 
 
 class AlibiDetectConceptDriftModel(
@@ -75,7 +124,14 @@ def process_event(self, inputs: List, headers: Dict) -> Optional[Dict]:
             )
             cd_preds = self.model.predict(self.batch)
             self.batch = None
-            return json.loads(json.dumps(cd_preds, cls=NumpyEncoder))
+
+            output = json.loads(json.dumps(cd_preds, cls=NumpyEncoder))
+
+            metrics = _drift_to_metrics(output.get("data", {}))
+
+            seldon_response = SeldonResponse(output, None, metrics)
+
+            return seldon_response
         else:
             logging.info(
                 "Not running drift detection. Batch size is %d. Need %d",

components/alibi-detect-server/adserver/server.py

@@ -233,6 +233,13 @@ def post(self):
         response = self.model.process_event(request, headers)
         seldon_response = SeldonResponse.create(response)
 
+        runtime_metrics = seldon_response.metrics
+        if runtime_metrics is not None:
+            if validate_metrics(runtime_metrics):
+                self.seldon_metrics.update(runtime_metrics, self.event_type)
+            else:
+                logging.error("Metrics returned are invalid: " + str(runtime_metrics))
+
         if seldon_response.data is not None:
             responseStr = json.dumps(seldon_response.data)
 
@@ -255,16 +262,6 @@ def post(self):
                 sendCloudEvent(revent, self.reply_url)
             self.write(json.dumps(seldon_response.data))
 
-        runtime_metrics = seldon_response.metrics
-        if runtime_metrics is not None:
-            if not validate_metrics(runtime_metrics):
-                raise SeldonMicroserviceException(
-                    f"Bad metric created during request: {json.dumps(runtime_metrics)}",
-                    status_code=500,
-                    reason="MICROSERVICE_BAD_METRIC",
-                )
-            self.seldon_metrics.update(runtime_metrics, "ce_server")
-
 
 class LivenessHandler(tornado.web.RequestHandler):
     def get(self):

components/alibi-detect-server/adserver/tests/test_cd_model.py

@@ -31,7 +31,7 @@ def test_basic(self):
         req = [[1, 2]]
         headers = {}
         res = ad_model.process_event(req, headers)
-        self.assertEqual(res["data"]["is_drift"], 0)
+        self.assertEqual(res.data["data"]["is_drift"], 0)
 
     def test_batch(self):
         model = DummyCDModel()
@@ -44,7 +44,7 @@ def test_batch(self):
         self.assertEqual(res, None)
 
         res = ad_model.process_event(req, headers)
-        self.assertEqual(res["data"]["is_drift"], 0)
+        self.assertEqual(res.data["data"]["is_drift"], 0)
 
         res = ad_model.process_event(req, headers)
         self.assertEqual(res, None)

components/alibi-detect-server/setup.py

@@ -11,7 +11,7 @@
     python_requires=">3.4",
     packages=find_packages(),
     install_requires=[
-        "alibi-detect",
+        "alibi-detect==0.4.1",
         "kfserving>=0.2.0",
         "argparse >= 1.4.0",
         "numpy >= 1.8.2",