Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Removing proto functionality from REST /predict endpoint [#803] #806

Merged
merged 5 commits into from
Aug 22, 2019
Merged

Removing proto functionality from REST /predict endpoint [#803] #806

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8c352a9
Added initial implementation with tests
axsaucedo Aug 15, 2019
47f2d8a
removed print statements
axsaucedo Aug 15, 2019
d225ea8
Added seldon core utils
axsaucedo Aug 15, 2019
4918ff9
Added tests
axsaucedo Aug 22, 2019
1767430
Merged with master
axsaucedo Aug 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions python/seldon_core/seldon_methods.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ def predict(
return construct_response(user_model, False, request, client_response)
else:
(features, meta, datadef, data_type) = extract_request_parts_json(request)
client_response = client_predict(user_model, features, datadef.names, meta=meta)
print(client_response)
class_names = datadef["names"] if datadef and "names" in datadef else []
client_response = client_predict(user_model, features, class_names, meta=meta)
return construct_response_json(user_model, False, request, client_response)


Expand Down
133 changes: 115 additions & 18 deletions python/seldon_core/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
from google.protobuf.json_format import MessageToDict, ParseDict
from seldon_core.proto import prediction_pb2
from seldon_core.flask_utils import SeldonMicroserviceException
from tensorflow.core.framework.tensor_pb2 import TensorProto
import numpy as np
import sys
import tensorflow as tf
from google.protobuf.struct_pb2 import ListValue
from seldon_core.user_model import client_class_names, client_custom_metrics, client_custom_tags, client_feature_names, \
SeldonComponent
from typing import Tuple, Dict, Union, List, Optional, Iterable, Any
import base64


def json_to_seldon_message(message_json: Union[List, Dict]) -> prediction_pb2.SeldonMessage:
Expand Down Expand Up @@ -142,7 +144,6 @@ def get_data_from_proto(request: prediction_pb2.SeldonMessage) -> Union[np.ndarr
else:
raise SeldonMicroserviceException("Unknown data in SeldonMessage")


def grpc_datadef_to_array(datadef: prediction_pb2.DefaultData) -> np.ndarray:
"""
Convert a SeldonMessage DefaultData to a numpy array.
Expand Down Expand Up @@ -301,8 +302,11 @@ def array_to_list_value(array: np.ndarray, lv: Optional[ListValue] = None) -> Li
array_to_list_value(sub_array, sub_lv)
return lv

def construct_response_json(user_model: SeldonComponent, is_request: bool, client_request_raw: Union[List, Dict],
client_raw_response: Union[np.ndarray, str, bytes, dict]) -> Union[List, Dict]:
def construct_response_json(
user_model: SeldonComponent,
is_request: bool,
client_request_raw: Union[List, Dict],
client_raw_response: Union[np.ndarray, str, bytes, dict]) -> Union[List, Dict]:
"""
This class converts a raw REST response into a JSON object that has the same structure as
the SeldonMessage proto. This is necessary as the conversion using the SeldonMessage proto
Expand All @@ -324,18 +328,80 @@ def construct_response_json(user_model: SeldonComponent, is_request: bool, clien
A SeldonMessage JSON response

"""
client_request = json_to_seldon_message(client_request_raw)
data_type = client_request.WhichOneof("data_oneof")
response = {}

sm = construct_response(user_model, is_request, client_request, client_raw_response)
sm_json = seldon_message_to_json(sm)
if "jsonData" in client_request_raw:
response["jsonData"] = client_raw_response
elif isinstance(client_raw_response, (bytes, bytearray)):
response["binData"] = client_raw_response
elif isinstance(client_raw_response, str):
response["strData"] = client_raw_response
else:
is_np = isinstance(client_raw_response, np.ndarray)
is_list = isinstance(client_raw_response, list)
if not (is_np or is_list):
raise SeldonMicroserviceException(
"Unknown data type returned as payload (must be list or np array):"
+ str(client_raw_response))
if is_np:
np_client_raw_response = client_raw_response
list_client_raw_response = client_raw_response.tolist()
else:
np_client_raw_response = np.array(client_raw_response)
list_client_raw_response = client_raw_response

result_client_response = None

response["data"] = {}
if "data" in client_request_raw:
if np.issubdtype(np_client_raw_response.dtype, np.number):
if "tensor" in client_request_raw["data"]:
default_data_type = "tensor"
result_client_response = {
"values": list_client_raw_response,
"shape": np_client_raw_response.shape
}
elif "tftensor" in client_request_raw["data"]:
default_data_type = "tftensor"
tf_json_str = json_format.MessageToJson(
tf.make_tensor_proto(np_client_raw_response))
result_client_response = json.loads(tf_json_str)
else:
default_data_type = "ndarray"
result_client_response = list_client_raw_response
else:
default_data_type = "ndarray"
result_client_response = list_client_raw_response
else:
if np.issubdtype(np_client_raw_response.dtype, np.number):
default_data_type = "tensor"
result_client_response = {
"values": np_client_raw_response.ravel().tolist(),
"shape": np_client_raw_response.shape
}
else:
default_data_type = "ndarray"
result_client_response = list_client_raw_repsonse

response_data_type = sm.WhichOneof("data_oneof")
response["data"][default_data_type] = result_client_response

if response_data_type == "jsonData":
sm_json["jsonData"] = client_raw_response
if is_request:
req_names = client_request_raw.get("data", {}).get("names", [])
names = client_feature_names(user_model, req_names)
else:
names = client_class_names(user_model, np_client_raw_response)
response["data"]["names"] = names

return sm_json
response["meta"] = {}
client_custom_tags(user_model)
tags = client_custom_tags(user_model)
if tags:
response["meta"]["tags"] = tags
metrics = client_custom_metrics(user_model)
if metrics:
response["meta"]["metrics"] = metrics

return response


def construct_response(user_model: SeldonComponent, is_request: bool, client_request: prediction_pb2.SeldonMessage,
Expand Down Expand Up @@ -397,8 +463,12 @@ def construct_response(user_model: SeldonComponent, is_request: bool, client_req
raise SeldonMicroserviceException("Unknown data type returned as payload:" + client_raw_response)


def extract_request_parts_json(request_raw: Union[Dict, List]) -> Tuple[
Union[np.ndarray, str, bytes, dict], Dict, prediction_pb2.DefaultData, str]:
def extract_request_parts_json(request: Union[Dict, List]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This PR looks good to me.
Probably not related to this PR, but I was wondering why gRPC path should not go this way.
From my profiler commented in issue it shows in gRPC the input was first converted to datadef in SeldonMessage:

100    0.240    0.002    1.281    0.013 utils.py:234(array_to_grpc_datadef)
20100/100    0.038    0.000    1.041    0.010 utils.py:280(array_to_list_value)

then in predict function it is converted back to Numpy array:

100    0.000    0.000    2.781    0.028 utils.py:505(extract_request_parts)
100    0.001    0.000    2.778    0.028 utils.py:120(get_data_from_proto)
100    0.001    0.000    2.778    0.028 utils.py:147(grpc_datadef_to_array)

The round-trip conversion seems double the latency I guess?

) -> Tuple[
Union[np.ndarray, str, bytes, Dict, List],
Union[Dict, None],
Union[np.ndarray, str, bytes, Dict, List, None],
str]:
"""

Parameters
Expand All @@ -411,11 +481,38 @@ def extract_request_parts_json(request_raw: Union[Dict, List]) -> Tuple[
Key parts of the request extracted

"""
request_proto = json_to_seldon_message(request_raw)
(features, meta, datadef, data_type) = extract_request_parts(request_proto)

if data_type == "jsonData":
features = request_raw["jsonData"]
meta = request.get("meta", None)
datadef_type = None
datadef = None

if "data" in request:
data_type = "data"
datadef = request["data"]
if "tensor" in datadef:
datadef_type = "tensor"
tensor = datadef["tensor"]
features = np.array(tensor["values"]).reshape(tensor["shape"])
elif "ndarray" in datadef:
datadef_type = "ndarray"
features = np.array(datadef["ndarray"])
elif "tftensor" in datadef:
datadef_type = "tftensor"
tf_proto = TensorProto()
json_format.ParseDict(datadef["tftensor"], tf_proto)
features = tf.make_ndarray(tf_proto)
else:
features = np.array([])
elif "jsonData" in request:
data_type = "jsonData"
features = request["jsonData"]
elif "strData" in request:
data_type = "strData"
features = request["strData"]
elif "binData" in request:
data_type = "binData"
features = bytes(request["binData"], "utf8")
else:
raise SeldonMicroserviceException(f"Invalid request data type: {request}")

return features, meta, datadef, data_type

Expand Down
95 changes: 84 additions & 11 deletions python/tests/test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,17 +50,44 @@ def metrics(self):
else:
return [{"type": "BAD", "key": "mycounter", "value": 1}]


def test_create_reponse_nparray():
def test_create_rest_reponse_nparray():
user_model = UserObject()
request = {}
raw_response = np.array([[1, 2, 3]])
result = scu.construct_response_json(
user_model,
True,
request,
raw_response)
assert "tensor" in result.get("data", {})
assert result["data"]["tensor"]["values"] == [1, 2, 3]

def test_create_grpc_reponse_nparray():
user_model = UserObject()
request = prediction_pb2.SeldonMessage()
raw_response = np.array([[1, 2, 3]])
sm = scu.construct_response(user_model, True, request, raw_response)
assert sm.data.WhichOneof("data_oneof") == "tensor"
assert sm.data.tensor.values == [1, 2, 3]


def test_create_reponse_ndarray():
def test_create_rest_reponse_ndarray():
user_model = UserObject()
request = {
"data": {
"ndarray": np.array([[5, 6, 7]]),
"names": []
}
}
raw_response = np.array([[1, 2, 3]])
result = scu.construct_response_json(
user_model,
True,
request,
raw_response)
assert "ndarray" in result.get("data", {})
assert np.array_equal(result["data"]["ndarray"], raw_response)

def test_create_grpc_reponse_ndarray():
user_model = UserObject()
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_grpc_datadef("ndarray", request_data)
Expand All @@ -69,8 +96,29 @@ def test_create_reponse_ndarray():
sm = scu.construct_response(user_model, True, request, raw_response)
assert sm.data.WhichOneof("data_oneof") == "ndarray"


def test_create_reponse_tensor():
def test_create_rest_reponse_tensor():
user_model = UserObject()
tensor = {
"values": [[1,2,3]],
"shape": (1,3)
}
request = {
"data": {
"tensor": tensor,
"names": []
}
}
raw_response = np.array([[1, 2, 3]])
result = scu.construct_response_json(
user_model,
True,
request,
raw_response)
assert "tensor" in result.get("data", {})
assert np.array_equal(
result["data"]["tensor"], tensor)

def test_create_grpc_reponse_tensor():
user_model = UserObject()
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_grpc_datadef("tensor", request_data)
Expand All @@ -79,8 +127,23 @@ def test_create_reponse_tensor():
sm = scu.construct_response(user_model, True, request, raw_response)
assert sm.data.WhichOneof("data_oneof") == "tensor"


def test_create_response_strdata():
def test_create_rest_response_strdata():
user_model = UserObject()
request_data = "Request data"
request = {
"strData": request_data
}
raw_response = "hello world"
sm = scu.construct_response_json(
user_model,
True,
request,
raw_response)
assert "strData" in sm
assert len(sm["strData"]) > 0
assert sm["strData"] == raw_response

def test_create_grpc_response_strdata():
user_model = UserObject()
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_grpc_datadef("ndarray", request_data)
Expand Down Expand Up @@ -122,7 +185,7 @@ def test_symmetric_json_conversion():
result_json_request = scu.seldon_message_to_json(seldon_message_request)
assert json_request == result_json_request

def test_create_reponse_list():
def test_create_grpc_reponse_list():
user_model = UserObject()
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_grpc_datadef("tensor", request_data)
Expand All @@ -131,8 +194,19 @@ def test_create_reponse_list():
sm = scu.construct_response(user_model, True, request, raw_response)
assert sm.data.WhichOneof("data_oneof") == "ndarray"

def test_create_rest_reponse_binary():
user_model = UserObject()
request_data = b"input"
request = {
"binData": request_data
}
raw_response = b"binary"
sm = scu.construct_response_json(user_model, True, request, raw_response)
assert "strData" not in sm
assert "binData" in sm
assert sm["binData"] == raw_response

def test_create_reponse_binary():
def test_create_grpc_reponse_binary():
user_model = UserObject()
request_data = np.array([[5, 6, 7]])
datadef = scu.array_to_grpc_datadef("tensor", request_data)
Expand All @@ -143,7 +217,6 @@ def test_create_reponse_binary():
assert len(sm.strData) == 0
assert len(sm.binData) > 0


def test_json_to_seldon_message_normal_data():
data = {"data": {"tensor": {"shape": [1, 1], "values": [1]}}}
requestProto = scu.json_to_seldon_message(data)
Expand Down