Make GCS optional

jenkins-x.yml

@@ -26,6 +26,20 @@ pipelineConfig:
               - test
               command: make
               name: test-python-tf
+            - args:
+              - -C python
+              - update_package
+              - install-gcs
+              - test
+              command: make
+              name: test-python-gcs
+            - args:
+              - -C python
+              - update_package
+              - install-all
+              - test
+              command: make
+              name: test-python-all
           - agent:
               image: seldonio/core-builder:0.4
             name: seldon-engine

python/Makefile

@@ -53,6 +53,14 @@ install:
 install-tf:
 	pip install -e .[tensorflow]
 
+.PHONY: install-gcs
+install-gcs:
+	pip install -e .[gcs]
+
+.PHONY: install-all
+install-all:
+	pip install -e .[all]
+
 .PHONY: install-dev
 install-dev:
 	pip install -e . -r requirements.txt

python/seldon_core/imports_helper.py

@@ -0,0 +1,52 @@
+import logging
+import textwrap
+
+logger = logging.getLogger(__name__)
+
+# Variables to check if certain extra dependencies are included or
+# not
+_TF_PRESENT = False
+_GCS_PRESENT = False
+
+try:
+    import tensorflow  # noqa: F401
+
+    _TF_PRESENT = True
+except ImportError:
+    _TF_PRESENT = False
+    notice = textwrap.dedent(
+        """
+        Tensorflow is not installed.
+        If you want to use `tftensor` and Tensorflow's data types
+        install `tensorflow` or install `seldon_core` as
+
+            $ pip install seldon_core[tensorflow]
+
+        or
+
+            $ pip install seldon_core[all]
+    """
+    )
+    logger.info(notice)
+
+
+try:
+    from google.cloud import storage  # noqa: F401
+
+    _GCS_PRESENT = True
+except ImportError:
+    _GCS_PRESENT = False
+    notice = textwrap.dedent(
+        """
+        Support for Google Cloud Storage is not installed.
+        If you want to download resources from Google Cloud Storage
+        install `google-cloud-storage` or install `seldon_core` as
+
+            $ pip install seldon_core[gcs]
+
+        or
+
+            $ pip install seldon_core[all]
+    """
+    )
+    logger.info(notice)

python/seldon_core/proto/tensorflow/core/framework/__init__.py

@@ -1,18 +1,5 @@
-import logging
-import textwrap
+from seldon_core.imports_helper import _TF_PRESENT
 
-logger = logging.getLogger(__name__)
-
-try:
+if _TF_PRESENT:
     # Let tensorflow shadow these imports if present
-    from tensorflow.core.framework import (
-        tensor_pb2, types_pb2, tensor_shape_pb2)
-except ImportError:
-    notice = textwrap.dedent("""
-        Tensorflow is not installed.
-        If you want to use `tftensor` and Tensorflow's data types
-        install `tensorflow` or install `seldon_core` as
-
-            $ pip install seldon_core[tensorflow]
-    """)
-    logger.info(notice)
+    from tensorflow.core.framework import tensor_pb2, types_pb2, tensor_shape_pb2

python/seldon_core/storage.py

@@ -23,9 +23,13 @@
 import re
 from urllib.parse import urlparse
 from azure.storage.blob import BlockBlobService
-from google.auth import exceptions
-from google.cloud import storage
 from minio import Minio
+from seldon_core.imports_helper import _GCS_PRESENT
+
+if _GCS_PRESENT:
+    from google.auth import exceptions
+    from google.cloud import storage
+
 
 _GCS_PREFIX = "gs://"
 _S3_PREFIX = "s3://"

python/seldon_core/utils.py

@@ -1,5 +1,4 @@
 import json
-import logging
 import sys
 import base64
 import numpy as np
@@ -17,11 +16,11 @@
     client_feature_names,
     SeldonComponent,
 )
-from seldon_core.tf_helper import _TF_MISSING
+from seldon_core.imports_helper import _TF_PRESENT
 
 from typing import Tuple, Dict, Union, List, Optional, Iterable
 
-if not _TF_MISSING:
+if _TF_PRESENT:
     import tensorflow as tf
     from tensorflow.core.framework.tensor_pb2 import TensorProto
 

python/setup.py

@@ -1,11 +1,17 @@
-from setuptools import find_packages, setup
 import os
+from itertools import chain
+from setuptools import find_packages, setup
 
 version = {}
 dir_path = os.path.dirname(os.path.realpath(__file__))
 with open(os.path.join(dir_path, "seldon_core/version.py")) as fp:
     exec(fp.read(), version)
 
+# Extra dependencies, with special 'all' key
+extras = {"tensorflow": ["tensorflow"], "gcs": ["google-cloud-storage >= 1.16.0"]}
+all_extra_deps = chain.from_iterable(extras.values())
+extras["all"] = list(set(all_extra_deps))
+
 setup(
     name="seldon-core",
     author="Seldon Technologies Ltd.",
@@ -30,16 +36,15 @@
         "Flask-OpenTracing >= 1.1.0, < 1.2.0",
         "opentracing >= 2.2.0, < 2.3.0",
         "jaeger-client >= 4.1.0, < 4.2.0",
-        "grpcio-opentracing",
+        "grpcio-opentracing >= 1.1.4, < 1.2.0",
         "pyyaml",
-        "gunicorn >= 19.9.0",
-        "minio >= 4.0.9",
-        "google-cloud-storage >= 1.16.0",
-        "azure-storage-blob >= 2.0.1",
+        "gunicorn >= 19.9.0, < 19.10.0",
+        "minio >= 4.0.9, < 6.0.0",
+        "azure-storage-blob >= 2.0.1, < 3.0.0",
         "setuptools >= 41.0.0",
     ],
     tests_require=["pytest", "pytest-cov", "Pillow"],
-    extras_require={"tensorflow": ["tensorflow"]},
+    extras_require=extras,
     test_suite="tests",
     entry_points={
         "console_scripts": [

python/tests/test_model_microservice.py

@@ -10,11 +10,11 @@
 from seldon_core.user_model import SeldonComponent
 from seldon_core.utils import seldon_message_to_json, json_to_seldon_message
 from seldon_core.flask_utils import SeldonMicroserviceException
-from seldon_core.tf_helper import _TF_MISSING
+from seldon_core.imports_helper import _TF_PRESENT
 
 from utils import skipif_tf_missing
 
-if not _TF_MISSING:
+if _TF_PRESENT:
     from tensorflow.core.framework.tensor_pb2 import TensorProto
     import tensorflow as tf
 

python/tests/test_s3_storage.py

@@ -0,0 +1,116 @@
+# Copyright 2019 kubeflow.org.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Copied from kfserving project as starter.
+#
+
+import unittest.mock as mock
+import seldon_core
+
+
+def create_mock_obj(path):
+    mock_obj = mock.MagicMock()
+    mock_obj.object_name = path
+    mock_obj.is_dir = False
+    return mock_obj
+
+
+def create_mock_minio_client(mock_storage, paths):
+    mock_minio_client = mock_storage.return_value
+    mock_minio_client.list_objects.return_value = [create_mock_obj(p) for p in paths]
+    return mock_minio_client
+
+
+def get_call_args(call_args_list):
+    arg_list = []
+    for call in call_args_list:
+        args, _ = call
+        arg_list.append(args)
+    return arg_list
+
+
+def expected_call_args_list(bucket_name, parent_key, dest, paths):
+    return [
+        (bucket_name, f"{parent_key}/{p}".strip("/"), f"{dest}/{p}".strip("/"))
+        for p in paths
+    ]
+
+
+# pylint: disable=protected-access
+
+
+@mock.patch("seldon_core.storage.Minio")
+def test_parent_key(mock_storage):
+
+    # given
+    bucket_name = "foo"
+    paths = ["models/weights.pt", "0002.h5", "a/very/long/path/config.json"]
+    object_paths = ["bar/" + p for p in paths]
+
+    # when
+    mock_minio_client = create_mock_minio_client(mock_storage, object_paths)
+    seldon_core.Storage._download_s3(f"s3://{bucket_name}/bar", "dest_path")
+
+    # then
+    arg_list = get_call_args(mock_minio_client.fget_object.call_args_list)
+    assert arg_list == expected_call_args_list(bucket_name, "bar", "dest_path", paths)
+
+    mock_minio_client.list_objects.assert_called_with(
+        bucket_name, prefix="bar", recursive=True
+    )
+
+
+@mock.patch("seldon_core.storage.Minio")
+def test_no_key(mock_storage):
+
+    # given
+    bucket_name = "foo"
+    object_paths = ["models/weights.pt", "0002.h5", "a/very/long/path/config.json"]
+
+    # when
+    mock_minio_client = create_mock_minio_client(mock_storage, object_paths)
+    seldon_core.Storage._download_s3(f"s3://{bucket_name}/", "dest_path")
+
+    # then
+    arg_list = get_call_args(mock_minio_client.fget_object.call_args_list)
+    assert arg_list == expected_call_args_list(
+        bucket_name, "", "dest_path", object_paths
+    )
+
+    mock_minio_client.list_objects.assert_called_with(
+        bucket_name, prefix="", recursive=True
+    )
+
+
+@mock.patch("seldon_core.storage.Minio")
+def test_full_name_key(mock_storage):
+
+    # given
+    bucket_name = "foo"
+    object_key = "path/to/model/name.pt"
+
+    # when
+    mock_minio_client = create_mock_minio_client(mock_storage, [object_key])
+    seldon_core.Storage._download_s3(f"s3://{bucket_name}/{object_key}", "dest_path")
+
+    # then
+    arg_list = get_call_args(mock_minio_client.fget_object.call_args_list)
+    assert arg_list == expected_call_args_list(
+        bucket_name, "", "dest_path", [object_key]
+    )
+
+    mock_minio_client.list_objects.assert_called_with(
+        bucket_name, prefix=object_key, recursive=True
+    )