From 7a68369dbea2d78e803d61dcd4c610c580f9d2df Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Thu, 31 Oct 2019 10:46:26 +0000 Subject: [PATCH 1/8] Make google-cloud-storage optional --- python/seldon_core/storage.py | 19 +++++++++++++++++-- python/setup.py | 6 ++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/python/seldon_core/storage.py b/python/seldon_core/storage.py index 3d6470066c..e806f287aa 100644 --- a/python/seldon_core/storage.py +++ b/python/seldon_core/storage.py @@ -18,15 +18,30 @@ import glob import logging +import textwrap import tempfile import os import re from urllib.parse import urlparse from azure.storage.blob import BlockBlobService -from google.auth import exceptions -from google.cloud import storage from minio import Minio +try: + from google.auth import exceptions + from google.cloud import storage +except ImportError: + notice = textwrap.dedent( + """ + Support for Google Cloud Storage is not installed. + If you want to download resources from Google Cloud Storage + install `google-cloud-storage` or install `seldon_core` as + + $ pip install seldon_core[gs] + """ + ) + logging.info(notice) + + _GCS_PREFIX = "gs://" _S3_PREFIX = "s3://" _BLOB_RE = "https://(.+?).blob.core.windows.net/(.+)" diff --git a/python/setup.py b/python/setup.py index 6754336b39..054e5a8b2b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -34,12 +34,14 @@ "pyyaml", "gunicorn >= 19.9.0", "minio >= 4.0.9", - "google-cloud-storage >= 1.16.0", "azure-storage-blob >= 2.0.1", "setuptools >= 41.0.0", ], tests_require=["pytest", "pytest-cov", "Pillow"], - extras_require={"tensorflow": ["tensorflow"]}, + extras_require={ + "tensorflow": ["tensorflow"], + "gcs": ["google-cloud-storage >= 1.16.0"], + }, test_suite="tests", entry_points={ "console_scripts": [ From dbe8168cfadaf105830a9ce50a35b5c2e6305ab3 Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Thu, 31 Oct 2019 11:41:23 +0000 Subject: [PATCH 2/8] Move optional imports to imports_helper.py --- python/seldon_core/imports_helper.py | 44 +++++++++++++++++++ .../tensorflow/core/framework/__init__.py | 19 ++------ python/seldon_core/storage.py | 15 +------ python/seldon_core/tf_helper.py | 9 ---- python/seldon_core/utils.py | 3 +- python/tests/test_model_microservice.py | 2 +- python/tests/test_utils.py | 2 +- python/tests/utils.py | 5 ++- 8 files changed, 56 insertions(+), 43 deletions(-) create mode 100644 python/seldon_core/imports_helper.py delete mode 100644 python/seldon_core/tf_helper.py diff --git a/python/seldon_core/imports_helper.py b/python/seldon_core/imports_helper.py new file mode 100644 index 0000000000..b9b2113df5 --- /dev/null +++ b/python/seldon_core/imports_helper.py @@ -0,0 +1,44 @@ +import logging +import textwrap + +logger = logging.getLogger(__name__) + +# Variables to check if certain extra dependencies are included or +# not +_TF_MISSING = True +_GCS_MISSING = True + +try: + import tensorflow # noqa: F401 + + _TF_MISSING = False +except ImportError: + _TF_MISSING = True + notice = textwrap.dedent( + """ + Tensorflow is not installed. + If you want to use `tftensor` and Tensorflow's data types + install `tensorflow` or install `seldon_core` as + + $ pip install seldon_core[tensorflow] + """ + ) + logger.info(notice) + + +try: + from google.cloud import storage # noqa: F401 + + _GCS_MISSING = False +except ImportError: + _GCS_MISSING = True + notice = textwrap.dedent( + """ + Support for Google Cloud Storage is not installed. + If you want to download resources from Google Cloud Storage + install `google-cloud-storage` or install `seldon_core` as + + $ pip install seldon_core[gcs] + """ + ) + logger.info(notice) diff --git a/python/seldon_core/proto/tensorflow/core/framework/__init__.py b/python/seldon_core/proto/tensorflow/core/framework/__init__.py index 3bb78b6118..1a85a523fc 100644 --- a/python/seldon_core/proto/tensorflow/core/framework/__init__.py +++ b/python/seldon_core/proto/tensorflow/core/framework/__init__.py @@ -1,18 +1,5 @@ -import logging -import textwrap +from seldon_core.imports_helper import _TF_MISSING -logger = logging.getLogger(__name__) - -try: +if not _TF_MISSING: # Let tensorflow shadow these imports if present - from tensorflow.core.framework import ( - tensor_pb2, types_pb2, tensor_shape_pb2) -except ImportError: - notice = textwrap.dedent(""" - Tensorflow is not installed. - If you want to use `tftensor` and Tensorflow's data types - install `tensorflow` or install `seldon_core` as - - $ pip install seldon_core[tensorflow] - """) - logger.info(notice) + from tensorflow.core.framework import tensor_pb2, types_pb2, tensor_shape_pb2 diff --git a/python/seldon_core/storage.py b/python/seldon_core/storage.py index e806f287aa..862ff4f2fc 100644 --- a/python/seldon_core/storage.py +++ b/python/seldon_core/storage.py @@ -18,28 +18,17 @@ import glob import logging -import textwrap import tempfile import os import re from urllib.parse import urlparse from azure.storage.blob import BlockBlobService from minio import Minio +from seldon_core.imports_helper import _GCS_MISSING -try: +if not _GCS_MISSING: from google.auth import exceptions from google.cloud import storage -except ImportError: - notice = textwrap.dedent( - """ - Support for Google Cloud Storage is not installed. - If you want to download resources from Google Cloud Storage - install `google-cloud-storage` or install `seldon_core` as - - $ pip install seldon_core[gs] - """ - ) - logging.info(notice) _GCS_PREFIX = "gs://" diff --git a/python/seldon_core/tf_helper.py b/python/seldon_core/tf_helper.py deleted file mode 100644 index 087ee47de8..0000000000 --- a/python/seldon_core/tf_helper.py +++ /dev/null @@ -1,9 +0,0 @@ -# Variable to check if TF is present or not -_TF_MISSING = True - -try: - import tensorflow # noqa: F401 - - _TF_MISSING = False -except ImportError: - _TF_MISSING = True diff --git a/python/seldon_core/utils.py b/python/seldon_core/utils.py index 51fb4d2ebd..e08fd3beb2 100644 --- a/python/seldon_core/utils.py +++ b/python/seldon_core/utils.py @@ -1,5 +1,4 @@ import json -import logging import sys import base64 import numpy as np @@ -17,7 +16,7 @@ client_feature_names, SeldonComponent, ) -from seldon_core.tf_helper import _TF_MISSING +from seldon_core.imports_helper import _TF_MISSING from typing import Tuple, Dict, Union, List, Optional, Iterable diff --git a/python/tests/test_model_microservice.py b/python/tests/test_model_microservice.py index a1733c5deb..42fef69cf6 100644 --- a/python/tests/test_model_microservice.py +++ b/python/tests/test_model_microservice.py @@ -10,7 +10,7 @@ from seldon_core.user_model import SeldonComponent from seldon_core.utils import seldon_message_to_json, json_to_seldon_message from seldon_core.flask_utils import SeldonMicroserviceException -from seldon_core.tf_helper import _TF_MISSING +from seldon_core.imports_helper import _TF_MISSING from utils import skipif_tf_missing diff --git a/python/tests/test_utils.py b/python/tests/test_utils.py index 5e7053a0f5..9e29b03290 100644 --- a/python/tests/test_utils.py +++ b/python/tests/test_utils.py @@ -6,7 +6,7 @@ import seldon_core.utils as scu from seldon_core.proto import prediction_pb2 from seldon_core.flask_utils import SeldonMicroserviceException -from seldon_core.tf_helper import _TF_MISSING +from seldon_core.imports_helper import _TF_MISSING from google.protobuf.struct_pb2 import Value from utils import skipif_tf_missing diff --git a/python/tests/utils.py b/python/tests/utils.py index 535ab65f59..55947228a2 100644 --- a/python/tests/utils.py +++ b/python/tests/utils.py @@ -1,4 +1,7 @@ import pytest -from seldon_core.tf_helper import _TF_MISSING +from seldon_core.imports_helper import _TF_MISSING, _GCS_MISSING skipif_tf_missing = pytest.mark.skipif(_TF_MISSING, reason="tensorflow is not present") +skipif_gcs_missing = pytest.mark.skipif( + _GCS_MISSING, reason="google-cloud-storage is not present" +) From ef27d32639053797d8754b1c56535d3037be043d Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Thu, 31 Oct 2019 13:26:48 +0000 Subject: [PATCH 3/8] Copy tests from kfserving --- python/tests/test_s3_storage.py | 116 ++++++++++++++++++++++++++++++++ python/tests/test_storage.py | 90 +++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 python/tests/test_s3_storage.py create mode 100644 python/tests/test_storage.py diff --git a/python/tests/test_s3_storage.py b/python/tests/test_s3_storage.py new file mode 100644 index 0000000000..6900fe78c8 --- /dev/null +++ b/python/tests/test_s3_storage.py @@ -0,0 +1,116 @@ +# Copyright 2019 kubeflow.org. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Copied from kfserving project as starter. +# + +import unittest.mock as mock +import seldon_core + + +def create_mock_obj(path): + mock_obj = mock.MagicMock() + mock_obj.object_name = path + mock_obj.is_dir = False + return mock_obj + + +def create_mock_minio_client(mock_storage, paths): + mock_minio_client = mock_storage.return_value + mock_minio_client.list_objects.return_value = [create_mock_obj(p) for p in paths] + return mock_minio_client + + +def get_call_args(call_args_list): + arg_list = [] + for call in call_args_list: + args, _ = call + arg_list.append(args) + return arg_list + + +def expected_call_args_list(bucket_name, parent_key, dest, paths): + return [ + (bucket_name, f"{parent_key}/{p}".strip("/"), f"{dest}/{p}".strip("/")) + for p in paths + ] + + +# pylint: disable=protected-access + + +@mock.patch("seldon_core.storage.Minio") +def test_parent_key(mock_storage): + + # given + bucket_name = "foo" + paths = ["models/weights.pt", "0002.h5", "a/very/long/path/config.json"] + object_paths = ["bar/" + p for p in paths] + + # when + mock_minio_client = create_mock_minio_client(mock_storage, object_paths) + seldon_core.Storage._download_s3(f"s3://{bucket_name}/bar", "dest_path") + + # then + arg_list = get_call_args(mock_minio_client.fget_object.call_args_list) + assert arg_list == expected_call_args_list(bucket_name, "bar", "dest_path", paths) + + mock_minio_client.list_objects.assert_called_with( + bucket_name, prefix="bar", recursive=True + ) + + +@mock.patch("seldon_core.storage.Minio") +def test_no_key(mock_storage): + + # given + bucket_name = "foo" + object_paths = ["models/weights.pt", "0002.h5", "a/very/long/path/config.json"] + + # when + mock_minio_client = create_mock_minio_client(mock_storage, object_paths) + seldon_core.Storage._download_s3(f"s3://{bucket_name}/", "dest_path") + + # then + arg_list = get_call_args(mock_minio_client.fget_object.call_args_list) + assert arg_list == expected_call_args_list( + bucket_name, "", "dest_path", object_paths + ) + + mock_minio_client.list_objects.assert_called_with( + bucket_name, prefix="", recursive=True + ) + + +@mock.patch("seldon_core.storage.Minio") +def test_full_name_key(mock_storage): + + # given + bucket_name = "foo" + object_key = "path/to/model/name.pt" + + # when + mock_minio_client = create_mock_minio_client(mock_storage, [object_key]) + seldon_core.Storage._download_s3(f"s3://{bucket_name}/{object_key}", "dest_path") + + # then + arg_list = get_call_args(mock_minio_client.fget_object.call_args_list) + assert arg_list == expected_call_args_list( + bucket_name, "", "dest_path", [object_key] + ) + + mock_minio_client.list_objects.assert_called_with( + bucket_name, prefix=object_key, recursive=True + ) diff --git a/python/tests/test_storage.py b/python/tests/test_storage.py new file mode 100644 index 0000000000..bc86de9ff4 --- /dev/null +++ b/python/tests/test_storage.py @@ -0,0 +1,90 @@ +# Copyright 2019 kubeflow.org. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Copied from kfserving project as starter. +# + +import pytest +import seldon_core +from minio import Minio, error +import unittest.mock as mock +from utils import skipif_gcs_missing +from seldon_core.imports_helper import _GCS_MISSING + +if not _GCS_MISSING: + from google.cloud import exceptions + +STORAGE_MODULE = "seldon_core.storage" + + +def test_storage_local_path(): + abs_path = "file:///" + relative_path = "file://." + assert seldon_core.Storage.download(abs_path) == abs_path.replace("file://", "", 1) + assert seldon_core.Storage.download(relative_path) == relative_path.replace( + "file://", "", 1 + ) + + +def test_storage_local_path_exception(): + not_exist_path = "file:///some/random/path" + with pytest.raises(Exception): + seldon_core.Storage.download(not_exist_path) + + +def test_no_prefix_local_path(): + abs_path = "/" + relative_path = "." + assert seldon_core.Storage.download(abs_path) == abs_path + assert seldon_core.Storage.download(relative_path) == relative_path + + +@skipif_gcs_missing +@mock.patch(STORAGE_MODULE + ".storage") +def test_mock_gcs(mock_storage): + gcs_path = "gs://foo/bar" + mock_obj = mock.MagicMock() + mock_obj.name = "mock.object" + mock_storage.Client().bucket().list_blobs().__iter__.return_value = [mock_obj] + assert seldon_core.Storage.download(gcs_path) + + +def test_storage_blob_exception(): + blob_path = "https://accountname.blob.core.windows.net/container/some/blob/" + with pytest.raises(Exception): + seldon_core.Storage.download(blob_path) + + +@mock.patch("urllib3.PoolManager") +@mock.patch(STORAGE_MODULE + ".Minio") +def test_no_permission_buckets(mock_connection, mock_minio): + bad_s3_path = "s3://random/path" + # Access private buckets without credentials + mock_minio.return_value = Minio( + "s3.us.cloud-object-storage.appdomain.cloud", secure=True + ) + mock_connection.side_effect = error.AccessDenied(None) + with pytest.raises(error.AccessDenied): + seldon_core.Storage.download(bad_s3_path) + + +@skipif_gcs_missing +@mock.patch("urllib3.PoolManager") +def test_no_permission_buckets_gcs(mock_connection): + bad_gcs_path = "gs://random/path" + + mock_connection.side_effect = exceptions.Forbidden(None) + with pytest.raises(exceptions.Forbidden): + seldon_core.Storage.download(bad_gcs_path) From afa3f695e24366a460c5348709db966e30410c6e Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Thu, 31 Oct 2019 13:38:24 +0000 Subject: [PATCH 4/8] Change _X_MISSING to _X_PRESENT --- python/seldon_core/imports_helper.py | 12 ++++++------ .../proto/tensorflow/core/framework/__init__.py | 4 ++-- python/seldon_core/storage.py | 4 ++-- python/seldon_core/utils.py | 4 ++-- python/tests/test_model_microservice.py | 4 ++-- python/tests/test_storage.py | 4 ++-- python/tests/test_utils.py | 4 ++-- python/tests/utils.py | 8 +++++--- 8 files changed, 23 insertions(+), 21 deletions(-) diff --git a/python/seldon_core/imports_helper.py b/python/seldon_core/imports_helper.py index b9b2113df5..c5bcef310c 100644 --- a/python/seldon_core/imports_helper.py +++ b/python/seldon_core/imports_helper.py @@ -5,15 +5,15 @@ # Variables to check if certain extra dependencies are included or # not -_TF_MISSING = True -_GCS_MISSING = True +_TF_PRESENT = False +_GCS_PRESENT = False try: import tensorflow # noqa: F401 - _TF_MISSING = False + _TF_PRESENT = True except ImportError: - _TF_MISSING = True + _TF_PRESENT = False notice = textwrap.dedent( """ Tensorflow is not installed. @@ -29,9 +29,9 @@ try: from google.cloud import storage # noqa: F401 - _GCS_MISSING = False + _GCS_PRESENT = True except ImportError: - _GCS_MISSING = True + _GCS_PRESENT = False notice = textwrap.dedent( """ Support for Google Cloud Storage is not installed. diff --git a/python/seldon_core/proto/tensorflow/core/framework/__init__.py b/python/seldon_core/proto/tensorflow/core/framework/__init__.py index 1a85a523fc..76d1d80037 100644 --- a/python/seldon_core/proto/tensorflow/core/framework/__init__.py +++ b/python/seldon_core/proto/tensorflow/core/framework/__init__.py @@ -1,5 +1,5 @@ -from seldon_core.imports_helper import _TF_MISSING +from seldon_core.imports_helper import _TF_PRESENT -if not _TF_MISSING: +if _TF_PRESENT: # Let tensorflow shadow these imports if present from tensorflow.core.framework import tensor_pb2, types_pb2, tensor_shape_pb2 diff --git a/python/seldon_core/storage.py b/python/seldon_core/storage.py index 862ff4f2fc..e3b22afadd 100644 --- a/python/seldon_core/storage.py +++ b/python/seldon_core/storage.py @@ -24,9 +24,9 @@ from urllib.parse import urlparse from azure.storage.blob import BlockBlobService from minio import Minio -from seldon_core.imports_helper import _GCS_MISSING +from seldon_core.imports_helper import _GCS_PRESENT -if not _GCS_MISSING: +if _GCS_PRESENT: from google.auth import exceptions from google.cloud import storage diff --git a/python/seldon_core/utils.py b/python/seldon_core/utils.py index e08fd3beb2..f51fcdb674 100644 --- a/python/seldon_core/utils.py +++ b/python/seldon_core/utils.py @@ -16,11 +16,11 @@ client_feature_names, SeldonComponent, ) -from seldon_core.imports_helper import _TF_MISSING +from seldon_core.imports_helper import _TF_PRESENT from typing import Tuple, Dict, Union, List, Optional, Iterable -if not _TF_MISSING: +if _TF_PRESENT: import tensorflow as tf from tensorflow.core.framework.tensor_pb2 import TensorProto diff --git a/python/tests/test_model_microservice.py b/python/tests/test_model_microservice.py index 42fef69cf6..33c9e990d4 100644 --- a/python/tests/test_model_microservice.py +++ b/python/tests/test_model_microservice.py @@ -10,11 +10,11 @@ from seldon_core.user_model import SeldonComponent from seldon_core.utils import seldon_message_to_json, json_to_seldon_message from seldon_core.flask_utils import SeldonMicroserviceException -from seldon_core.imports_helper import _TF_MISSING +from seldon_core.imports_helper import _TF_PRESENT from utils import skipif_tf_missing -if not _TF_MISSING: +if _TF_PRESENT: from tensorflow.core.framework.tensor_pb2 import TensorProto import tensorflow as tf diff --git a/python/tests/test_storage.py b/python/tests/test_storage.py index bc86de9ff4..b83a42e114 100644 --- a/python/tests/test_storage.py +++ b/python/tests/test_storage.py @@ -21,9 +21,9 @@ from minio import Minio, error import unittest.mock as mock from utils import skipif_gcs_missing -from seldon_core.imports_helper import _GCS_MISSING +from seldon_core.imports_helper import _GCS_PRESENT -if not _GCS_MISSING: +if _GCS_PRESENT: from google.cloud import exceptions STORAGE_MODULE = "seldon_core.storage" diff --git a/python/tests/test_utils.py b/python/tests/test_utils.py index 9e29b03290..f277bb4710 100644 --- a/python/tests/test_utils.py +++ b/python/tests/test_utils.py @@ -6,11 +6,11 @@ import seldon_core.utils as scu from seldon_core.proto import prediction_pb2 from seldon_core.flask_utils import SeldonMicroserviceException -from seldon_core.imports_helper import _TF_MISSING +from seldon_core.imports_helper import _TF_PRESENT from google.protobuf.struct_pb2 import Value from utils import skipif_tf_missing -if not _TF_MISSING: +if _TF_PRESENT: import tensorflow as tf diff --git a/python/tests/utils.py b/python/tests/utils.py index 55947228a2..68cf9fbd6c 100644 --- a/python/tests/utils.py +++ b/python/tests/utils.py @@ -1,7 +1,9 @@ import pytest -from seldon_core.imports_helper import _TF_MISSING, _GCS_MISSING +from seldon_core.imports_helper import _TF_PRESENT, _GCS_PRESENT -skipif_tf_missing = pytest.mark.skipif(_TF_MISSING, reason="tensorflow is not present") +skipif_tf_missing = pytest.mark.skipif( + not _TF_PRESENT, reason="tensorflow is not present" +) skipif_gcs_missing = pytest.mark.skipif( - _GCS_MISSING, reason="google-cloud-storage is not present" + not _GCS_PRESENT, reason="google-cloud-storage is not present" ) From 1a5e5f8c95ac300c41ef15f2f0626386954e8d36 Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Thu, 31 Oct 2019 13:57:23 +0000 Subject: [PATCH 5/8] Add extra key 'all' to install all extra dependencies --- python/seldon_core/imports_helper.py | 8 ++++++++ python/setup.py | 12 +++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/python/seldon_core/imports_helper.py b/python/seldon_core/imports_helper.py index c5bcef310c..cb49a1cbd9 100644 --- a/python/seldon_core/imports_helper.py +++ b/python/seldon_core/imports_helper.py @@ -21,6 +21,10 @@ install `tensorflow` or install `seldon_core` as $ pip install seldon_core[tensorflow] + + or + + $ pip install seldon_core[all] """ ) logger.info(notice) @@ -39,6 +43,10 @@ install `google-cloud-storage` or install `seldon_core` as $ pip install seldon_core[gcs] + + or + + $ pip install seldon_core[all] """ ) logger.info(notice) diff --git a/python/setup.py b/python/setup.py index 054e5a8b2b..4b7a6b6598 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,11 +1,16 @@ -from setuptools import find_packages, setup +import itertools import os +from setuptools import find_packages, setup version = {} dir_path = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(dir_path, "seldon_core/version.py")) as fp: exec(fp.read(), version) +# Extra dependencies, with special 'all' key +extras = {"tensorflow": ["tensorflow"], "gcs": ["google-cloud-storage >= 1.16.0"]} +extras["all"] = list(set(itertools.chain.from_iterable(extras.values()))) + setup( name="seldon-core", author="Seldon Technologies Ltd.", @@ -38,10 +43,7 @@ "setuptools >= 41.0.0", ], tests_require=["pytest", "pytest-cov", "Pillow"], - extras_require={ - "tensorflow": ["tensorflow"], - "gcs": ["google-cloud-storage >= 1.16.0"], - }, + extras_require=extras, test_suite="tests", entry_points={ "console_scripts": [ From 8ce15d0b09079f61ee409653f19367d814adcc63 Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Thu, 31 Oct 2019 14:02:35 +0000 Subject: [PATCH 6/8] Add CI tests for [gcs] and [all] --- jenkins-x.yml | 14 ++++++++++++++ python/Makefile | 8 ++++++++ python/setup.py | 4 ++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/jenkins-x.yml b/jenkins-x.yml index d0bcd1a25b..fcad8f4594 100644 --- a/jenkins-x.yml +++ b/jenkins-x.yml @@ -26,6 +26,20 @@ pipelineConfig: - test command: make name: test-python-tf + - args: + - -C python + - update_package + - install-gcs + - test + command: make + name: test-python-gcs + - args: + - -C python + - update_package + - install-all + - test + command: make + name: test-python-all - agent: image: seldonio/core-builder:0.4 name: seldon-engine diff --git a/python/Makefile b/python/Makefile index c0c2adfd56..c22c65d368 100644 --- a/python/Makefile +++ b/python/Makefile @@ -53,6 +53,14 @@ install: install-tf: pip install -e .[tensorflow] +.PHONY: install-gcs +install-gcs: + pip install -e .[gcs] + +.PHONY: install-all +install-all: + pip install -e .[all] + .PHONY: install-dev install-dev: pip install -e . -r requirements.txt diff --git a/python/setup.py b/python/setup.py index 4b7a6b6598..92a3ac9e45 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,5 +1,5 @@ -import itertools import os +from itertools.chain import from_iterable from setuptools import find_packages, setup version = {} @@ -9,7 +9,7 @@ # Extra dependencies, with special 'all' key extras = {"tensorflow": ["tensorflow"], "gcs": ["google-cloud-storage >= 1.16.0"]} -extras["all"] = list(set(itertools.chain.from_iterable(extras.values()))) +extras["all"] = list(set(from_iterable(extras.values()))) setup( name="seldon-core", From a3b7f90ce0aaa0adba5df1954c9b8545ea8a4498 Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Thu, 31 Oct 2019 14:48:46 +0000 Subject: [PATCH 7/8] Fix itertools import --- python/setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 92a3ac9e45..19822a5322 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,5 +1,5 @@ import os -from itertools.chain import from_iterable +from itertools import chain from setuptools import find_packages, setup version = {} @@ -9,7 +9,8 @@ # Extra dependencies, with special 'all' key extras = {"tensorflow": ["tensorflow"], "gcs": ["google-cloud-storage >= 1.16.0"]} -extras["all"] = list(set(from_iterable(extras.values()))) +all_extra_deps = chain.from_iterable(extras.values()) +extras["all"] = list(set(all_extra_deps)) setup( name="seldon-core", From fb8612c59b0dff0b32a0d849abda417b3bd89345 Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Fri, 1 Nov 2019 10:21:06 +0000 Subject: [PATCH 8/8] Pin a couple dependencies --- python/setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index 19822a5322..b54c53a1d9 100644 --- a/python/setup.py +++ b/python/setup.py @@ -36,11 +36,11 @@ "Flask-OpenTracing >= 1.1.0, < 1.2.0", "opentracing >= 2.2.0, < 2.3.0", "jaeger-client >= 4.1.0, < 4.2.0", - "grpcio-opentracing", + "grpcio-opentracing >= 1.1.4, < 1.2.0", "pyyaml", - "gunicorn >= 19.9.0", - "minio >= 4.0.9", - "azure-storage-blob >= 2.0.1", + "gunicorn >= 19.9.0, < 19.10.0", + "minio >= 4.0.9, < 6.0.0", + "azure-storage-blob >= 2.0.1, < 3.0.0", "setuptools >= 41.0.0", ], tests_require=["pytest", "pytest-cov", "Pillow"],