Merge pull request #942 from adriangonz/extra-python-dependencies

Extra python dependencies

.gitignore

@@ -36,28 +36,6 @@ api-frontend/src/main/proto/seldon_deployment.proto
 engine/src/main/proto/k8s.io/
 engine/src/main/proto/seldon_deployment.proto
 engine/src/main/proto/v1.proto
-api-frontend/settings.sh
-api-frontend/apife.json
-api-frontend/apife_istio.json
-api-frontend/src/main/proto/prediction.proto
-api-frontend/tests/API_ENDPOINT
-api-frontend/tests/api-client-key.txt
-api-frontend/tests/api-client-secret.txt
-api-frontend/tests/python/prediction_pb2.py
-api-frontend/tests/python/seldonengine_pb2.py
-api-frontend/src/main/proto/k8s.io/
-api-frontend/src/main/proto/v1.proto
-api-frontend/src/main/proto/seldonengine.proto
-cluster-manager/cluster-manager-client-secret.txt
-cluster-manager/cluster-manager.json
-api-frontend/target/
-cluster-manager/src/main/proto/seldon_deployment.proto
-cluster-manager/target/
-cluster-manager/tests/CLUSTER_MANAGER_ENDPOINT
-cluster-manager/tests/out.log
-cluster-manager/src/main/proto/k8s.io
-cluster-manager/src/main/proto/v1.proto
-cluster-manager/java_client/
 engine/src/main/proto/prediction.proto
 engine/src/main/proto/seldonengine.proto
 redis-memonly/redis-memonly.json
@@ -67,22 +45,21 @@ tests/out.log
 cluster-manager/settings.sh
 engine/target/
 zookeeper-k8s/zookeeper.json
-kafka/tests/src/prediction_pb2.py
-kafka/tests/src/seldonengine_pb2.py
 monitoring/alertmanager/config.yml
 monitoring/grafana/auth/grafana-prom-admin-password.txt
 monitoring/glusterfs-volume/_glusterfs.json
 wrappers/**/*.proto
-pyseldon/**/*.proto
 **/*/.ipynb_checkpoints
 proto/k8s/k8s.io/
 proto/k8s/v1.proto
-wrappers/python/proto/prediction_pb2.py
-wrappers/python/proto/prediction_pb2_grpc.py
 
 notebooks/**/*.proto
 *pb2.py
 *pb2_grpc.py
+!python/seldon_core/proto/*_pb2.py
+!python/seldon_core/proto/*_pb2_grpc.py
+!python/seldon_core/proto/**/*_pb2.py
+!python/seldon_core/proto/**/*_pb2_grpc.py
 examples/models/keras_mnist/build/
 examples/models/mean_classifier/build/
 notebooks/my-ml-deployment/

jenkins-x.yml

@@ -15,16 +15,16 @@ pipelineConfig:
                 - name: test-python
                   command: make
                   args:
-                    - -C
-                    - python
-                    - update_package &&
-                    - make
-                    - -C
-                    - python
-                    - install-dev &&
-                    - make
-                    - -C
-                    - python
+                    - -C python
+                    - update_package
+                    - install
+                    - test
+                - name: test-python-tf
+                  command: make
+                  args:
+                    - -C python
+                    - update_package
+                    - install-tf
                     - test
               - name: seldon-engine
                 agent:
@@ -33,8 +33,6 @@ pipelineConfig:
                 - name: test-engine
                   command: make
                   args:
-                  - -C
-                  - engine
-                  - -f
-                  - Makefile.ci
+                  - -C engine
+                  - -f Makefile.ci
                   - build_jar

python/Makefile

@@ -6,7 +6,8 @@ get_apis:
 	# Protobuf
 	cp ${SELDON_CORE_DIR}/proto/prediction.proto seldon_core/proto/
 	$(MAKE) -C ../proto/tensorflow/ create_protos
-	cp -r $(SELDON_CORE_DIR)/proto/tensorflow/tensorflow seldon_core/
+	cp -r $(SELDON_CORE_DIR)/proto/tensorflow/tensorflow \
+		seldon_core/proto/
 	$(MAKE) -C ../proto/tensorflow clean
 
 	# OpenAPI
@@ -15,7 +16,26 @@ get_apis:
 .PHONY: build_apis
 build_apis: get_apis
 	# Protobuf
-	cd seldon_core && python -m grpc.tools.protoc -I./ --python_out=./ --grpc_python_out=./ --mypy_out=./ ./proto/prediction.proto
+	cd seldon_core && python \
+		-m grpc.tools.protoc \
+		-I./ \
+		-I./proto/ \
+		--python_out=./ \
+		--grpc_python_out=./ \
+		--mypy_out=./ \
+		./proto/prediction.proto
+	## We build TF's protobufs as well in case the tensorflow package
+	## is not found
+	cd seldon_core/proto && python \
+		-m grpc.tools.protoc \
+		-I./ \
+		-I../ \
+		--python_out=./ \
+		./tensorflow/core/framework/*.proto
+	sed -i "s/from tensorflow/from seldon_core.proto.tensorflow/" seldon_core/proto/*.py
+	sed -i "s/from tensorflow.core.framework/from ./" \
+		$(addprefix seldon_core/proto/tensorflow/core/framework/, \
+			resource_handle_pb2.py tensor_pb2.py tensor_shape_pb2.py types_pb2.py)
 	sed -i "s/from proto/from seldon_core.proto/g" seldon_core/proto/prediction_pb2_grpc.py
 
 .PHONY: update_version
@@ -29,6 +49,10 @@ update_package: get_apis build_apis update_version
 install:
 	pip install -e .
 
+.PHONY: install-tf
+install-tf:
+	pip install -e .[tensorflow]
+
 .PHONY: install-dev
 install-dev:
 	pip install -e . -r requirements.txt