feat: sklearnserver integration (#11)

* feat: sklearnserver rock integration

Summary of changes:
- Added ROCK integrity tests.
- Updated ROCK image definition.
- Added tox with unit and integration tests.
- Addressed review comments.
- Modified to run tests.
- Added handling of jinja2 templating
- Updated tox.ini to properly update configmap template.
- Updated rockcraft.yaml with new run-user option to run as non-root. canonical/seldon-core-operator#133
- Updated import procedure.
- Tested with integration tests on the branch.
- Reverted to use of shell commands and `yq` instead of CheckRock test
  class from chisme package.
- Remove chisme package

NOTE: Use of bash shell commands significantly reduces maintability of
tox.ini ACK: @kimwnasptd

sklearnserver/rockcraft.yaml

@@ -1,25 +1,27 @@
+# Based on https://github.com/SeldonIO/seldon-core/tree/master/servers/sklearnserver/sklearnserver
 name: sklearnserver
 summary: An image for Seldon SKLearn Server
 description: |
   This image is used as part of the Charmed Kubeflow product. The SKLearn Server serves
   models which have been stored as pickles.
-version: v1.16.0_1 # version format: <KF-upstream-version>_<Charmed-KF-version>
+version: v1.16.0_20.04_1 # <upstream-version>_<base-version>_<Charmed-KF-version>
 license: Apache-2.0
 base: ubuntu:20.04
+run-user: _daemon_
 services:
   sklearnserver:
     override: replace
     summary: "sklearnserver service"
     startup: enabled
     # Yet again, use a subshell to jam conda into a working state. Can't use bashrc, because it immediately
-    # exits if PS1 isn't set, so no-go from scripts
-    command: bash -c 'cd /microservice && export PATH=/opt/conda/bin/${PATH} && eval $(/opt/conda/bin/conda shell.bash hook 2> /dev/null) && source /opt/conda/etc/profile.d/conda.sh && conda activate && seldon-core-microservice $MODEL_NAME --service-type $SERVICE_TYPE --persistence $PERSISTENCE'
+    # exits if PS1 isn't set, so no-go from scripts.
+    command: bash -c 'cd /microservice && export PATH=/opt/conda/bin/:${PATH} && eval $(/opt/conda/bin/conda shell.bash hook 2> /dev/null) && source /opt/conda/etc/profile.d/conda.sh && conda activate && seldon-core-microservice ${MODEL_NAME} --service-type ${SERVICE_TYPE} &> /tmp/log.txt'
     environment:
+      # the following environment variables are taken from:
+      # https://github.com/SeldonIO/seldon-core/blob/master/servers/sklearnserver/environment
+      # NOTE: PERSISTENCE is omitted because it is depricated
       MODEL_NAME: "SKLearnServer"
       SERVICE_TYPE: "MODEL"
-      PERSISTENCE: "0"
-    user: ubuntu
-
 platforms:
   amd64:
 
@@ -78,15 +80,13 @@ parts:
       # but it does need to match pebble's workdir
       install -D -m 755 ${CRAFT_STAGE}/microservice/SKLearnServer.py microservice/SKLearnServer.py
 
+  security-team-requirement:
+    plugin: nil
+    after: [sklearnserver]
+    override-build: |
       # security requirement
+      # there are no packages installed in `bare` base which is used in this rock
+      # `--root` option is not available in dpkg-query version which is packaged with 20.04
       mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks
-      (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && \
-      dpkg-query -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) \
-      > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query
-
-  non-root-user:
-    plugin: nil
-    overlay-script: |
-      # Create a user in the $CRAFT_OVERLAY chroot
-      groupadd -R $CRAFT_OVERLAY -g 1001 ubuntu
-      useradd -R $CRAFT_OVERLAY -M -r -u 1001 -g ubuntu ubuntu
+      (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query") \
+       > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query

sklearnserver/tests/test_rock.py

@@ -0,0 +1,41 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+#
+# Tests for required artifacts to be present in the ROCK image.
+#
+
+from charmed_kubeflow_chisme.rock import CheckRock
+from pathlib import Path
+
+import os
+import logging
+import random
+import pytest
+import string
+import subprocess
+import yaml
+from pytest_operator.plugin import OpsTest
+
+@pytest.fixture()
+def rock_test_env():
+    """Yields a temporary directory and random docker container name, then cleans them up after."""
+    container_name = "".join([str(i) for i in random.choices(string.ascii_lowercase, k=8)])
+    yield container_name
+
+    try:
+        subprocess.run(["docker", "rm", container_name])
+    except Exception:
+        pass
+
+@pytest.mark.abort_on_fail
+def test_rock(ops_test: OpsTest, rock_test_env):
+    """Test rock."""
+    check_rock = CheckRock("rockcraft.yaml")
+    container_name = rock_test_env
+    LOCAL_ROCK_IMAGE = f"{check_rock.get_image_name()}:{check_rock.get_version()}"
+
+    # verify that all artifacts are in correct locations
+    subprocess.run(["docker", "run", LOCAL_ROCK_IMAGE, "exec", "ls", "-la", "/microservice/SKLearnServer.py"], check=True)
+
+    # verify that rockcraft.yaml contains correct image name for PREDICTIVE_UNIT_IMAGE environment variable
+    #assert CheckRock.get_environment()["PREDICTIVE_UNIT_IMAGE"].contains(LOCAL_ROCK_IMAGE)

seldon-core-operator/tox.ini → sklearnserver/tox.ini

@@ -30,8 +30,8 @@ commands =
              VERSION=$(yq eval .version rockcraft.yaml) && \
              ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \
              ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \
-             sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \
-             docker save $ROCK > $ROCK.tar'
+             sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION'
+
     # run rock tests
     pytest -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests
 
@@ -43,6 +43,7 @@ allowlist_externals =
     rm
     tox
     rockcraft
+    sed
 deps =
     juju~=2.9.0
     pytest
@@ -54,14 +55,21 @@ commands =
     # clone related charm
     rm -rf {env:LOCAL_CHARM_DIR}
     git clone --branch {env:CHARM_BRANCH} {env:CHARM_REPO} {env:LOCAL_CHARM_DIR}
+    # replace jinja2 templated value with yq safe placeholder
+    sed -i "s/namespace: {{ namespace }}/namespace: YQ_SAFE/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2
     # upload rock to docker and microk8s cache, replace charm's container with local rock reference
     bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \
              VERSION=$(yq eval .version rockcraft.yaml) && \
              ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \
              ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \
              sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \
              docker save $ROCK > $ROCK.tar && \
-             microk8s ctr image import $ROCK.tar && \
-             yq e -i ".resources.oci-image.upstream-source=\"$ROCK:$VERSION\"" {env:LOCAL_CHARM_DIR}/metadata.yaml'
+             microk8s ctr image import $ROCK.tar --digests=true && \
+	     predictor_servers=$(yq e ".data.predictor_servers" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2) && \
+             predictor_servers=$(jq --arg jq_rock $ROCK -r '\''.SKLEARN_SERVER.protocols.seldon.image=$jq_rock'\'' <<< $predictor_servers) && \
+             predictor_servers=$(jq --arg jq_version $VERSION -r '\''.SKLEARN_SERVER.protocols.seldon.defaultImageVersion=$jq_version'\'' <<< $predictor_servers) yq e -i ".data.predictor_servers=strenv(predictor_servers)" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2'
+    # replace yq safe placeholder with original value
+    sed -i "s/namespace: YQ_SAFE/namespace: {{ namespace }}/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2
     # run charm integration test with rock
     tox -c {env:LOCAL_CHARM_DIR} -e integration
+