Update wordcount example to use docker-flink image #135

examples/wordcount/Dockerfile

@@ -1,34 +1,14 @@
-FROM openjdk:8-jdk
+FROM flink:1.8.2-scala_2.12
 
 # Prepare environment
-ENV FLINK_HOME=/opt/flink
 ENV MAVEN_HOME=/opt/maven
-ENV HADOOP_HOME=/opt/hadoop
-ENV PATH=$FLINK_HOME/bin:$HADOOP_HOME/bin:$MAVEN_HOME/bin:$PATH
-
-COPY . /code
-
-# Configure Flink version
-ENV FLINK_VERSION=1.8.2 \
-    HADOOP_SCALA_VARIANT=scala_2.12
+ENV PATH=$MAVEN_HOME/bin:$PATH
 
 # Install dependencies
 RUN set -ex; \
-  apt-get update; \
-  apt-get -y install libsnappy1v5; \
-  apt-get -y install netcat net-tools; \
-  apt-get -y install gettext-base; \
-  rm -rf /var/lib/apt/lists/*
-
-# Grab gosu for easy step-down from root
-ENV GOSU_VERSION 1.11
-RUN set -ex; \
-  wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \
-  wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \
-  export GNUPGHOME="$(mktemp -d)"; \
-  rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
-  chmod +x /usr/local/bin/gosu; \
-  gosu nobody true
+  apt-get update \
+  && apt-get -y install gettext-base openjdk-8-jdk-headless \
+  && rm -rf /var/lib/apt/lists/*
 
 # Install Maven
 ENV MAVEN_VERSION 3.6.1
@@ -38,39 +18,13 @@ RUN \
   mv apache-maven-$MAVEN_VERSION $MAVEN_HOME; \
   rm apache-maven-$MAVEN_VERSION-bin.tar.gz
 
+# Build application jar
+COPY . /code
 WORKDIR /code
-
 RUN \
-  mvn package; \
-  mkdir -p /opt/flink/flink-web-upload; \
-  cp flink-conf.yaml /usr/local/; \
-  cp /code/target/*.jar /opt/flink/flink-web-upload/
-
-RUN groupadd --system --gid=9999 flink && \
-    useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink
-WORKDIR $FLINK_HOME
-
-ENV FLINK_URL_FILE_PATH=flink/flink-${FLINK_VERSION}/flink-${FLINK_VERSION}-bin-${HADOOP_SCALA_VARIANT}.tgz
-ENV FLINK_TGZ_URL=https://archive.apache.org/dist/$FLINK_URL_FILE_PATH
-
-# Install Flink
-RUN set -ex; \
-  wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
-  \
-  tar -xf flink.tgz --strip-components=1; \
-  rm flink.tgz; \
-  \
-  cp ./opt/flink-s3-fs-presto-${FLINK_VERSION}.jar ./lib/;\
-  cp ./opt/flink-s3-fs-hadoop-${FLINK_VERSION}.jar ./lib/;\
-  \  
-  chown -R flink:flink .;
-
-# Needed on OpenShift for the entrypoint script to work
-RUN chmod -R 777 /opt/flink
+  JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 mvn package \
+  && ln -s /code/target $FLINK_HOME/flink-web-upload
 
-#  control script expects manifest.yaml at this location
-RUN chown -R flink:flink /var
-COPY docker-entrypoint.sh /
-ENTRYPOINT ["/docker-entrypoint.sh"]
-EXPOSE 6123 8081
-CMD ["local"]
+COPY docker-entrypoint.sh /flinkk8soperator-entrypoint.sh
+ENTRYPOINT ["/flinkk8soperator-entrypoint.sh"]
+CMD ["help"]

examples/wordcount/docker-entrypoint.sh

@@ -1,45 +1,10 @@
 #!/bin/sh
 
-drop_privs_cmd() {
-    if [ $(id -u) != 0 ]; then
-        # Don't need to drop privs if EUID != 0
-        return
-    elif [ -x /sbin/su-exec ]; then
-        # Alpine
-        echo su-exec
-    else
-        # Others
-        echo gosu flink
-    fi
-}
-
-# Add in extra configs set by the operator
+# Map config from FlinkK8sOperator to base container
+# https://github.com/lyft/flinkk8soperator/issues/135
+# https://github.com/docker-flink/docker-flink/pull/91
 if [ -n "$OPERATOR_FLINK_CONFIG" ]; then
-    echo "$OPERATOR_FLINK_CONFIG" >> "/usr/local/flink-conf.yaml"
-fi
-
-envsubst < /usr/local/flink-conf.yaml > $FLINK_HOME/conf/flink-conf.yaml
-
-COMMAND=$@
-
-if [ $# -lt 1 ]; then
-    COMMAND="local"
-fi
-
-if [ "$COMMAND" = "help" ]; then
-    echo "Usage: $(basename "$0") (jobmanager|taskmanager|local|help)"
-    exit 0
-elif [ "$FLINK_DEPLOYMENT_TYPE" = "jobmanager" ]; then
-    echo "Starting Job Manager"
-    echo "config file: " && grep '^[^\n#]' "$FLINK_HOME/conf/flink-conf.yaml"
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground
-elif [ "$FLINK_DEPLOYMENT_TYPE" = "taskmanager" ]; then
-    echo "Starting Task Manager"
-    echo "config file: " && grep '^[^\n#]' "$FLINK_HOME/conf/flink-conf.yaml"
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground
-elif [ "$COMMAND" = "local" ]; then
-    echo "Starting local cluster"
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground local
+    export FLINK_PROPERTIES="`echo \"${OPERATOR_FLINK_CONFIG}\" | envsubst`"
 fi
 
-exec "$@"
+exec /docker-entrypoint.sh "$@"

examples/wordcount/flink-operator-custom-resource.yaml

@@ -8,23 +8,27 @@ metadata:
     environment: development
 spec:
   image: docker.io/lyft/wordcount-operator-example:{sha}
+  deleteMode: None
   flinkConfig:
     taskmanager.heap.size: 200
+    taskmanager.network.memory.fraction: 0.1
+    taskmanager.network.memory.min: 10m
     state.backend.fs.checkpointdir: file:///checkpoints/flink/checkpoints
     state.checkpoints.dir: file:///checkpoints/flink/externalized-checkpoints
     state.savepoints.dir: file:///checkpoints/flink/savepoints
+    web.upload.dir: /opt/flink
   jobManagerConfig:
     resources:
       requests:
         memory: "200Mi"
-        cpu: "0.2"
+        cpu: "0.1"
     replicas: 1
   taskManagerConfig:
     taskSlots: 2
     resources:
       requests:
         memory: "200Mi"
-        cpu: "0.2"
+        cpu: "0.1"
   flinkVersion: "1.8"
   jarName: "wordcount-operator-example-1.0.0-SNAPSHOT.jar"
   parallelism: 3