From e8897827aa611c405899ccad1dacdf61e4295c0c Mon Sep 17 00:00:00 2001 From: John Roesler Date: Fri, 21 Jun 2019 14:19:49 -0500 Subject: [PATCH 1/3] Move docker image to break circular dependency --- ksql-examples/Dockerfile | 12 ------------ ksql-examples/pom.xml | 7 ------- 2 files changed, 19 deletions(-) delete mode 100644 ksql-examples/Dockerfile diff --git a/ksql-examples/Dockerfile b/ksql-examples/Dockerfile deleted file mode 100644 index 8f115cd365f4..000000000000 --- a/ksql-examples/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -ARG DOCKER_UPSTREAM_REGISTRY -ARG DOCKER_UPSTREAM_TAG - -FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-new:${DOCKER_UPSTREAM_TAG} - -ARG PROJECT_VERSION -ARG ARTIFACT_ID - -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/bin/* /usr/bin/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/etc/* /etc/ksql/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ diff --git a/ksql-examples/pom.xml b/ksql-examples/pom.xml index 3c70444f4fce..e3c0cf969509 100644 --- a/ksql-examples/pom.xml +++ b/ksql-examples/pom.xml @@ -17,7 +17,6 @@ io.confluent.ksql.datagen.DataGen false ${main-class} - false 0.2.0 @@ -51,12 +50,6 @@ test - - io.confluent - monitoring-interceptors - ${project.version} - - io.confluent.avro avro-random-generator From 00ac8d785bcc33a732a6ef043fc4a480ca134bc1 Mon Sep 17 00:00:00 2001 From: John Roesler Date: Fri, 21 Jun 2019 15:26:08 -0500 Subject: [PATCH 2/3] remove other docker images --- cp-ksql-cli/Dockerfile | 32 ----- .../include/etc/confluent/docker/configure | 30 ----- .../docker/log4j.properties.template | 16 --- cp-ksql-cli/pom.xml | 100 ---------------- cp-ksql-server/Dockerfile | 34 ------ .../include/etc/confluent/docker/configure | 35 ------ .../include/etc/confluent/docker/ensure | 42 ------- .../docker/ksql-server.properties.template | 4 - .../include/etc/confluent/docker/launch | 37 ------ .../docker/log4j.properties.template | 13 -- .../include/etc/confluent/docker/run | 32 ----- cp-ksql-server/pom.xml | 111 ------------------ pom.xml | 2 - 13 files changed, 488 deletions(-) delete mode 100644 cp-ksql-cli/Dockerfile delete mode 100644 cp-ksql-cli/include/etc/confluent/docker/configure delete mode 100644 cp-ksql-cli/include/etc/confluent/docker/log4j.properties.template delete mode 100644 cp-ksql-cli/pom.xml delete mode 100644 cp-ksql-server/Dockerfile delete mode 100644 cp-ksql-server/include/etc/confluent/docker/configure delete mode 100644 cp-ksql-server/include/etc/confluent/docker/ensure delete mode 100644 cp-ksql-server/include/etc/confluent/docker/ksql-server.properties.template delete mode 100644 cp-ksql-server/include/etc/confluent/docker/launch delete mode 100644 cp-ksql-server/include/etc/confluent/docker/log4j.properties.template delete mode 100644 cp-ksql-server/include/etc/confluent/docker/run delete mode 100644 cp-ksql-server/pom.xml diff --git a/cp-ksql-cli/Dockerfile b/cp-ksql-cli/Dockerfile deleted file mode 100644 index cef3ada4f321..000000000000 --- a/cp-ksql-cli/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -ARG DOCKER_UPSTREAM_REGISTRY -ARG DOCKER_UPSTREAM_TAG - -FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-new:${DOCKER_UPSTREAM_TAG} - -ENV COMPONENT=ksql-cli -ENV KSQL_CLASSPATH=/usr/share/java/${COMPONENT}/* - -ARG COMMIT_ID=unknown -LABEL io.confluent.docker.git.id=$COMMIT_ID -ARG BUILD_NUMBER=-1 -LABEL io.confluent.docker.build.number=$BUILD_NUMBER -LABEL io.confluent.docker=true - -ARG PROJECT_VERSION -ARG ARTIFACT_ID - -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${COMPONENT}/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/bin/* /usr/bin/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/etc/* /etc/ksql/ - -COPY include/etc/confluent/docker /etc/confluent/docker - -RUN mkdir -p /etc/${COMPONENT} /etc/${COMPONENT}/secrets /var/log/${COMPONENT} - -RUN bash /etc/confluent/docker/configure -ENV KSQL_LOG4J_OPTS=-Dlog4j.configuration=file:/etc/${COMPONENT}/log4j.properties - -ENTRYPOINT ["ksql"] -CMD ["http://localhost:8088"] - diff --git a/cp-ksql-cli/include/etc/confluent/docker/configure b/cp-ksql-cli/include/etc/confluent/docker/configure deleted file mode 100644 index ea8a510f52b8..000000000000 --- a/cp-ksql-cli/include/etc/confluent/docker/configure +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018 Confluent Inc. -# -# Licensed under the Confluent Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.confluent.io/confluent-community-license -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -. /etc/confluent/docker/bash-config - -echo "===> ENV Variables ..." -show_env - -echo "===> User" -id - -echo "===> Configuring ..." -dub path /etc/"${COMPONENT}"/ writable -dub path /var/log/"${COMPONENT}"/ writable - -dub template "/etc/confluent/docker/log4j.properties.template" "/etc/${COMPONENT}/log4j.properties" - diff --git a/cp-ksql-cli/include/etc/confluent/docker/log4j.properties.template b/cp-ksql-cli/include/etc/confluent/docker/log4j.properties.template deleted file mode 100644 index cf925e542b19..000000000000 --- a/cp-ksql-cli/include/etc/confluent/docker/log4j.properties.template +++ /dev/null @@ -1,16 +0,0 @@ -log4j.rootLogger={{ env["KSQL_LOG4J_ROOT_LOGLEVEL"] | default('INFO') }}, default.file - -log4j.appender.default.file=io.confluent.ksql.util.TimestampLogFileAppender -log4j.appender.default.file.ImmediateFlush=true -log4j.appender.default.file.append=false - -log4j.appender.default.file.file=/var/log/{{ env["COMPONENT"] }}/cli-%timestamp.log -log4j.appender.default.file.layout=org.apache.log4j.PatternLayout -log4j.appender.default.file.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - -{% if env['KSQL_LOG4J_LOGGERS'] %} -{% set loggers = parse_log4j_loggers(env['KSQL_LOG4J_LOGGERS']) %} -{% for logger,loglevel in loggers.iteritems() %} -log4j.logger.{{logger}}={{loglevel}}, stdout -{% endfor %} -{% endif %} diff --git a/cp-ksql-cli/pom.xml b/cp-ksql-cli/pom.xml deleted file mode 100644 index e7ead74d6b51..000000000000 --- a/cp-ksql-cli/pom.xml +++ /dev/null @@ -1,100 +0,0 @@ - - - - - 4.0.0 - - - io.confluent.ksql - ksql-parent - 5.1.4-SNAPSHOT - - - io.confluent.ksql - cp-ksql-cli - KSQL Server CLI image - - - false - - - - - junit - junit - test - - - - ${project.groupId} - ksql-cli - ${project.version} - - - - ${project.groupId} - ksql-console-scripts - ${project.version} - resources - zip - provided - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.0.2 - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 2.6 - - - none - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack-console-scripts - - unpack-dependencies - - generate-resources - - ksql-console-scripts - - - - - - - diff --git a/cp-ksql-server/Dockerfile b/cp-ksql-server/Dockerfile deleted file mode 100644 index 66e95c429d0b..000000000000 --- a/cp-ksql-server/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -ARG DOCKER_UPSTREAM_REGISTRY -ARG DOCKER_UPSTREAM_TAG - -FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-new:${DOCKER_UPSTREAM_TAG} - -EXPOSE 8088 - -ENV COMPONENT=ksql-server -ENV KSQL_CLASSPATH=/usr/share/java/${COMPONENT}/* - -ARG COMMIT_ID=unknown -LABEL io.confluent.docker.git.id=$COMMIT_ID -ARG BUILD_NUMBER=-1 -LABEL io.confluent.docker.build.number=$BUILD_NUMBER -LABEL io.confluent.docker=true - -ARG PROJECT_VERSION -ARG ARTIFACT_ID - -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${COMPONENT}/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/bin/* /usr/bin/ -ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/etc/* /etc/ksql/ - -COPY include/etc/confluent/docker /etc/confluent/docker - -RUN mkdir -p /etc/${COMPONENT} /etc/${COMPONENT}/secrets - -RUN chmod +x /etc/confluent/docker/run -RUN chmod +x /etc/confluent/docker/configure -RUN chmod +x /etc/confluent/docker/launch -RUN chmod +x /etc/confluent/docker/ensure - -CMD ["/etc/confluent/docker/run"] diff --git a/cp-ksql-server/include/etc/confluent/docker/configure b/cp-ksql-server/include/etc/confluent/docker/configure deleted file mode 100644 index e794004cf9a8..000000000000 --- a/cp-ksql-server/include/etc/confluent/docker/configure +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018 Confluent Inc. -# -# Licensed under the Confluent Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.confluent.io/confluent-community-license -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -. /etc/confluent/docker/bash-config - -dub ensure-atleast-one KSQL_BOOTSTRAP_SERVERS KSQL_KSQL_STREAMS_BOOTSTRAP_SERVERS - -dub path /etc/"${COMPONENT}"/ writable - -if [[ -n "${KSQL_JMX_OPTS-}" ]] -then - if [[ ! $KSQL_JMX_OPTS == *"com.sun.management.jmxremote.rmi.port"* ]] - then - echo "KSQL_OPTS should contain the 'com.sun.management.jmxremote.rmi.port' property. It is required for accessing the JMX metrics externally. Also remember to map the JMX port so that it is accessible from outside the container." - fi -fi - -dub template "/etc/confluent/docker/${COMPONENT}.properties.template" "/etc/${COMPONENT}/${COMPONENT}.properties" -dub template "/etc/confluent/docker/log4j.properties.template" "/etc/${COMPONENT}/log4j.properties" - -# Ensure that KSQL picks up the correct log4j properties file -export KSQL_LOG4J_OPTS=-Dlog4j.configuration=file:/etc/${COMPONENT}/log4j.properties diff --git a/cp-ksql-server/include/etc/confluent/docker/ensure b/cp-ksql-server/include/etc/confluent/docker/ensure deleted file mode 100644 index ada5bd0bf879..000000000000 --- a/cp-ksql-server/include/etc/confluent/docker/ensure +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018 Confluent Inc. -# -# Licensed under the Confluent Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.confluent.io/confluent-community-license -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -. /etc/confluent/docker/bash-config - -echo "===> Check if Kafka is healthy ..." - -BOOTSTRAP_SERVER_CONFIG="" -if [[ -n "${KSQL_BOOTSTRAP_SERVERS-}" ]] -then - BOOTSTRAP_SERVER_CONFIG=${KSQL_BOOTSTRAP_SERVERS} -else - BOOTSTRAP_SERVER_CONFIG=${KSQL_KSQL_STREAMS_BOOTSTRAP_SERVERS} -fi - -if [[ -n "${KSQL_SECURITY_PROTOCOL-}" ]] && [[ $KSQL_SECURITY_PROTOCOL != "PLAINTEXT" ]] -then - cub kafka-ready \ - "${KSQL_CUB_KAFKA_MIN_BROKERS:-1}" \ - "${KSQL_CUB_KAFKA_TIMEOUT:-180}" \ - -b "${BOOTSTRAP_SERVER_CONFIG}" \ - --config /etc/"${COMPONENT}"/"${COMPONENT}".properties -elif [[ -n "${BOOTSTRAP_SERVER_CONFIG-}" ]] -then - cub kafka-ready \ - "${KSQL_CUB_KAFKA_MIN_BROKERS:-1}" \ - "${KSQL_CUB_KAFKA_TIMEOUT:-180}" \ - -b "${BOOTSTRAP_SERVER_CONFIG}" -fi diff --git a/cp-ksql-server/include/etc/confluent/docker/ksql-server.properties.template b/cp-ksql-server/include/etc/confluent/docker/ksql-server.properties.template deleted file mode 100644 index 2efa78161e60..000000000000 --- a/cp-ksql-server/include/etc/confluent/docker/ksql-server.properties.template +++ /dev/null @@ -1,4 +0,0 @@ -{% set kr_props = env_to_props('KSQL_', '') -%} -{% for name, value in kr_props.iteritems() -%} -{{name}}={{value}} -{% endfor -%} diff --git a/cp-ksql-server/include/etc/confluent/docker/launch b/cp-ksql-server/include/etc/confluent/docker/launch deleted file mode 100644 index edb2b2a26740..000000000000 --- a/cp-ksql-server/include/etc/confluent/docker/launch +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018 Confluent Inc. -# -# Licensed under the Confluent Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.confluent.io/confluent-community-license -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -# JMX settings -if [ -z "$KSQL_JMX_OPTS" ]; then - KSQL_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " -fi - -# The JMX client needs to be able to connect to java.rmi.server.hostname. -# The default for bridged network is the bridged IP so you will only be able to connect from another docker container. -# For host network, this is the IP that the hostname on the host resolves to. - -# If you have more that one network configured, hostname -i gives you all the IPs, -# the default is to pick the first IP (or network). -export KSQL_JMX_HOSTNAME=${KSQL_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} - -# JMX port to use -if [ $KSQL_JMX_PORT ]; then - export JMX_PORT=$KSQL_JMX_PORT -export KSQL_JMX_OPTS="$KSQL_JMX_OPTS -Djava.rmi.server.hostname=$KSQL_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" -fi - -echo "===> Launching ${COMPONENT} ... " -exec "${COMPONENT}"-start /etc/"${COMPONENT}"/"${COMPONENT}".properties diff --git a/cp-ksql-server/include/etc/confluent/docker/log4j.properties.template b/cp-ksql-server/include/etc/confluent/docker/log4j.properties.template deleted file mode 100644 index 810d5dfe0258..000000000000 --- a/cp-ksql-server/include/etc/confluent/docker/log4j.properties.template +++ /dev/null @@ -1,13 +0,0 @@ - -log4j.rootLogger={{ env["KSQL_LOG4J_ROOT_LOGLEVEL"] | default('INFO') }}, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -{% if env['KSQL_LOG4J_LOGGERS'] %} -{% set loggers = parse_log4j_loggers(env['KSQL_LOG4J_LOGGERS']) %} -{% for logger,loglevel in loggers.iteritems() %} -log4j.logger.{{logger}}={{loglevel}}, stdout -{% endfor %} -{% endif %} diff --git a/cp-ksql-server/include/etc/confluent/docker/run b/cp-ksql-server/include/etc/confluent/docker/run deleted file mode 100644 index e582674b4d1f..000000000000 --- a/cp-ksql-server/include/etc/confluent/docker/run +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018 Confluent Inc. -# -# Licensed under the Confluent Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.confluent.io/confluent-community-license -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -. /etc/confluent/docker/bash-config - -echo "===> ENV Variables ..." -show_env - -echo "===> User" -id - -echo "===> Configuring ..." -/etc/confluent/docker/configure - -echo "===> Running preflight checks ... " -/etc/confluent/docker/ensure - -echo "===> Launching ... " -exec /etc/confluent/docker/launch diff --git a/cp-ksql-server/pom.xml b/cp-ksql-server/pom.xml deleted file mode 100644 index 60e5d23a0d20..000000000000 --- a/cp-ksql-server/pom.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - - 4.0.0 - - - io.confluent.ksql - ksql-parent - 5.1.4-SNAPSHOT - - - io.confluent.ksql - cp-ksql-server - KSQL Server Docker image - - - false - - - - - junit - junit - test - - - org.hamcrest - hamcrest-core - - - - - - io.confluent - monitoring-interceptors - ${project.version} - - - - ${project.groupId} - ksql-rest-app - ${project.version} - - - - ${project.groupId} - ksql-console-scripts - ${project.version} - resources - zip - provided - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.0.2 - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack-console-scripts - - unpack-dependencies - - generate-resources - - ksql-console-scripts - - - - - - org.apache.maven.plugins - maven-jar-plugin - 2.6 - - - none - - - - - - diff --git a/pom.xml b/pom.xml index 4e553e29d674..9d2cfa1746cf 100644 --- a/pom.xml +++ b/pom.xml @@ -38,8 +38,6 @@ ksql-console-scripts ksql-package ksql-test-util - cp-ksql-cli - cp-ksql-server From e138b10d6a01c77026f7f4d6aeb0def2a53564c0 Mon Sep 17 00:00:00 2001 From: John Roesler Date: Fri, 21 Jun 2019 17:20:53 -0500 Subject: [PATCH 3/3] fix jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fb750b513bbf..3a89ad5e29a5 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ dockerfile { slackChannel = '#ksql-eng' upstreamProjects = 'confluentinc/schema-registry' - dockerRepos = ['confluentinc/ksql-examples', 'confluentinc/ksql-cli', 'confluentinc/cp-ksql-cli', 'confluentinc/cp-ksql-server'] + dockerRepos = ['confluentinc/ksql-cli'] extraBuildArgs = '-Ddocker.skip=false' extraDeployArgs = '-Ddocker.skip=true' }