diff --git a/.ci/get-vault-secret.sh b/.ci/get-vault-secret.sh new file mode 100644 index 000000000000..781ac5500d3c --- /dev/null +++ b/.ci/get-vault-secret.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Copyright [2017 - 2020] Confluent Inc. +# + +# Get a secret from Vault. You must already be logged in using the vault-login.sh script. +# You must provide the path to the secret and the secret key for which you want the value. +# The path to the secret is relative to v1/ci/kv +# +# Arguments: +# $1 The path to the secret relative to v1/ci/kv +# $2 The key in the secret you want the value for +# + +set +x + +SECRET=$1 +KEY=$2 +export VAULT_ADDR=https://vault.cireops.gcp.internal.confluent.cloud + +if SECRET_VALUE=$(vault kv get -field "$KEY" v1/ci/kv/"$SECRET"); then + echo "$SECRET_VALUE" +else + echo "Failed to get secret and key: $SECRET, $KEY" + exit 1 +fi diff --git a/.ci/vault-login.sh b/.ci/vault-login.sh new file mode 100644 index 000000000000..b68d525b1917 --- /dev/null +++ b/.ci/vault-login.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# Copyright [2017 - 2020] Confluent Inc. +# + +# Login to Vault using the tools role ID and Secret ID. +# + +export VAULT_ADDR=https://vault.cireops.gcp.internal.confluent.cloud + +if [[ -z "$VAULT_ROLE_ID" ]]; then + echo "Error: The VAULT_ROLE_ID is not set." + exit 1 +fi + +if [[ -z "$VAULT_SECRET_ID" ]]; then + echo "Error: The VAULT_SECRET_ID is not set." + exit 1 +fi + +echo "Getting Vault token with tools Role ID and Secret ID" +VAULT_TOKEN=$(vault write -field=token auth/app/prod/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") + +if [[ -z "$VAULT_TOKEN" ]]; then + echo "Failed to get Vault token." + exit 1 +fi + +echo "Logging into Vault with token." +if vault login "$VAULT_TOKEN" >/dev/null 2>&1; then + echo "Successfully logged into Vault." +else + echo "Failed to login to Vault." + exit 1 +fi diff --git a/create-pip-conf-with-jfrog.sh b/create-pip-conf-with-jfrog.sh new file mode 100644 index 000000000000..3349917fc185 --- /dev/null +++ b/create-pip-conf-with-jfrog.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Copyright [2017 - 2020] Confluent Inc. +# + +set -e + +if [[ -z "${ARTIFACTORY_USERNAME}" || -z "${ARTIFACTORY_PASSWORD}" ]]; then + echo "Please export env ARTIFACTORY_USERNAME and ARTIFACTORY_PASSWORD with Jfrog Artifactory username and password" + exit 1 +fi + +PIP_CONF=~/.pip/pip.conf + +mkdir -p `dirname ${PIP_CONF}` + +if [ -e "${PIP_CONF}" ]; then + echo "${PIP_CONF} already exists and will be used" +else + echo -e "[global]\nindex-url = https://${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD}@confluent.jfrog.io/confluent/api/pypi/pypi/simple" > ${PIP_CONF} + echo "Created ${PIP_CONF} with Jfrog Artifactory repository" +fi diff --git a/create-pypirc-with-jfrog.sh b/create-pypirc-with-jfrog.sh new file mode 100644 index 000000000000..35c6ef78c9fc --- /dev/null +++ b/create-pypirc-with-jfrog.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Copyright [2017 - 2020] Confluent Inc. +# + +set -e + +if [[ -z $ARTIFACTORY_USERNAME || -z $ARTIFACTORY_PASSWORD ]]; then + echo "Please export env ARTIFACTORY_USERNAME and ARTIFACTORY_PASSWORD with Jfrog Artifactory username and password" + exit 1 +fi + +PYPIRC=~/.pypirc + +if [ -e $PYPIRC ]; then + echo "$PYPIRC already exists and will be used." +else + echo "\ +[distutils] +index-servers = + jfrog + +[jfrog] +repository: https://confluent.jfrog.io/confluent/api/pypi/pypi-internal +username: $ARTIFACTORY_USERNAME +password: $ARTIFACTORY_PASSWORD +" > $PYPIRC + echo "Created $PYPIRC with Jfrog Artifactory repository" +fi diff --git a/extract-iam-credential.sh b/extract-iam-credential.sh new file mode 100644 index 000000000000..7bc15b410c7f --- /dev/null +++ b/extract-iam-credential.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# Copyright [2017 - 2020] Confluent Inc. +# + +# Extract AWS credential using its assigned IAM role. +# +# This script is meant to be sourced by a parent script, so from your parent script: +# . jenkins-common/scripts/aws/extract-iam-credential.sh + +[ -o xtrace ] && restore_trace='set -x' || restore_trace='set +x' +set +x + + +if [[ ${AWS_ACCESS_KEY_ID:-} && ${AWS_ACCESS_KEY:-} && ${AWS_SECRET_ACCESS_KEY:-} && ${AWS_SECRET_KEY:-} && ! ${AWS_SESSION_TOKEN:-} ]]; then + echo "Using existing AWS credential from env" +else + echo "Extracting AWS credential based on IAM role" + + if [[ ! "${AWS_IAM:-}" ]];then + # Get the iam rolename associated with this ec2 instance. The curl command actually fetches the iam instance profile associated with this ec2 node, + # and what we really need is the iam role name, so we rely on the convention that + # iam instance profile name and iam role name are the same + export AWS_IAM=$(curl -s http://169.254.169.254/latest/meta-data/iam/info | grep InstanceProfileArn | cut -d '"' -f 4 | cut -d '/' -f 2) + fi + + export AWS_ACCESS_KEY_ID=`curl -s http://169.254.169.254/latest/meta-data/iam/security-credentials/$AWS_IAM | grep AccessKeyId | awk -F\" '{ print $4 }'` + export AWS_ACCESS_KEY="$AWS_ACCESS_KEY_ID" + export AWS_SECRET_KEY=`curl -s http://169.254.169.254/latest/meta-data/iam/security-credentials/$AWS_IAM | grep SecretAccessKey | awk -F\" '{ print $4 }'` + export AWS_SECRET_ACCESS_KEY="$AWS_SECRET_KEY" + export AWS_SESSION_TOKEN=`curl -s http://169.254.169.254/latest/meta-data/iam/security-credentials/$AWS_IAM | grep Token | awk -F\" '{ print $4 }'` + + if [ -z "$AWS_ACCESS_KEY" ]; then + echo "Failed to populate environment variables AWS_ACCESS_KEY, AWS_SECRET_KEY, and AWS_SESSION_TOKEN." + echo "AWS_IAM is currently $AWS_IAM. Double-check that this is correct. If not, update AWS_IAM env in aws.sh" + exit 1 + fi + +fi + +export AWS_DEFAULT_REGION=us-west-2 + +eval "$restore_trace" diff --git a/ksqldb-api-client/pom.xml b/ksqldb-api-client/pom.xml index 1865794b6ee1..c611f6d67e50 100644 --- a/ksqldb-api-client/pom.xml +++ b/ksqldb-api-client/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-api-client diff --git a/ksqldb-api-reactive-streams-tck/pom.xml b/ksqldb-api-reactive-streams-tck/pom.xml index 5fe7ebcd02c2..e2424498e102 100644 --- a/ksqldb-api-reactive-streams-tck/pom.xml +++ b/ksqldb-api-reactive-streams-tck/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-api-reactive-streams-tck diff --git a/ksqldb-benchmark/pom.xml b/ksqldb-benchmark/pom.xml index 18e42578c542..56c8f4dd5efb 100644 --- a/ksqldb-benchmark/pom.xml +++ b/ksqldb-benchmark/pom.xml @@ -47,7 +47,7 @@ questions. io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-benchmark diff --git a/ksqldb-cli/pom.xml b/ksqldb-cli/pom.xml index d8e517d9aaa6..030ef4620ba5 100644 --- a/ksqldb-cli/pom.xml +++ b/ksqldb-cli/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-cli diff --git a/ksqldb-common/pom.xml b/ksqldb-common/pom.xml index 211d65012fb1..9c176b20a221 100644 --- a/ksqldb-common/pom.xml +++ b/ksqldb-common/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-common diff --git a/ksqldb-console-scripts/pom.xml b/ksqldb-console-scripts/pom.xml index 8063e25e4945..824a97b7cac9 100644 --- a/ksqldb-console-scripts/pom.xml +++ b/ksqldb-console-scripts/pom.xml @@ -22,7 +22,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 io.confluent.ksql diff --git a/ksqldb-docker/pom.xml b/ksqldb-docker/pom.xml index c25072943caf..21ff57546ff0 100644 --- a/ksqldb-docker/pom.xml +++ b/ksqldb-docker/pom.xml @@ -22,7 +22,7 @@ ksqldb-parent io.confluent.ksql - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-docker diff --git a/ksqldb-engine/pom.xml b/ksqldb-engine/pom.xml index 32edf1399939..ef47e3122cb8 100644 --- a/ksqldb-engine/pom.xml +++ b/ksqldb-engine/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-engine diff --git a/ksqldb-etc/pom.xml b/ksqldb-etc/pom.xml index 681e584386b9..052b4f577cf3 100644 --- a/ksqldb-etc/pom.xml +++ b/ksqldb-etc/pom.xml @@ -22,7 +22,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 io.confluent.ksql diff --git a/ksqldb-examples/pom.xml b/ksqldb-examples/pom.xml index bd7d789e88b7..42079a6324f3 100644 --- a/ksqldb-examples/pom.xml +++ b/ksqldb-examples/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-examples diff --git a/ksqldb-execution/pom.xml b/ksqldb-execution/pom.xml index f4b19b7ea90b..dd4ba52e4270 100644 --- a/ksqldb-execution/pom.xml +++ b/ksqldb-execution/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-execution diff --git a/ksqldb-functional-tests/pom.xml b/ksqldb-functional-tests/pom.xml index 0096d0cd0c34..a998cb243ce1 100644 --- a/ksqldb-functional-tests/pom.xml +++ b/ksqldb-functional-tests/pom.xml @@ -21,7 +21,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 4.0.0 diff --git a/ksqldb-metastore/pom.xml b/ksqldb-metastore/pom.xml index 72ee021bee01..4322b7ea10b2 100644 --- a/ksqldb-metastore/pom.xml +++ b/ksqldb-metastore/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-metastore diff --git a/ksqldb-package/pom.xml b/ksqldb-package/pom.xml index ab2e00db6f62..c86840c93421 100644 --- a/ksqldb-package/pom.xml +++ b/ksqldb-package/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-package diff --git a/ksqldb-parser/pom.xml b/ksqldb-parser/pom.xml index 076a3c9dc0be..3556e45c6042 100644 --- a/ksqldb-parser/pom.xml +++ b/ksqldb-parser/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-parser diff --git a/ksqldb-rest-app/pom.xml b/ksqldb-rest-app/pom.xml index 12285125021b..e940bccb5a33 100644 --- a/ksqldb-rest-app/pom.xml +++ b/ksqldb-rest-app/pom.xml @@ -23,7 +23,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-rest-app diff --git a/ksqldb-rest-client/pom.xml b/ksqldb-rest-client/pom.xml index ed93d396398d..a3ffe73688d4 100644 --- a/ksqldb-rest-client/pom.xml +++ b/ksqldb-rest-client/pom.xml @@ -23,7 +23,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-rest-client diff --git a/ksqldb-rest-model/pom.xml b/ksqldb-rest-model/pom.xml index a929806d2238..7d47dfe055fb 100644 --- a/ksqldb-rest-model/pom.xml +++ b/ksqldb-rest-model/pom.xml @@ -23,7 +23,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-rest-model diff --git a/ksqldb-rocksdb-config-setter/pom.xml b/ksqldb-rocksdb-config-setter/pom.xml index e818528f50b2..182eb12bd94d 100644 --- a/ksqldb-rocksdb-config-setter/pom.xml +++ b/ksqldb-rocksdb-config-setter/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-rocksdb-config-setter diff --git a/ksqldb-serde/pom.xml b/ksqldb-serde/pom.xml index 7905de984aa9..3a1dc209a12e 100644 --- a/ksqldb-serde/pom.xml +++ b/ksqldb-serde/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-serde diff --git a/ksqldb-streams/pom.xml b/ksqldb-streams/pom.xml index 18e79f99d1ae..221233782704 100644 --- a/ksqldb-streams/pom.xml +++ b/ksqldb-streams/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-streams diff --git a/ksqldb-test-util/pom.xml b/ksqldb-test-util/pom.xml index cee8d59f18c7..5f70265f5cd7 100644 --- a/ksqldb-test-util/pom.xml +++ b/ksqldb-test-util/pom.xml @@ -20,7 +20,7 @@ ksqldb-parent io.confluent.ksql - 6.0.0-SNAPSHOT + 0.10.0 4.0.0 diff --git a/ksqldb-tools/pom.xml b/ksqldb-tools/pom.xml index c48226295505..5674b718d757 100644 --- a/ksqldb-tools/pom.xml +++ b/ksqldb-tools/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-tools diff --git a/ksqldb-udf-quickstart/pom.xml b/ksqldb-udf-quickstart/pom.xml index 0ce7e0804362..5bab73f9a6a5 100644 --- a/ksqldb-udf-quickstart/pom.xml +++ b/ksqldb-udf-quickstart/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-udf-quickstart diff --git a/ksqldb-udf/pom.xml b/ksqldb-udf/pom.xml index b386a5879476..8c6b86745fd3 100644 --- a/ksqldb-udf/pom.xml +++ b/ksqldb-udf/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-udf diff --git a/ksqldb-version-metrics-client/pom.xml b/ksqldb-version-metrics-client/pom.xml index fb1c751b6008..daa9f14f9517 100644 --- a/ksqldb-version-metrics-client/pom.xml +++ b/ksqldb-version-metrics-client/pom.xml @@ -22,7 +22,7 @@ io.confluent.ksql ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 ksqldb-version-metrics-client diff --git a/maven-settings.xml b/maven-settings.xml new file mode 100644 index 000000000000..e883f7f24910 --- /dev/null +++ b/maven-settings.xml @@ -0,0 +1,39 @@ + + + + release + + + true + + + + + confluent-other + Confluent + https://jenkins-confluent-packages-beta-maven.s3-us-west-2.amazonaws.com/6.0.0-beta200608020919/1/maven + + + + + + + confluent-artifactory-central + Confluent Plugin Repository + https://jenkins-confluent-packages-beta-maven.s3-us-west-2.amazonaws.com/6.0.0-beta200608020919/1/maven + default + + true + + + true + + + + + + + diff --git a/pom.xml b/pom.xml index 002fbc323360..7f4870213f5d 100644 --- a/pom.xml +++ b/pom.xml @@ -22,14 +22,14 @@ io.confluent common - 6.0.0-SNAPSHOT + 6.0.0-beta200608020919 io.confluent.ksql ksqldb-parent pom ksqldb-parent - 6.0.0-SNAPSHOT + 0.10.0 Confluent Community License diff --git a/set-global-user.sh b/set-global-user.sh new file mode 100644 index 000000000000..69988e7d647e --- /dev/null +++ b/set-global-user.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Copyright [2017 - 2020] Confluent Inc. +# + +set -ex + +git config --global user.email "jenkins@confluent.io" +git config --global user.name "Confluent Jenkins Bot" diff --git a/setup-credential-store.sh b/setup-credential-store.sh new file mode 100644 index 000000000000..26de9ab902b8 --- /dev/null +++ b/setup-credential-store.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright [2017 - 2020] Confluent Inc. +# + +set -e + +if [[ -z $GIT_CREDENTIAL ]]; then + echo "Please export GIT_CREDENTIAL to contain git username:password" + exit 1 +fi + +git config --global credential.helper store +echo "https://$GIT_CREDENTIAL@github.com" > ~/.git-credentials