From 32329f89d2be6f12cf1ee1b14d015a86b39a2ae1 Mon Sep 17 00:00:00 2001 From: Andrew Ash Date: Tue, 4 Oct 2016 17:54:35 -0700 Subject: [PATCH 1/6] Re-enable tests --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index b8d1a36da76c9..a52d688c0307d 100644 --- a/circle.yml +++ b/circle.yml @@ -14,7 +14,7 @@ checkout: dependencies: override: - - ./build/mvn -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive dependency:go-offline + - ./build/mvn -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive dependency:go-offline test: override: From d7c239cea5ca7d141c2f8df7b9da87c5f5750cc4 Mon Sep 17 00:00:00 2001 From: Andrew Ash Date: Mon, 10 Oct 2016 21:10:59 -0700 Subject: [PATCH 2/6] circle cleanup - use existing ./dev/run-tests for running tests - only publish from publish block for tags and master branch and not on every CI build --- circle.yml | 104 ++++++++++++++++++++++++++++++------------------- dev/publish.sh | 17 -------- 2 files changed, 63 insertions(+), 58 deletions(-) delete mode 100755 dev/publish.sh diff --git a/circle.yml b/circle.yml index a52d688c0307d..55abd7f82e6bc 100644 --- a/circle.yml +++ b/circle.yml @@ -14,55 +14,77 @@ checkout: dependencies: override: - - ./build/mvn -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive dependency:go-offline + - ./build/mvn -Phadoop-2.7 -DskipTests -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive dependency:go-offline test: override: - - ? | - set -euo pipefail - version=$(git describe --tags) - - publish_artifacts() { - tmp_settings="tmp-settings.xml" - echo "" > $tmp_settings - echo "bintray-palantir-release$BINTRAY_USERNAME" >> $tmp_settings - echo "$BINTRAY_PASSWORD" >> $tmp_settings - echo "" >> $tmp_settings - - ./build/mvn versions:set -DnewVersion=$version - ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy - } - - make_dist() { - dist_name="$1" - build_flags="$2" - file_name="spark-${version}-bin-${dist_name}.tgz" - sudo apt-get --assume-yes install r-base r-base-dev - ./dev/make-distribution.sh --name $dist_name --tgz $build_flags - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}" - } - - case $CIRCLE_NODE_INDEX in - 0) - publish_artifacts - ;; - 1) - make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive" - ;; - 2) - make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos" - ;; - esac - : - parallel: true - timeout: 1200 + - ./dev/run-tests deployment: release: tag: /[0-9]+(?:\.[0-9]+)+-palantir[0-9]+/ commands: - - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish + - ? | + set -euo pipefail + version=$(git describe --tags) + + make_dist() { + dist_name="$1" + build_flags="$2" + file_name="spark-${version}-bin-${dist_name}.tgz" + sudo apt-get --assume-yes install r-base r-base-dev + ./dev/make-distribution.sh --name $dist_name --tgz $build_flags + curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}" + } + + publish_artifacts() { + tmp_settings="tmp-settings.xml" + echo "" > $tmp_settings + echo "bintray-palantir-release$BINTRAY_USERNAME" >> $tmp_settings + echo "$BINTRAY_PASSWORD" >> $tmp_settings + echo "" >> $tmp_settings + + ./build/mvn versions:set -DnewVersion=$version + ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy + } + + make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive" + make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos" + + publish_artifacts + + curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish snapshot: branch: master commands: - - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish + - ? | + # same as above + set -euo pipefail + version=$(git describe --tags) + + make_dist() { + dist_name="$1" + build_flags="$2" + file_name="spark-${version}-bin-${dist_name}.tgz" + sudo apt-get --assume-yes install r-base r-base-dev + ./dev/make-distribution.sh --name $dist_name --tgz $build_flags + curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}" + } + + publish_artifacts() { + tmp_settings="tmp-settings.xml" + echo "" > $tmp_settings + echo "bintray-palantir-release$BINTRAY_USERNAME" >> $tmp_settings + echo "$BINTRAY_PASSWORD" >> $tmp_settings + echo "" >> $tmp_settings + + ./build/mvn versions:set -DnewVersion=$version + ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy + } + + make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive" + make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos" + + publish_artifacts + + curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish diff --git a/dev/publish.sh b/dev/publish.sh deleted file mode 100755 index 0d170d950b428..0000000000000 --- a/dev/publish.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -FLAGS="-Psparkr -Phive -Phive-thriftserver -Pyarn -Pmesos" -case $CIRCLE_NODE_INDEX in -0) - ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive publish - ;; -1) - ./dev/make-distribution.sh --name without-hadoop --tgz "-Psparkr -Phadoop-provided -Pyarn -Pmesos" \ - 2>&1 > binary-release-without-hadoop.log - ;; -2) - ./dev/make-distribution.sh --name hadoop2.7 --tgz "-Phadoop2.7 $FLAGS" \ - 2>&1 > binary-release-hadoop2.7.log - ;; -esac From 8af80abdbb326e4f6aec3450a261bb0c8359148e Mon Sep 17 00:00:00 2001 From: Andrew Ash Date: Mon, 10 Oct 2016 21:14:46 -0700 Subject: [PATCH 3/6] Use hadoop2.7 profile for running tests --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 55abd7f82e6bc..3b971664e9efb 100644 --- a/circle.yml +++ b/circle.yml @@ -18,7 +18,7 @@ dependencies: test: override: - - ./dev/run-tests + - HADOOP_PROFILE="hadoop2.7" ./dev/run-tests deployment: release: From a3d7067a9c4463e6da4d5deabbc8e237886a693c Mon Sep 17 00:00:00 2001 From: Andrew Ash Date: Mon, 10 Oct 2016 21:25:06 -0700 Subject: [PATCH 4/6] Add timeout to make the prior commands list a map key --- circle.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/circle.yml b/circle.yml index 3b971664e9efb..56d320d5bce72 100644 --- a/circle.yml +++ b/circle.yml @@ -54,6 +54,9 @@ deployment: publish_artifacts curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish + : + - timeout: 1200 + snapshot: branch: master commands: @@ -88,3 +91,5 @@ deployment: publish_artifacts curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish + : + - timeout: 1200 From fa7f5a56f73ff4631c0442dcba7473835d414dcc Mon Sep 17 00:00:00 2001 From: Andrew Ash Date: Mon, 10 Oct 2016 21:32:54 -0700 Subject: [PATCH 5/6] Move publishing out to a shell script (that must be Apache licensed to past RAT checks) --- circle.yml | 67 ++--------------------------------------- dev/publish-palantir.sh | 48 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 65 deletions(-) create mode 100755 dev/publish-palantir.sh diff --git a/circle.yml b/circle.yml index 56d320d5bce72..73b5ecdfb88d5 100644 --- a/circle.yml +++ b/circle.yml @@ -24,72 +24,9 @@ deployment: release: tag: /[0-9]+(?:\.[0-9]+)+-palantir[0-9]+/ commands: - - ? | - set -euo pipefail - version=$(git describe --tags) - - make_dist() { - dist_name="$1" - build_flags="$2" - file_name="spark-${version}-bin-${dist_name}.tgz" - sudo apt-get --assume-yes install r-base r-base-dev - ./dev/make-distribution.sh --name $dist_name --tgz $build_flags - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}" - } - - publish_artifacts() { - tmp_settings="tmp-settings.xml" - echo "" > $tmp_settings - echo "bintray-palantir-release$BINTRAY_USERNAME" >> $tmp_settings - echo "$BINTRAY_PASSWORD" >> $tmp_settings - echo "" >> $tmp_settings - - ./build/mvn versions:set -DnewVersion=$version - ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy - } - - make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive" - make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos" - - publish_artifacts - - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish - : - - timeout: 1200 + - ./dev/publish-palantir.sh snapshot: branch: master commands: - - ? | - # same as above - set -euo pipefail - version=$(git describe --tags) - - make_dist() { - dist_name="$1" - build_flags="$2" - file_name="spark-${version}-bin-${dist_name}.tgz" - sudo apt-get --assume-yes install r-base r-base-dev - ./dev/make-distribution.sh --name $dist_name --tgz $build_flags - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}" - } - - publish_artifacts() { - tmp_settings="tmp-settings.xml" - echo "" > $tmp_settings - echo "bintray-palantir-release$BINTRAY_USERNAME" >> $tmp_settings - echo "$BINTRAY_PASSWORD" >> $tmp_settings - echo "" >> $tmp_settings - - ./build/mvn versions:set -DnewVersion=$version - ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy - } - - make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive" - make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos" - - publish_artifacts - - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish - : - - timeout: 1200 + - ./dev/publish-palantir.sh diff --git a/dev/publish-palantir.sh b/dev/publish-palantir.sh new file mode 100755 index 0000000000000..bf790305cd3bb --- /dev/null +++ b/dev/publish-palantir.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +version=$(git describe --tags) + +make_dist() { + dist_name="$1" + build_flags="$2" + file_name="spark-${version}-bin-${dist_name}.tgz" + sudo apt-get --assume-yes install r-base r-base-dev + ./dev/make-distribution.sh --name $dist_name --tgz $build_flags + curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}" +} + +publish_artifacts() { + tmp_settings="tmp-settings.xml" + echo "" > $tmp_settings + echo "bintray-palantir-release$BINTRAY_USERNAME" >> $tmp_settings + echo "$BINTRAY_PASSWORD" >> $tmp_settings + echo "" >> $tmp_settings + + ./build/mvn versions:set -DnewVersion=$version + ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy +} + +make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive" +make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos" + +publish_artifacts + +curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish From 9df15ef36ed63559326846a27e5b8e89032f14fa Mon Sep 17 00:00:00 2001 From: Andrew Ash Date: Mon, 10 Oct 2016 21:44:36 -0700 Subject: [PATCH 6/6] Update expected deps to match new palantir version of parquet --- dev/deps/spark-deps-hadoop-2.7 | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 9008aa80bc877..d6a2b35d36106 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -55,6 +55,7 @@ datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar derby-10.12.1.1.jar eigenbase-properties-1.1.5.jar +fastutil-6.5.7.jar gson-2.2.4.jar guava-14.0.1.jar guice-3.0.jar @@ -146,13 +147,13 @@ opencsv-2.3.jar oro-2.0.8.jar osgi-resource-locator-1.0.1.jar paranamer-2.3.jar -parquet-column-1.8.1.jar -parquet-common-1.8.1.jar -parquet-encoding-1.8.1.jar -parquet-format-2.3.0-incubating.jar -parquet-hadoop-1.8.1.jar +parquet-column-1.8.1-palantir7.jar +parquet-common-1.8.1-palantir7.jar +parquet-encoding-1.8.1-palantir7.jar +parquet-format-2.3.1.jar +parquet-hadoop-1.8.1-palantir7.jar parquet-hadoop-bundle-1.6.0.jar -parquet-jackson-1.8.1.jar +parquet-jackson-1.8.1-palantir7.jar pmml-model-1.2.15.jar pmml-schema-1.2.15.jar protobuf-java-2.5.0.jar