diff --git a/circle.yml b/circle.yml
index b8d1a36da76c9..73b5ecdfb88d5 100644
--- a/circle.yml
+++ b/circle.yml
@@ -14,55 +14,19 @@ checkout:
dependencies:
override:
- - ./build/mvn -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive dependency:go-offline
+ - ./build/mvn -Phadoop-2.7 -DskipTests -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive dependency:go-offline
test:
override:
- - ? |
- set -euo pipefail
- version=$(git describe --tags)
-
- publish_artifacts() {
- tmp_settings="tmp-settings.xml"
- echo "" > $tmp_settings
- echo "bintray-palantir-release$BINTRAY_USERNAME" >> $tmp_settings
- echo "$BINTRAY_PASSWORD" >> $tmp_settings
- echo "" >> $tmp_settings
-
- ./build/mvn versions:set -DnewVersion=$version
- ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy
- }
-
- make_dist() {
- dist_name="$1"
- build_flags="$2"
- file_name="spark-${version}-bin-${dist_name}.tgz"
- sudo apt-get --assume-yes install r-base r-base-dev
- ./dev/make-distribution.sh --name $dist_name --tgz $build_flags
- curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}"
- }
-
- case $CIRCLE_NODE_INDEX in
- 0)
- publish_artifacts
- ;;
- 1)
- make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive"
- ;;
- 2)
- make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos"
- ;;
- esac
- :
- parallel: true
- timeout: 1200
+ - HADOOP_PROFILE="hadoop2.7" ./dev/run-tests
deployment:
release:
tag: /[0-9]+(?:\.[0-9]+)+-palantir[0-9]+/
commands:
- - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish
+ - ./dev/publish-palantir.sh
+
snapshot:
branch: master
commands:
- - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish
+ - ./dev/publish-palantir.sh
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 9008aa80bc877..d6a2b35d36106 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -55,6 +55,7 @@ datanucleus-core-3.2.10.jar
datanucleus-rdbms-3.2.9.jar
derby-10.12.1.1.jar
eigenbase-properties-1.1.5.jar
+fastutil-6.5.7.jar
gson-2.2.4.jar
guava-14.0.1.jar
guice-3.0.jar
@@ -146,13 +147,13 @@ opencsv-2.3.jar
oro-2.0.8.jar
osgi-resource-locator-1.0.1.jar
paranamer-2.3.jar
-parquet-column-1.8.1.jar
-parquet-common-1.8.1.jar
-parquet-encoding-1.8.1.jar
-parquet-format-2.3.0-incubating.jar
-parquet-hadoop-1.8.1.jar
+parquet-column-1.8.1-palantir7.jar
+parquet-common-1.8.1-palantir7.jar
+parquet-encoding-1.8.1-palantir7.jar
+parquet-format-2.3.1.jar
+parquet-hadoop-1.8.1-palantir7.jar
parquet-hadoop-bundle-1.6.0.jar
-parquet-jackson-1.8.1.jar
+parquet-jackson-1.8.1-palantir7.jar
pmml-model-1.2.15.jar
pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
diff --git a/dev/publish-palantir.sh b/dev/publish-palantir.sh
new file mode 100755
index 0000000000000..bf790305cd3bb
--- /dev/null
+++ b/dev/publish-palantir.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+version=$(git describe --tags)
+
+make_dist() {
+ dist_name="$1"
+ build_flags="$2"
+ file_name="spark-${version}-bin-${dist_name}.tgz"
+ sudo apt-get --assume-yes install r-base r-base-dev
+ ./dev/make-distribution.sh --name $dist_name --tgz $build_flags
+ curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}"
+}
+
+publish_artifacts() {
+ tmp_settings="tmp-settings.xml"
+ echo "" > $tmp_settings
+ echo "bintray-palantir-release$BINTRAY_USERNAME" >> $tmp_settings
+ echo "$BINTRAY_PASSWORD" >> $tmp_settings
+ echo "" >> $tmp_settings
+
+ ./build/mvn versions:set -DnewVersion=$version
+ ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy
+}
+
+make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive"
+make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos"
+
+publish_artifacts
+
+curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish
diff --git a/dev/publish.sh b/dev/publish.sh
deleted file mode 100755
index 0d170d950b428..0000000000000
--- a/dev/publish.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-FLAGS="-Psparkr -Phive -Phive-thriftserver -Pyarn -Pmesos"
-case $CIRCLE_NODE_INDEX in
-0)
- ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive publish
- ;;
-1)
- ./dev/make-distribution.sh --name without-hadoop --tgz "-Psparkr -Phadoop-provided -Pyarn -Pmesos" \
- 2>&1 > binary-release-without-hadoop.log
- ;;
-2)
- ./dev/make-distribution.sh --name hadoop2.7 --tgz "-Phadoop2.7 $FLAGS" \
- 2>&1 > binary-release-hadoop2.7.log
- ;;
-esac