Implement a Docker image generator gradle plugin for Spark applications

.circleci/config.yml

@@ -163,6 +163,11 @@ jobs:
       - save_cache:
           key: build-maven-{{ .Branch }}-{{ .BuildNum }}
           paths: .
+  build-spark-docker-gradle-plugin:
+    <<: *defaults
+    steps:
+      - *checkout-code
+      - run: dev/build-spark-docker-gradle-plugin.sh
 
   run-style-tests:
     # depends only on build-maven
@@ -252,6 +257,15 @@ jobs:
             - 'examples/target/scala-*/jars'
             - 'external/*/target/scala-*/*.jar'
 
+  run-spark-docker-gradle-plugin-tests:
+    <<: *test-defaults
+    resource_class: small
+    steps:
+      - *checkout-code
+      - setup_remote_docker
+      - run: |
+          dev/run-spark-docker-gradle-plugin-tests.sh | tee /tmp/run-spark-docker-gradle-plugin-tests.log
+
   run-backcompat-tests:
     # depends on build-sbt
     <<: *defaults
@@ -471,6 +485,8 @@ workflows:
           <<: *all-branches-and-tags
       - build-sbt:
           <<: *all-branches-and-tags
+      - build-spark-docker-gradle-plugin:
+          <<: *all-branches-and-tags
       - run-backcompat-tests:
           requires:
             - build-sbt
@@ -479,6 +495,10 @@ workflows:
           requires:
             - build-sbt
           <<: *all-branches-and-tags
+      - run-spark-docker-gradle-plugin-tests:
+          requires:
+            - build-spark-docker-gradle-plugin
+          <<: *all-branches-and-tags
       - run-python-tests:
           requires:
             - build-sbt

.gitignore

@@ -14,6 +14,7 @@
 .ensime_cache/
 .ensime_lucene
 .generated-mima*
+.gradle
 .idea/
 .idea_modules/
 .project

dev/build-spark-docker-gradle-plugin.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+ROOT=$(git rev-parse --show-toplevel)
+DOCKER_PLUGIN_PROJECT_DIR=$ROOT/resource-managers/kubernetes/docker
+$DOCKER_PLUGIN_PROJECT_DIR/gradlew -p $DOCKER_PLUGIN_PROJECT_DIR --info compileJava compileTestJava

dev/publish_functions.sh

@@ -22,6 +22,8 @@ publish_artifacts() {
   echo "</server></servers></settings>" >> $tmp_settings
 
   ./build/mvn -T 1C --settings $tmp_settings -DskipTests "${PALANTIR_FLAGS[@]}" deploy
+  DOCKER_PLUGIN_PROJECT_DIR=resource-managers/kubernetes/docker
+  $DOCKER_PLUGIN_PROJECT_DIR/gradlew -p $DOCKER_PLUGIN_PROJECT_DIR --info bintrayUpload
 }
 
 make_dist() {

dev/run-spark-docker-gradle-plugin-tests.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euo pipefail
+ROOT=$(git rev-parse --show-toplevel)
+DOCKER_PLUGIN_PROJECT_DIR=$ROOT/resource-managers/kubernetes/docker
+$DOCKER_PLUGIN_PROJECT_DIR/gradlew -p $DOCKER_PLUGIN_PROJECT_DIR --info test

resource-managers/kubernetes/docker/.gitignore

@@ -0,0 +1,4 @@
+src/main/resources/docker-resources
+.gradle
+build
+

resource-managers/kubernetes/docker/build.gradle

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.charset.StandardCharsets
+
+buildscript {
+  repositories {
+    jcenter()
+    maven { url "http://palantir.bintray.com/releases" }
+  }
+
+  dependencies {
+    classpath 'gradle.plugin.com.palantir:gradle-circle-style:1.1.2'
+    classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.7.3'
+    classpath 'com.netflix.nebula:nebula-dependency-recommender:5.2.0'
+    classpath 'com.netflix.nebula:nebula-publishing-plugin:5.1.5'
+  }
+}
+
+plugins {
+  id 'com.palantir.git-version' version '0.9.1'
+  id 'java-gradle-plugin'
+}
+
+repositories {
+  jcenter()
+  maven { url "http://palantir.bintray.com/releases" }
+}
+
+apply plugin: 'java'
+apply plugin: 'idea'
+apply plugin: 'nebula.dependency-recommender'
+version System.env.CIRCLE_TAG ?: gitVersion()
+group 'org.apache.spark'
+
+sourceCompatibility = 1.8
+
+dependencyRecommendations {
+  strategy OverrideTransitives
+  propertiesFile file: project.rootProject.file('versions.props')
+}
+
+test {
+  minHeapSize = "512m"
+  maxHeapSize = "512m"
+}
+
+dependencies {
+  compileOnly gradleApi()
+  compile 'org.apache.commons:commons-compress'
+  compile 'commons-io:commons-io'
+  testCompile 'org.assertj:assertj-core'
+  testCompile 'org.mockito:mockito-core'
+  testCompile 'junit:junit'
+  testCompile 'com.spotify:docker-client'
+  testCompile gradleApi()
+  testCompile gradleTestKit()
+}
+
+def getGitRepoRoot = { ->
+  def bashStdOut = new ByteArrayOutputStream()
+  exec {
+    commandLine 'git', 'rev-parse', '--show-toplevel'
+    standardOutput = bashStdOut
+  }
+  return new String(bashStdOut.toByteArray(), StandardCharsets.UTF_8).trim()
+}
+
+
+task prepareDockerBundleDir(type: Sync) {
+  from('src/main/dockerfiles/spark/Dockerfile') {
+    into 'kubernetes/dockerfiles/spark'
+    rename 'Dockerfile', 'Dockerfile.original'
+  }
+
+  from('src/main/dockerfiles/spark/entrypoint.sh') {
+    into 'kubernetes/dockerfiles/spark'
+  }
+
+  from(fileTree("$getGitRepoRoot/bin")) {
+    into 'bin'
+  }
+
+  from(fileTree("$getGitRepoRoot/sbin")) {
+    into 'sbin'
+  }
+
+  into file("src/main/resources/docker-resources")
+  includeEmptyDirs = false
+}
+
+tasks.compileJava.dependsOn tasks.prepareDockerBundleDir
+tasks.idea.dependsOn tasks.prepareDockerBundleDir
+
+apply from: "${rootDir}/gradle/publish.gradle"

resource-managers/kubernetes/docker/gradle/publish.gradle

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'com.jfrog.bintray'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-resolved-dependencies'
+apply plugin: 'nebula.javadoc-jar'
+apply plugin: 'nebula.source-jar'
+
+jar {
+  manifest {
+    attributes(
+        "Implementation-Title"   : project.name,
+        "Implementation-Version" : project.version,
+        "Implementation-Vendor"  : "Palantir Technologies Inc.")
+  }
+}
+
+bintray {
+  user = System.env.BINTRAY_USERNAME
+  key = System.env.BINTRAY_PASSWORD
+  publish = true
+  pkg {
+    repo = 'releases'
+    name = 'spark'
+    userOrg = 'palantir'
+    licenses = ['Apache-2.0']
+    publications = ['nebula']
+  }
+}
+
+bintrayUpload.dependsOn { generatePomFileForNebulaPublication }
+bintrayUpload.dependsOn { sourceJar }
+bintrayUpload.dependsOn { build }
+
+publishing {
+  publications {
+    nebula(MavenPublication) {
+      from components.java
+    }
+  }
+}

resource-managers/kubernetes/docker/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists