diff --git a/assembly/pom.xml b/assembly/pom.xml
index e187ebf04b726..ba94bedc2f956 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index e5aa9d3adf52e..46cbeeac6f7b2 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 34fc234d78b21..2074cd4907ced 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index f234eadbd4283..29c1f8d61f70a 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index deb8a8662e37d..22adc79d658b9 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 98137b87fc746..0badd26ed838d 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 46c009f11330a..b13be60387402 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 298b4c9d3942e..fa9ab669e81ca 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/core/pom.xml b/core/pom.xml
index 3f5a54850825d..494f156a1b54d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/core/src/main/java/org/apache/spark/log4j/LocalFileAppender.java b/core/src/main/java/org/apache/spark/log4j/LocalFileAppender.java
new file mode 100644
index 0000000000000..9ee66440e758b
--- /dev/null
+++ b/core/src/main/java/org/apache/spark/log4j/LocalFileAppender.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.log4j;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.UUID;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.Layout;
+import org.apache.log4j.helpers.LogLog;
+import org.apache.spark.SparkEnv;
+
+import com.google.common.annotations.VisibleForTesting;
+
+public class LocalFileAppender extends FileAppender {
+ private String metadataIdentifier;
+ private String category;
+ private String identifier;
+ private String jobName;
+ private String project;
+ private String executorId;
+ private String mountDir;
+
+ private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+
+ public LocalFileAppender() {
+ super();
+ }
+
+ public LocalFileAppender(Layout layout, String filename, boolean append) throws IOException {
+ super(layout, filename, append);
+ }
+
+ public LocalFileAppender(Layout layout, String filename) throws IOException {
+ super(layout, filename);
+ }
+
+ @Override
+ public void activateOptions() {
+ LogLog.warn(String.format("%s starting ...", name));
+ this.fileName = getOutPutDir();
+ LogLog.warn(String.format("Output path is %s", this.fileName));
+ LogLog.warn("metadataIdentifier -> " + getMetadataIdentifier());
+ LogLog.warn("category -> " + getCategory());
+ LogLog.warn("identifier -> " + getIdentifier());
+ LogLog.warn("mountDir -> " + getMountDir());
+
+ if (null != getProject()) {
+ LogLog.warn("project -> " + getProject());
+ }
+
+ if (null != getJobName()) {
+ LogLog.warn("jobName -> " + getJobName());
+ }
+ super.activateOptions();
+ }
+
+ @VisibleForTesting
+ String getOutPutDir() {
+ String rollingDir = dateFormat.format(new Date());
+ if (StringUtils.isBlank(executorId)) {
+ executorId = SparkEnv.get() != null ? SparkEnv.get().executorId() : UUID.randomUUID().toString();
+ LogLog.warn("executorId set to " + executorId);
+ }
+
+ if ("job".equals(getCategory())) {
+ return getRootPathName() + "/" + rollingDir +
+ "/" + getIdentifier() + "/" + getJobName() + "/" + "executor-"
+ + executorId + ".log";
+ }
+ return getRootPathName() + "/" + rollingDir
+ + "/" + getIdentifier() + "/" + "executor-" + executorId + ".log";
+ }
+
+ String getRootPathName() {
+ if (!mountDir.endsWith("/")) {
+ mountDir = mountDir + "/";
+ }
+ if ("job".equals(getCategory())) {
+ return mountDir + getProject() + "/spark_logs";
+ } else if ("sparder".equals(getCategory())) {
+ return mountDir + "_sparder_logs";
+ } else {
+ throw new IllegalArgumentException("illegal category: " + getCategory());
+ }
+ }
+
+
+ public String getMetadataIdentifier() {
+ return metadataIdentifier;
+ }
+
+ public void setMetadataIdentifier(String metadataIdentifier) {
+ this.metadataIdentifier = metadataIdentifier;
+ }
+
+ public String getCategory() {
+ return category;
+ }
+
+ public void setCategory(String category) {
+ this.category = category;
+ }
+
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ public void setIdentifier(String identifier) {
+ this.identifier = identifier;
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ public void setJobName(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public String getProject() {
+ return project;
+ }
+
+ public void setProject(String project) {
+ this.project = project;
+ }
+
+ public String getExecutorId() {
+ return executorId;
+ }
+
+ public void setExecutorId(String executorId) {
+ this.executorId = executorId;
+ }
+
+ public String getMountDir() {
+ return mountDir;
+ }
+
+ public void setMountDir(String mountDir) {
+ this.mountDir = mountDir;
+ }
+
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/spark/log4j/SensitivePatternLayout.java b/core/src/main/java/org/apache/spark/log4j/SensitivePatternLayout.java
new file mode 100644
index 0000000000000..8ebf63d42f74d
--- /dev/null
+++ b/core/src/main/java/org/apache/spark/log4j/SensitivePatternLayout.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.log4j;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.apache.log4j.spi.LoggingEvent;
+
+public class SensitivePatternLayout extends PatternLayout {
+ private static final String PREFIX_GROUP_NAME = "prefix";
+ private static final String SENSITIVE_GROUP_NAME = "sensitive";
+ private static final String MASK = "******";
+ private static final Pattern SENSITIVE_PATTERN = Pattern.compile(
+ String.format("(?<%s>password\\s*[:=])(?<%s>[^,.!]*)", PREFIX_GROUP_NAME, SENSITIVE_GROUP_NAME),
+ Pattern.CASE_INSENSITIVE);
+
+ @Override
+ public String format(LoggingEvent event) {
+ if (event.getMessage() instanceof String) {
+ String maskedMessage = mask(event.getRenderedMessage());
+
+ Throwable throwable = event.getThrowableInformation() != null
+ ? event.getThrowableInformation().getThrowable()
+ : null;
+ LoggingEvent maskedEvent = new LoggingEvent(event.fqnOfCategoryClass,
+ Logger.getLogger(event.getLoggerName()), event.timeStamp, event.getLevel(), maskedMessage,
+ throwable);
+
+ return super.format(maskedEvent);
+ }
+ return super.format(event);
+ }
+
+ private String mask(String message) {
+ Matcher matcher = SENSITIVE_PATTERN.matcher(message);
+ if (matcher.find()) {
+ return matcher.replaceAll(String.format("${%s}%s", PREFIX_GROUP_NAME, MASK));
+ }
+ return message;
+ }
+}
diff --git a/examples/pom.xml b/examples/pom.xml
index 6bda2112caa26..71883263a9bd5 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 7fc71fe095ee7..3ffa631359e2b 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 75c874f9676d2..4fa5605e94cfa 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml
index 38ca15f3c5c9a..11aab1b798356 100644
--- a/external/flume-assembly/pom.xml
+++ b/external/flume-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
index 792ee81993b57..0e8df5502c65f 100644
--- a/external/flume-sink/pom.xml
+++ b/external/flume-sink/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/flume/pom.xml b/external/flume/pom.xml
index cd141f59bde7d..41764887cbdc0 100644
--- a/external/flume/pom.xml
+++ b/external/flume/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 1395b99d761a6..a052a2a95158e 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 01da3a3701f01..0d3d5f5335ada 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 9d10d7f925896..cb399d85ab45f 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml
index 4e605a64b173b..2453f0f29186a 100644
--- a/external/kafka-0-8-assembly/pom.xml
+++ b/external/kafka-0-8-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml
index 213aeef64e275..93faf7c2e6ff8 100644
--- a/external/kafka-0-8/pom.xml
+++ b/external/kafka-0-8/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index e5d9d332ce0ca..00fdc93407c12 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 4e71668a57725..57833e8f22874 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index 6e132513b180b..8e2cc609a26e0 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/graphx/pom.xml b/graphx/pom.xml
index d2b00500d0872..e9e88ce3d7b0b 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index e56a1c7c261c4..094b9b4d78a6b 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 7cc248da68f96..d63537e8a8db3 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index f05999f62630e..58f8b1243b0d4 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 7b5b877f18ac8..7336ab30e43d9 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/pom.xml b/pom.xml
index a8348d0c17306..eb1c721120020 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
pom
Spark Project Parent POM
http://spark.apache.org/
diff --git a/repl/pom.xml b/repl/pom.xml
index 4f0665604a483..727d4bdec9d1a 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 41be9fb066dde..5a31b73319790 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../../pom.xml
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index f63f8a9d9d139..d4be3c91b46e6 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../../pom.xml
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 3f7d626c65361..40a459689a8c2 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index dd765615a71a7..d68a2d21c805a 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index b7dd19acb8bf5..40265c4dbf1c4 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 22d6c79b13507..b5a2d7bf1d3be 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index be9f1006eca26..32c8bda1e92d4 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 9d217ea91b68c..bfb390de9ce84 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../../pom.xml
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 021d9a3c92222..c29c3d11e276f 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml
diff --git a/tools/pom.xml b/tools/pom.xml
index 3ed780a9fe5ed..1b315cfeb424d 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.1-kylin-r34
+ 2.4.1-kylin-r35
../pom.xml