Merge remote-tracking branch 'upstream/master' into skip_transform

.github/workflows/benchmark.yml

@@ -47,6 +47,7 @@ jobs:
       SPARK_BENCHMARK_NUM_SPLITS: ${{ github.event.inputs.num-splits }}
       SPARK_BENCHMARK_CUR_SPLIT: ${{ matrix.split }}
       SPARK_GENERATE_BENCHMARK_FILES: 1
+      SPARK_LOCAL_IP: localhost
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2

.github/workflows/build_and_test.yml

@@ -83,6 +83,7 @@ jobs:
       CONDA_PREFIX: /usr/share/miniconda
       GITHUB_PREV_SHA: ${{ github.event.before }}
       GITHUB_INPUT_BRANCH: ${{ github.event.inputs.target }}
+      SPARK_LOCAL_IP: localhost
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
@@ -171,6 +172,7 @@ jobs:
       CONDA_PREFIX: /usr/share/miniconda
       GITHUB_PREV_SHA: ${{ github.event.before }}
       GITHUB_INPUT_BRANCH: ${{ github.event.inputs.target }}
+      SPARK_LOCAL_IP: localhost
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
@@ -238,6 +240,7 @@ jobs:
       HIVE_PROFILE: hive2.3
       GITHUB_PREV_SHA: ${{ github.event.before }}
       GITHUB_INPUT_BRANCH: ${{ github.event.inputs.target }}
+      SPARK_LOCAL_IP: localhost
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
@@ -468,6 +471,8 @@ jobs:
   tpcds-1g:
     name: Run TPC-DS queries with SF=1
     runs-on: ubuntu-20.04
+    env:
+      SPARK_LOCAL_IP: localhost
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2

core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala

@@ -418,12 +418,7 @@ class MasterSuite extends SparkFunSuite
           (workerResponse \ "masterwebuiurl").extract[String] should be (reverseProxyUrl + "/")
         }
 
-        // with LocalCluster, we have masters and workers in the same JVM, each overwriting
-        // system property spark.ui.proxyBase.
-        // so we need to manage this property explicitly for test
-        System.getProperty("spark.ui.proxyBase") should startWith
-          (s"$reverseProxyUrl/proxy/worker-")
-        System.setProperty("spark.ui.proxyBase", reverseProxyUrl)
+        System.getProperty("spark.ui.proxyBase") should be (reverseProxyUrl)
         val html = Utils
           .tryWithResource(Source.fromURL(s"$masterUrl/"))(_.getLines().mkString("\n"))
         html should include ("Spark Master at spark://")

dev/sparktestsupport/modules.py

@@ -610,6 +610,8 @@ def __hash__(self):
         "pyspark.pandas.spark.accessors",
         "pyspark.pandas.spark.utils",
         "pyspark.pandas.typedef.typehints",
+        # unittests
+        "pyspark.pandas.tests.test_dataframe",
     ],
     excluded_python_implementations=[
         "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and

external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala

@@ -134,4 +134,7 @@ private[avro] case class AvroDataToCatalyst(
       """
     })
   }
+
+  override protected def withNewChildInternal(newChild: Expression): AvroDataToCatalyst =
+    copy(child = newChild)
 }

external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala

@@ -64,4 +64,7 @@ private[avro] case class CatalystDataToAvro(
     defineCodeGen(ctx, ev, input =>
       s"(byte[]) $expr.nullSafeEval($input)")
   }
+
+  override protected def withNewChildInternal(newChild: Expression): CatalystDataToAvro =
+    copy(child = newChild)
 }

mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala

@@ -374,6 +374,10 @@ private[spark] object SummaryBuilderImpl extends Logging {
     override def left: Expression = featuresExpr
     override def right: Expression = weightExpr
 
+    override protected def withNewChildrenInternal(
+        newLeft: Expression, newRight: Expression): MetricsAggregate =
+      copy(featuresExpr = newLeft, weightExpr = newRight)
+
     override def update(state: SummarizerBuffer, row: InternalRow): SummarizerBuffer = {
       val features = vectorUDT.deserialize(featuresExpr.eval(row))
       val weight = weightExpr.eval(row).asInstanceOf[Double]

pom.xml

@@ -138,7 +138,7 @@
     <derby.version>10.14.2.0</derby.version>
     <parquet.version>1.12.0</parquet.version>
     <orc.version>1.6.7</orc.version>
-    <jetty.version>9.4.37.v20210219</jetty.version>
+    <jetty.version>9.4.39.v20210325</jetty.version>
     <jakartaservlet.version>4.0.3</jakartaservlet.version>
     <chill.version>0.9.5</chill.version>
     <ivy.version>2.4.0</ivy.version>

python/pyspark/pandas/testing/__init__.py

@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#