Resolved comments

apache · Jul 6, 2021 · d81b3d3 · d81b3d3
1 parent 048d222
commit d81b3d3
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
@@ -204,15 +204,20 @@ def test_infer_nested_schema(self):
         df = self.spark.createDataFrame(rdd)
         self.assertEqual(Row(field1=1, field2=u'row1'), df.first())
 
-    def test_infer_nested_dict(self):
+    def test_infer_nested_dict_as_struct(self):
         # SPARK-35929: Test inferring nested dict as a struct type.
         NestedRow = Row("f1", "f2")
 
         with self.sql_conf({"spark.sql.pyspark.inferNestedDictAsStruct.enabled": True}):
             nestedRdd = self.sc.parallelize([NestedRow([{"payment": 200.5, "name": "A"}], [1, 2]),
                                              NestedRow([{"payment": 100.5, "name": "B"}], [2, 3])])
             df = self.spark.createDataFrame(nestedRdd)
-            self.assertEqual(Row(f1=[Row(payment=200.5, name='A')], f2=[1, 2]), df.collect()[0])
+            self.assertEqual(Row(f1=[Row(payment=200.5, name='A')], f2=[1, 2]), df.first())
+
+            data = [NestedRow([{"payment": 200.5, "name": "A"}], [1, 2]),
+                    NestedRow([{"payment": 100.5, "name": "B"}], [2, 3])]
+            df = self.spark.createDataFrame(data)
+            self.assertEqual(Row(f1=[Row(payment=200.5, name='A')], f2=[1, 2]), df.first())
 
     def test_create_dataframe_from_dict_respects_schema(self):
         df = self.spark.createDataFrame([{'a': 1}], ["b"])

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -3328,7 +3328,8 @@ object SQLConf {
     .createWithDefault(0)
 
   val INFER_NESTED_DICT_AS_STRUCT = buildConf("spark.sql.pyspark.inferNestedDictAsStruct.enabled")
-    .doc("When set to true, infers the nested dict as a struct. By default, it infers it as a map")
+    .doc("When set to true, infers the nested dict as a struct. By default, it infers it as a map " +
+      "when using SparkSession.createDataFrame.")
     .version("3.3.0")
     .booleanConf
     .createWithDefault(false)