[HUDI-3670] free temp views in sql transformers (#5080)

apache · Jun 1, 2022 · 7276d0e · 7276d0e
1 parent dfcd6d9
commit 7276d0e
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 5 deletions.
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java
@@ -49,8 +49,10 @@ public Dataset<Row> apply(JavaSparkContext jsc, SparkSession sparkSession, Datas
     // tmp table name doesn't like dashes
     String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_"));
     LOG.info("Registering tmp table : " + tmpTable);
-    rowDataset.registerTempTable(tmpTable);
-    return sparkSession.sql("select " + flattenSchema(rowDataset.schema(), null) + " from " + tmpTable);
+    rowDataset.createOrReplaceTempView(tmpTable);
+    Dataset<Row> transformed = sparkSession.sql("select " + flattenSchema(rowDataset.schema(), null) + " from " + tmpTable);
+    sparkSession.catalog().dropTempView(tmpTable);
+    return transformed;
   }
 
   public String flattenSchema(StructType schema, String prefix) {

diff --git a/...-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java b/...-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java
@@ -76,7 +76,7 @@ public Dataset<Row> apply(
     // tmp table name doesn't like dashes
     final String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_"));
     LOG.info("Registering tmp table : " + tmpTable);
-    rowDataset.registerTempTable(tmpTable);
+    rowDataset.createOrReplaceTempView(tmpTable);
 
     try (final Scanner scanner = new Scanner(fs.open(new Path(sqlFile)), "UTF-8")) {
       Dataset<Row> rows = null;
@@ -95,6 +95,8 @@ public Dataset<Row> apply(
       return rows;
     } catch (final IOException ioe) {
       throw new HoodieIOException("Error reading transformer SQL file.", ioe);
+    } finally {
+      sparkSession.catalog().dropTempView(tmpTable);
     }
   }
 

diff --git a/...utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java b/...utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java
@@ -60,9 +60,11 @@ public Dataset<Row> apply(JavaSparkContext jsc, SparkSession sparkSession, Datas
     // tmp table name doesn't like dashes
     String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_"));
     LOG.info("Registering tmp table : " + tmpTable);
-    rowDataset.registerTempTable(tmpTable);
+    rowDataset.createOrReplaceTempView(tmpTable);
     String sqlStr = transformerSQL.replaceAll(SRC_PATTERN, tmpTable);
     LOG.debug("SQL Query for transformation : (" + sqlStr + ")");
-    return sparkSession.sql(sqlStr);
+    Dataset<Row> transformed = sparkSession.sql(sqlStr);
+    sparkSession.catalog().dropTempView(tmpTable);
+    return transformed;
   }
 }