[HUDI-3876] Fixing fetching partitions in GlueSyncClient (#5318)

apache · Apr 14, 2022 · a081c2b · a081c2b
1 parent 571cbe4
commit a081c2b
Showing 1 changed file with 13 additions and 7 deletions.
diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -94,13 +94,19 @@ public AWSGlueCatalogSyncClient(HiveSyncConfig syncConfig, Configuration hadoopC
   @Override
   public List<Partition> getAllPartitions(String tableName) {
     try {
-      GetPartitionsRequest request = new GetPartitionsRequest();
-      request.withDatabaseName(databaseName).withTableName(tableName);
-      GetPartitionsResult result = awsGlue.getPartitions(request);
-      return result.getPartitions()
-          .stream()
-          .map(p -> new Partition(p.getValues(), p.getStorageDescriptor().getLocation()))
-          .collect(Collectors.toList());
+      List<Partition> partitions = new ArrayList<>();
+      String nextToken = null;
+      do {
+        GetPartitionsResult result = awsGlue.getPartitions(new GetPartitionsRequest()
+            .withDatabaseName(databaseName)
+            .withTableName(tableName)
+            .withNextToken(nextToken));
+        partitions.addAll(result.getPartitions().stream()
+            .map(p -> new Partition(p.getValues(), p.getStorageDescriptor().getLocation()))
+            .collect(Collectors.toList()));
+        nextToken = result.getNextToken();
+      } while (nextToken != null);
+      return partitions;
     } catch (Exception e) {
       throw new HoodieGlueSyncException("Failed to get all partitions for table " + tableId(databaseName, tableName), e);
     }