Added limit parameter to the DU

apache · Apr 10, 2024 · def81a0 · def81a0
1 parent 1d5a90f
commit def81a0
Show file tree

Hide file tree

Showing 10 changed files with 26 additions and 25 deletions.
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
@@ -40,7 +40,7 @@ private ReconConstants() {
   /**
    * The maximum number of top disk usage records to return in a /du response.
    */
-  public static final int DISK_USAGE_TOP_RECORDS_LIMIT = 30;
+  public static final String DEFAULT_DISK_USAGE_TOP_RECORDS_LIMIT = "30";
   public static final String DEFAULT_OPEN_KEY_INCLUDE_NON_FSO = "false";
   public static final String DEFAULT_OPEN_KEY_INCLUDE_FSO = "false";
   public static final String DEFAULT_FETCH_COUNT = "1000";

diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java
@@ -40,6 +40,8 @@
 import javax.ws.rs.core.Response;
 import java.io.IOException;
 
+import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_DISK_USAGE_TOP_RECORDS_LIMIT;
+
 /**
  * REST APIs for namespace metadata summary.
  */
@@ -112,7 +114,8 @@ public Response getBasicInfo(
   public Response getDiskUsage(@QueryParam("path") String path,
                                @DefaultValue("false") @QueryParam("files") boolean listFile,
                                @DefaultValue("false") @QueryParam("replica") boolean withReplica,
-                               @DefaultValue("true") @QueryParam("sortSubPaths") boolean sortSubpaths)
+                               @DefaultValue("true") @QueryParam("sortSubPaths") boolean sortSubpaths,
+                               @DefaultValue(DEFAULT_DISK_USAGE_TOP_RECORDS_LIMIT) @QueryParam("limit") int limit)
       throws IOException {
     if (path == null || path.length() == 0) {
       return Response.status(Response.Status.BAD_REQUEST).build();
@@ -125,10 +128,10 @@ public Response getDiskUsage(@QueryParam("path") String path,
     }
 
     EntityHandler handler = EntityHandler.getEntityHandler(
-            reconNamespaceSummaryManager,
-            omMetadataManager, reconSCM, path);
+        reconNamespaceSummaryManager,
+        omMetadataManager, reconSCM, path);
 
-    duResponse = handler.getDuResponse(listFile, withReplica, sortSubpaths);
+    duResponse = handler.getDuResponse(listFile, withReplica, sortSubpaths, limit);
 
     return Response.ok(duResponse).build();
   }

diff --git a/...e/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/...e/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java
@@ -36,7 +36,7 @@
 import java.util.List;
 import java.util.Set;
 
-import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT;
+import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_DISK_USAGE_TOP_RECORDS_LIMIT;
 import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit;
 
 /**
@@ -90,7 +90,7 @@ private BucketObjectDBInfo getBucketObjDbInfo(String[] names)
 
   @Override
   public DUResponse getDuResponse(
-      boolean listFile, boolean withReplica, boolean sortSubpaths)
+      boolean listFile, boolean withReplica, boolean sortSubpaths, int limit)
           throws IOException {
     DUResponse duResponse = new DUResponse();
     duResponse.setPath(getNormalizedPath());
@@ -149,7 +149,7 @@ public DUResponse getDuResponse(
     if (sortSubpaths) {
       // Parallel sort directory/files DU data in descending order of size and returns the top N elements.
       dirDUData = sortDiskUsageDescendingWithLimit(dirDUData,
-          DISK_USAGE_TOP_RECORDS_LIMIT);
+          limit);
     }
 
     duResponse.setDuData(dirDUData);

diff --git a/...econ/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/...econ/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java
@@ -39,7 +39,7 @@
 import java.util.List;
 import java.util.Set;
 
-import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT;
+import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_DISK_USAGE_TOP_RECORDS_LIMIT;
 import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit;
 
 /**
@@ -83,7 +83,7 @@ private ObjectDBInfo getDirectoryObjDbInfo(String[] names)
 
   @Override
   public DUResponse getDuResponse(
-      boolean listFile, boolean withReplica, boolean sortSubPaths)
+      boolean listFile, boolean withReplica, boolean sortSubPaths, int limit)
           throws IOException {
     DUResponse duResponse = new DUResponse();
     duResponse.setPath(getNormalizedPath());
@@ -161,7 +161,7 @@ public DUResponse getDuResponse(
     if (sortSubPaths) {
       // Parallel sort subdirDUData in descending order of size and returns the top N elements.
       subdirDUData = sortDiskUsageDescendingWithLimit(subdirDUData,
-          DISK_USAGE_TOP_RECORDS_LIMIT);
+          limit);
     }
 
     duResponse.setDuData(subdirDUData);

diff --git a/...p-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java b/...p-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java
@@ -79,7 +79,7 @@ public abstract NamespaceSummaryResponse getSummaryResponse()
           throws IOException;
 
   public abstract DUResponse getDuResponse(
-      boolean listFile, boolean withReplica, boolean sort)
+      boolean listFile, boolean withReplica, boolean sort, int limit)
           throws IOException;
 
   public abstract QuotaUsageResponse getQuotaResponse()

diff --git a/...zone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java b/...zone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java
@@ -71,7 +71,7 @@ private ObjectDBInfo getKeyDbObjectInfo(String[] names)
 
   @Override
   public DUResponse getDuResponse(
-      boolean listFile, boolean withReplica, boolean sort)
+      boolean listFile, boolean withReplica, boolean sort, int limit)
           throws IOException {
     DUResponse duResponse = new DUResponse();
     duResponse.setPath(getNormalizedPath());

diff --git a/...one/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/...one/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java
@@ -39,7 +39,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME_DEFAULT;
+import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_DISK_USAGE_TOP_RECORDS_LIMIT;
 import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit;
 
 /**
@@ -91,7 +92,7 @@ private ObjectDBInfo getPrefixObjDbInfo()
 
   @Override
   public DUResponse getDuResponse(
-      boolean listFile, boolean withReplica, boolean sortSubPaths)
+      boolean listFile, boolean withReplica, boolean sortSubPaths, int limit)
           throws IOException {
     DUResponse duResponse = new DUResponse();
     duResponse.setPath(getNormalizedPath());
@@ -143,8 +144,7 @@ public DUResponse getDuResponse(
 
     if (sortSubPaths) {
       // Parallel sort volumeDuData in descending order of size and returns the top N elements.
-      volumeDuData = sortDiskUsageDescendingWithLimit(volumeDuData,
-          DISK_USAGE_TOP_RECORDS_LIMIT);
+      volumeDuData = sortDiskUsageDescendingWithLimit(volumeDuData, limit);
     }
 
     duResponse.setDuData(volumeDuData);
@@ -159,7 +159,7 @@ public QuotaUsageResponse getQuotaResponse()
     SCMNodeStat stats = getReconSCM().getScmNodeManager().getStats();
     long quotaInBytes = stats.getCapacity().get();
     long quotaUsedInBytes =
-        getDuResponse(true, true, false).getSizeWithReplica();
+        getDuResponse(true, true, false, OZONE_OM_USER_MAX_VOLUME_DEFAULT).getSizeWithReplica();
     quotaUsageResponse.setQuota(quotaInBytes);
     quotaUsageResponse.setQuotaUsed(quotaUsedInBytes);
     return quotaUsageResponse;

diff --git a/.../recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java b/.../recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java
@@ -51,7 +51,7 @@ public NamespaceSummaryResponse getSummaryResponse()
 
   @Override
   public DUResponse getDuResponse(
-      boolean listFile, boolean withReplica, boolean sort)
+      boolean listFile, boolean withReplica, boolean sort, int limit)
           throws IOException {
     DUResponse duResponse = new DUResponse();
     duResponse.setStatus(ResponseStatus.PATH_NOT_FOUND);

diff --git a/...e/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/...e/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java
@@ -36,8 +36,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-
-import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT;
 import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit;
 
 /**
@@ -96,7 +94,7 @@ private VolumeObjectDBInfo getVolumeObjDbInfo(String[] names)
 
   @Override
   public DUResponse getDuResponse(
-      boolean listFile, boolean withReplica, boolean sortSubPaths)
+      boolean listFile, boolean withReplica, boolean sortSubPaths, int limit)
           throws IOException {
     DUResponse duResponse = new DUResponse();
     duResponse.setPath(getNormalizedPath());
@@ -138,8 +136,7 @@ public DUResponse getDuResponse(
 
     if (sortSubPaths) {
       // Parallel sort bucketDuData in descending order of size and returns the top N elements.
-      bucketDuData = sortDiskUsageDescendingWithLimit(bucketDuData,
-          DISK_USAGE_TOP_RECORDS_LIMIT);
+      bucketDuData = sortDiskUsageDescendingWithLimit(bucketDuData, limit);
     }
 
     duResponse.setDuData(bucketDuData);

diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java
@@ -71,7 +71,8 @@ private long getEntitySize(String path) throws IOException {
         EntityHandler.getEntityHandler(reconNamespaceSummaryManager,
             omMetadataManager, reconSCM, path);
     if (null != entityHandler) {
-      DUResponse duResponse = entityHandler.getDuResponse(false, false, false);
+      DUResponse duResponse =
+          entityHandler.getDuResponse(false, false, false, 10000);
       if (null != duResponse && duResponse.getStatus() == ResponseStatus.OK) {
         return duResponse.getSize();
       }