diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 134092146e5..9c79a869c41 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -36,6 +36,11 @@ private ReconConstants() { public static final String RECON_SCM_SNAPSHOT_DB = "scm.snapshot.db"; // By default, limit the number of results returned + + /** + * The maximum number of top disk usage records to return in a /du response. + */ + public static final int DISK_USAGE_TOP_RECORDS_LIMIT = 30; public static final String DEFAULT_OPEN_KEY_INCLUDE_NON_FSO = "false"; public static final String DEFAULT_OPEN_KEY_INCLUDE_FSO = "false"; public static final String DEFAULT_FETCH_COUNT = "1000"; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 39d091ee03c..f154f024fbd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; +import java.util.stream.Collectors; import com.google.common.base.Preconditions; import com.google.inject.Singleton; @@ -59,6 +60,7 @@ import static org.jooq.impl.DSL.select; import static org.jooq.impl.DSL.using; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.scm.ReconContainerReportQueue; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; @@ -322,6 +324,33 @@ public static void upsertGlobalStatsTable(Configuration sqlConfiguration, } } + /** + * Sorts a list of DiskUsage objects in descending order by size using parallel sorting and + * returns the top N records as specified by the limit. + * + * This method is optimized for large datasets and utilizes parallel processing to efficiently + * sort and retrieve the top N largest records by size. It's especially useful for reducing + * processing time and memory usage when only a subset of sorted records is needed. + * + * Advantages of this approach include: + * - Efficient handling of large datasets by leveraging multi-core processors. + * - Reduction in memory usage and improvement in processing time by limiting the + * number of returned records. + * - Scalability and easy integration with existing systems. + * + * @param diskUsageList the list of DiskUsage objects to be sorted. + * @param limit the maximum number of DiskUsage objects to return. + * @return a list of the top N DiskUsage objects sorted in descending order by size, + * where N is the specified limit. + */ + public static List sortDiskUsageDescendingWithLimit( + List diskUsageList, int limit) { + return diskUsageList.parallelStream() + .sorted((du1, du2) -> Long.compare(du2.getSize(), du1.getSize())) + .limit(limit) + .collect(Collectors.toList()); + } + public static long getFileSizeUpperBound(long fileSize) { if (fileSize >= ReconConstants.MAX_FILE_SIZE_UPPER_BOUND) { return Long.MAX_VALUE; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java index 5b104c46115..71040b9fdf6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java @@ -101,6 +101,8 @@ public Response getBasicInfo( * @param path request path * @param listFile show subpath/disk usage for each key * @param withReplica count actual DU with replication + * @param sortSubpaths determines whether to sort the subpaths by their sizes in descending order + * and returns the N largest subpaths based on the configuration value DISK_USAGE_TOP_RECORDS_LIMIT. * @return DU response * @throws IOException */ @@ -108,10 +110,9 @@ public Response getBasicInfo( @Path("/du") @SuppressWarnings("methodlength") public Response getDiskUsage(@QueryParam("path") String path, - @DefaultValue("false") - @QueryParam("files") boolean listFile, - @DefaultValue("false") - @QueryParam("replica") boolean withReplica) + @DefaultValue("false") @QueryParam("files") boolean listFile, + @DefaultValue("false") @QueryParam("replica") boolean withReplica, + @DefaultValue("true") @QueryParam("sortSubPaths") boolean sortSubpaths) throws IOException { if (path == null || path.length() == 0) { return Response.status(Response.Status.BAD_REQUEST).build(); @@ -127,8 +128,7 @@ public Response getDiskUsage(@QueryParam("path") String path, reconNamespaceSummaryManager, omMetadataManager, reconSCM, path); - duResponse = handler.getDuResponse( - listFile, withReplica); + duResponse = handler.getDuResponse(listFile, withReplica, sortSubpaths); return Response.ok(duResponse).build(); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java index 7ad961195ee..00cd9617b5d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java @@ -36,6 +36,9 @@ import java.util.List; import java.util.Set; +import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit; + /** * Class for handling bucket entity type. */ @@ -87,7 +90,7 @@ private BucketObjectDBInfo getBucketObjDbInfo(String[] names) @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sortSubpaths) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); @@ -142,7 +145,15 @@ public DUResponse getDuResponse( } duResponse.setCount(dirDUData.size()); duResponse.setSize(bucketDataSize); + + if (sortSubpaths) { + // Parallel sort directory/files DU data in descending order of size and returns the top N elements. + dirDUData = sortDiskUsageDescendingWithLimit(dirDUData, + DISK_USAGE_TOP_RECORDS_LIMIT); + } + duResponse.setDuData(dirDUData); + return duResponse; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index fc7022e2dab..ae7181af70b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -39,6 +39,9 @@ import java.util.List; import java.util.Set; +import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit; + /** * Class for handling directory entity type. */ @@ -80,7 +83,7 @@ private ObjectDBInfo getDirectoryObjDbInfo(String[] names) @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sortSubPaths) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); @@ -154,6 +157,13 @@ public DUResponse getDuResponse( } duResponse.setCount(subdirDUData.size()); duResponse.setSize(dirDataSize); + + if (sortSubPaths) { + // Parallel sort subdirDUData in descending order of size and returns the top N elements. + subdirDUData = sortDiskUsageDescendingWithLimit(subdirDUData, + DISK_USAGE_TOP_RECORDS_LIMIT); + } + duResponse.setDuData(subdirDUData); return duResponse; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java index 4f9e68ddff9..f2bcb58d356 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java @@ -79,7 +79,7 @@ public abstract NamespaceSummaryResponse getSummaryResponse() throws IOException; public abstract DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sort) throws IOException; public abstract QuotaUsageResponse getQuotaResponse() diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java index a687bf3d0bd..8ea26fd2846 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java @@ -71,7 +71,7 @@ private ObjectDBInfo getKeyDbObjectInfo(String[] names) @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sort) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index fd0e58f191a..b67703257ac 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -39,6 +39,9 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit; + /** * Class for handling root entity type. */ @@ -88,7 +91,7 @@ private ObjectDBInfo getPrefixObjDbInfo() @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sortSubPaths) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); @@ -137,6 +140,13 @@ public DUResponse getDuResponse( duResponse.setSizeWithReplica(totalDataSizeWithReplica); } duResponse.setSize(totalDataSize); + + if (sortSubPaths) { + // Parallel sort volumeDuData in descending order of size and returns the top N elements. + volumeDuData = sortDiskUsageDescendingWithLimit(volumeDuData, + DISK_USAGE_TOP_RECORDS_LIMIT); + } + duResponse.setDuData(volumeDuData); return duResponse; @@ -148,7 +158,8 @@ public QuotaUsageResponse getQuotaResponse() QuotaUsageResponse quotaUsageResponse = new QuotaUsageResponse(); SCMNodeStat stats = getReconSCM().getScmNodeManager().getStats(); long quotaInBytes = stats.getCapacity().get(); - long quotaUsedInBytes = getDuResponse(true, true).getSizeWithReplica(); + long quotaUsedInBytes = + getDuResponse(true, true, false).getSizeWithReplica(); quotaUsageResponse.setQuota(quotaInBytes); quotaUsageResponse.setQuotaUsed(quotaUsedInBytes); return quotaUsageResponse; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java index b5a5bd9a0be..ab61ec38e8b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java @@ -51,7 +51,7 @@ public NamespaceSummaryResponse getSummaryResponse() @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sort) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setStatus(ResponseStatus.PATH_NOT_FOUND); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java index fae508a99c9..2ca9c352ce7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java @@ -36,6 +36,10 @@ import java.util.ArrayList; import java.util.List; + +import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit; + /** * Class for handling volume entity type. */ @@ -92,7 +96,7 @@ private VolumeObjectDBInfo getVolumeObjDbInfo(String[] names) @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sortSubPaths) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); @@ -131,6 +135,13 @@ public DUResponse getDuResponse( duResponse.setSizeWithReplica(volDataSizeWithReplica); } duResponse.setSize(volDataSize); + + if (sortSubPaths) { + // Parallel sort bucketDuData in descending order of size and returns the top N elements. + bucketDuData = sortDiskUsageDescendingWithLimit(bucketDuData, + DISK_USAGE_TOP_RECORDS_LIMIT); + } + duResponse.setDuData(bucketDuData); return duResponse; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java index 2f3de1debcd..57f7686263f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java @@ -71,7 +71,7 @@ private long getEntitySize(String path) throws IOException { EntityHandler.getEntityHandler(reconNamespaceSummaryManager, omMetadataManager, reconSCM, path); if (null != entityHandler) { - DUResponse duResponse = entityHandler.getDuResponse(false, false); + DUResponse duResponse = entityHandler.getDuResponse(false, false, false); if (null != duResponse && duResponse.getStatus() == ResponseStatus.OK) { return duResponse.getSize(); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java new file mode 100644 index 00000000000..a244e4ff2ce --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; + +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; +import org.junit.jupiter.api.BeforeEach; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.ws.rs.core.Response; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.mockito.Mockito.when; + +/** + * Test NSSummary Disk Usage subpath ordering. + */ +public class TestNSSummaryDiskUsageOrdering { + + @TempDir + private Path temporaryFolder; + + private ReconOMMetadataManager reconOMMetadataManager; + private NSSummaryEndpoint nsSummaryEndpoint; + private OzoneConfiguration ozoneConfiguration; + private static final String ROOT_PATH = "/"; + private static final String TEST_USER = "TestUser"; + private OMMetadataManager omMetadataManager; + @BeforeEach + public void setUp() throws Exception { + ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, + 100); + omMetadataManager = initializeNewOmMetadataManager( + Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")) + .toFile()); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + getMockOzoneManagerServiceProviderWithFSO(); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .addBinding(OzoneStorageContainerManager.class, + getMockReconSCM()) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(NSSummaryEndpoint.class) + .build(); + ReconNamespaceSummaryManager reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); + + // populate OM DB and reprocess into Recon RocksDB + populateOMDB(); + NSSummaryTaskWithFSO nSSummaryTaskWithFso = + new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, + reconOMMetadataManager, ozoneConfiguration); + nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); + } + + /** + * Create a new OM Metadata manager instance with one user, one vol, and two + * buckets. + * @throws IOException ioEx + */ + private static OMMetadataManager initializeNewOmMetadataManager( + File omDbDir) + throws IOException { + OzoneConfiguration omConfiguration = new OzoneConfiguration(); + omConfiguration.set(OZONE_OM_DB_DIRS, + omDbDir.getAbsolutePath()); + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( + omConfiguration, null); + return omMetadataManager; + } + + @Test + public void testDiskUsageOrderingForRoot() throws Exception { + // root level DU + // Verify the ordering of subpaths under the root + verifyOrdering(ROOT_PATH); + } + + @Test + public void testDiskUsageOrderingForVolume() throws Exception { + // volume level DU + // Verify the ordering of subpaths under the volume + verifyOrdering("/volA"); + verifyOrdering("/volB"); + } + + @Test + public void testDiskUsageOrderingForBucket() throws Exception { + // bucket level DU + // Verify the ordering of subpaths under the bucket + verifyOrdering("/volA/bucketA1"); + verifyOrdering("/volA/bucketA2"); + verifyOrdering("/volA/bucketA3"); + verifyOrdering("/volB/bucketB1"); + } + + private void verifyOrdering(String path) + throws IOException { + Response response = + nsSummaryEndpoint.getDiskUsage(path, true, false, true); + DUResponse duRes = (DUResponse) response.getEntity(); + List duData = duRes.getDuData(); + List sortedDuData = new ArrayList<>(duData); + // Sort the DU data by size in descending order to compare with the original. + sortedDuData.sort( + Comparator.comparingLong(DUResponse.DiskUsage::getSize).reversed()); + + for (int i = 0; i < duData.size(); i++) { + assertEquals(sortedDuData.get(i).getSubpath(), + duData.get(i).getSubpath(), + "DU-Sub-Path under " + path + + " should be sorted by descending order of size"); + } + } + + /** + * Tests the NSSummaryEndpoint for a given volume, bucket, and directory structure. + * The test setup mimics the following filesystem structure with specified sizes: + * + * root + * ├── volA + * │ ├── bucketA1 + * │ │ ├── fileA1 (Size: 600KB) + * │ │ ├── fileA2 (Size: 80KB) + * │ │ ├── dirA1 (Total Size: 1500KB) + * │ │ ├── dirA2 (Total Size: 1700KB) + * │ │ └── dirA3 (Total Size: 1300KB) + * │ ├── bucketA2 + * │ │ ├── fileA3 (Size: 200KB) + * │ │ ├── fileA4 (Size: 4000KB) + * │ │ ├── dirA4 (Total Size: 1100KB) + * │ │ ├── dirA5 (Total Size: 1900KB) + * │ │ └── dirA6 (Total Size: 210KB) + * │ └── bucketA3 + * │ ├── fileA5 (Size: 5000KB) + * │ ├── fileA6 (Size: 700KB) + * │ ├── dirA7 (Total Size: 1200KB) + * │ ├── dirA8 (Total Size: 1600KB) + * │ └── dirA9 (Total Size: 180KB) + * └── volB + * └── bucketB1 + * ├── fileB1 (Size: 300KB) + * ├── fileB2 (Size: 500KB) + * ├── dirB1 (Total Size: 14000KB) + * ├── dirB2 (Total Size: 1800KB) + * └── dirB3 (Total Size: 2200KB) + * + * @throws Exception + */ + private void populateOMDB() throws Exception { + // Create Volumes + long volAObjectId = createVolume("volA"); + long volBObjectId = createVolume("volB"); + + // Create Buckets in volA + long bucketA1ObjectId = + createBucket("volA", "bucketA1", 600 + 80 + 1500 + 1700 + 1300); + long bucketA2ObjectId = + createBucket("volA", "bucketA2", 200 + 4000 + 1100 + 1900 + 210); + long bucketA3ObjectId = + createBucket("volA", "bucketA3", 5000 + 700 + 1200 + 1600 + 180); + + // Create Bucket in volB + long bucketB1ObjectId = + createBucket("volB", "bucketB1", 300 + 500 + 14000 + 1800 + 2200); + + // Create Directories and Files under bucketA1 + long dirA1ObjectId = + createDirectory(bucketA1ObjectId, bucketA1ObjectId, volAObjectId, + "dirA1"); + long dirA2ObjectId = + createDirectory(bucketA1ObjectId, bucketA1ObjectId, volAObjectId, + "dirA2"); + long dirA3ObjectId = + createDirectory(bucketA1ObjectId, bucketA1ObjectId, volAObjectId, + "dirA3"); + + // Files directly under bucketA1 + createFile("fileA1", "bucketA1", "volA", "fileA1", bucketA1ObjectId, + bucketA1ObjectId, volAObjectId, 600 * 1024); + createFile("fileA2", "bucketA1", "volA", "fileA2", bucketA1ObjectId, + bucketA1ObjectId, volAObjectId, 80 * 1024); + + // Create Directories and Files under bucketA2 + long dirA4ObjectId = + createDirectory(bucketA2ObjectId, bucketA2ObjectId, volAObjectId, + "dirA4"); + long dirA5ObjectId = + createDirectory(bucketA2ObjectId, bucketA2ObjectId, volAObjectId, + "dirA5"); + long dirA6ObjectId = + createDirectory(bucketA2ObjectId, bucketA2ObjectId, volAObjectId, + "dirA6"); + + // Files directly under bucketA2 + createFile("fileA3", "bucketA2", "volA", "fileA3", bucketA2ObjectId, + bucketA2ObjectId, volAObjectId, 200 * 1024); + createFile("fileA4", "bucketA2", "volA", "fileA4", bucketA2ObjectId, + bucketA2ObjectId, volAObjectId, 4000 * 1024); + + // Create Directories and Files under bucketA3 + long dirA7ObjectId = + createDirectory(bucketA3ObjectId, bucketA3ObjectId, volAObjectId, + "dirA7"); + long dirA8ObjectId = + createDirectory(bucketA3ObjectId, bucketA3ObjectId, volAObjectId, + "dirA8"); + long dirA9ObjectId = + createDirectory(bucketA3ObjectId, bucketA3ObjectId, volAObjectId, + "dirA9"); + + // Files directly under bucketA3 + createFile("fileA5", "bucketA3", "volA", "fileA5", bucketA3ObjectId, + bucketA3ObjectId, volAObjectId, 5000 * 1024); + createFile("fileA6", "bucketA3", "volA", "fileA6", bucketA3ObjectId, + bucketA3ObjectId, volAObjectId, 700 * 1024); + + // Create Directories and Files under bucketB1 + long dirB1ObjectId = + createDirectory(bucketB1ObjectId, bucketB1ObjectId, volBObjectId, + "dirB1"); + long dirB2ObjectId = + createDirectory(bucketB1ObjectId, bucketB1ObjectId, volBObjectId, + "dirB2"); + long dirB3ObjectId = + createDirectory(bucketB1ObjectId, bucketB1ObjectId, volBObjectId, + "dirB3"); + + // Files directly under bucketB1 + createFile("fileB1", "bucketB1", "volB", "fileB1", bucketB1ObjectId, + bucketB1ObjectId, volBObjectId, 300 * 1024); + createFile("fileB2", "bucketB1", "volB", "fileB2", bucketB1ObjectId, + bucketB1ObjectId, volBObjectId, 500 * 1024); + + // Create Inner files under directories + createFile("dirA1/innerFile", "bucketA1", "volA", "innerFile", + dirA1ObjectId, bucketA1ObjectId, volAObjectId, 1500 * 1024); + createFile("dirA2/innerFile", "bucketA1", "volA", "innerFile", + dirA2ObjectId, bucketA1ObjectId, volAObjectId, 1700 * 1024); + createFile("dirA3/innerFile", "bucketA1", "volA", "innerFile", + dirA3ObjectId, bucketA1ObjectId, volAObjectId, 1300 * 1024); + createFile("dirA4/innerFile", "bucketA2", "volA", "innerFile", + dirA4ObjectId, bucketA2ObjectId, volAObjectId, 1100 * 1024); + createFile("dirA5/innerFile", "bucketA2", "volA", "innerFile", + dirA5ObjectId, bucketA2ObjectId, volAObjectId, 1900 * 1024); + createFile("dirA6/innerFile", "bucketA2", "volA", "innerFile", + dirA6ObjectId, bucketA2ObjectId, volAObjectId, 210 * 1024); + createFile("dirA7/innerFile", "bucketA3", "volA", "innerFile", + dirA7ObjectId, bucketA3ObjectId, volAObjectId, 1200 * 1024); + createFile("dirA8/innerFile", "bucketA3", "volA", "innerFile", + dirA8ObjectId, bucketA3ObjectId, volAObjectId, 1600 * 1024); + createFile("dirA9/innerFile", "bucketA3", "volA", "innerFile", + dirA9ObjectId, bucketA3ObjectId, volAObjectId, 180 * 1024); + createFile("dirB1/innerFile", "bucketB1", "volB", "innerFile", + dirB1ObjectId, bucketB1ObjectId, volBObjectId, 14000 * 1024); + createFile("dirB2/innerFile", "bucketB1", "volB", "innerFile", + dirB2ObjectId, bucketB1ObjectId, volBObjectId, 1800 * 1024); + createFile("dirB3/innerFile", "bucketB1", "volB", "innerFile", + dirB3ObjectId, bucketB1ObjectId, volBObjectId, 2200 * 1024); + } + + /** + * Create a volume and add it to the Volume Table. + * @return volume Object ID + * @throws IOException + */ + private long createVolume(String volumeName) throws Exception { + String volumeKey = reconOMMetadataManager.getVolumeKey(volumeName); + long volumeId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Generate positive ID + OmVolumeArgs args = OmVolumeArgs.newBuilder() + .setObjectID(volumeId) + .setVolume(volumeName) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .build(); + + reconOMMetadataManager.getVolumeTable().put(volumeKey, args); + return volumeId; + } + + /** + * Create a bucket and add it to the Bucket Table. + * @return bucket Object ID + * @throws IOException + */ + private long createBucket(String volumeName, String bucketName, long dataSize) + throws Exception { + String bucketKey = + reconOMMetadataManager.getBucketKey(volumeName, bucketName); + long bucketId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Generate positive ID + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setObjectID(bucketId) + .setBucketLayout(getBucketLayout()) + .setUsedBytes(dataSize) + .build(); + + reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); + return bucketId; + } + + /** + * Create a directory and add it to the Directory Table. + * @return directory Object ID + * @throws IOException + */ + private long createDirectory(long parentObjectId, + long bucketObjectId, + long volumeObjectId, + String dirName) throws IOException { + long objectId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Ensure positive ID + writeDirToOm(reconOMMetadataManager, objectId, parentObjectId, + bucketObjectId, + volumeObjectId, dirName); + return objectId; + } + + /** + * Create a file and add it to the File Table. + * @return file Object ID + * @throws IOException + */ + @SuppressWarnings("checkstyle:ParameterNumber") + private long createFile(String key, + String bucket, + String volume, + String fileName, + long parentObjectId, + long bucketObjectId, + long volumeObjectId, + long dataSize) throws IOException { + long objectId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Ensure positive ID + writeKeyToOm(reconOMMetadataManager, key, bucket, volume, fileName, + objectId, + parentObjectId, bucketObjectId, volumeObjectId, dataSize, + getBucketLayout()); + return objectId; + } + + private static ReconStorageContainerManagerFacade getMockReconSCM() + throws ContainerNotFoundException { + ReconStorageContainerManagerFacade reconSCM = + mock(ReconStorageContainerManagerFacade.class); + ContainerManager containerManager = mock(ContainerManager.class); + + when(reconSCM.getContainerManager()).thenReturn(containerManager); + ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); + when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); + return reconSCM; + } + + private static BucketLayout getBucketLayout() { + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index cbe850b918f..a88064d565b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -449,7 +449,7 @@ public void testGetBasicInfoKey() throws Exception { public void testDiskUsageRoot() throws Exception { // root level DU Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false); + false, false, false); DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); assertEquals(2, duRootRes.getCount()); List duRootData = duRootRes.getDuData(); @@ -463,11 +463,12 @@ public void testDiskUsageRoot() throws Exception { assertEquals(VOL_DATA_SIZE, duVol1.getSize()); assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); } + @Test public void testDiskUsageVolume() throws Exception { // volume level DU Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false); + false, false, false); DUResponse duVolRes = (DUResponse) volResponse.getEntity(); assertEquals(2, duVolRes.getCount()); List duData = duVolRes.getDuData(); @@ -482,11 +483,12 @@ public void testDiskUsageVolume() throws Exception { assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); } + @Test public void testDiskUsageBucket() throws Exception { // bucket level DU Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false); + false, false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(1, duBucketResponse.getCount()); DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); @@ -494,11 +496,12 @@ public void testDiskUsageBucket() throws Exception { assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); } + @Test public void testDiskUsageDir() throws Exception { // dir level DU Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, false); + false, false, false); DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); assertEquals(3, duDirReponse.getCount()); List duSubDir = duDirReponse.getDuData(); @@ -517,21 +520,23 @@ public void testDiskUsageDir() throws Exception { assertEquals(KEY_SIX_SIZE, duDir4.getSize()); } + @Test public void testDiskUsageKey() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false); + false, false, false); DUResponse keyObj = (DUResponse) keyResponse.getEntity(); assertEquals(0, keyObj.getCount()); assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); } + @Test public void testDiskUsageUnknown() throws Exception { // invalid path check Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false); + false, false, false); DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); @@ -541,7 +546,7 @@ public void testDiskUsageUnknown() throws Exception { public void testDiskUsageWithReplication() throws Exception { setUpMultiBlockKey(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, @@ -553,7 +558,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); // withReplica is true Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, @@ -567,7 +572,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { public void testDataSizeUnderVolWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, @@ -580,7 +585,7 @@ public void testDataSizeUnderVolWithReplication() throws IOException { public void testDataSizeUnderBucketWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, @@ -599,7 +604,7 @@ public void testDataSizeUnderBucketWithReplication() throws IOException { public void testDataSizeUnderDirWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, @@ -612,7 +617,7 @@ public void testDataSizeUnderDirWithReplication() throws IOException { public void testDataSizeUnderKeyWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java index 765399f71e3..a5064ba5bef 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java @@ -245,10 +245,10 @@ public class TestNSSummaryEndpointWithLegacy { StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE6_SIZE_WITH_REPLICA = getReplicatedSize(KEY_SIX_SIZE, - StandaloneReplicationConfig.getInstance(ONE));; + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE7_SIZE_WITH_REPLICA = getReplicatedSize(KEY_SEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE));; + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE8_SIZE_WITH_REPLICA = getReplicatedSize(KEY_EIGHT_SIZE, StandaloneReplicationConfig.getInstance(ONE)); @@ -451,7 +451,7 @@ public void testGetBasicInfoKey() throws Exception { public void testDiskUsageRoot() throws Exception { // root level DU Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false); + false, false, false); DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); assertEquals(2, duRootRes.getCount()); List duRootData = duRootRes.getDuData(); @@ -470,7 +470,7 @@ public void testDiskUsageRoot() throws Exception { public void testDiskUsageVolume() throws Exception { // volume level DU Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false); + false, false, false); DUResponse duVolRes = (DUResponse) volResponse.getEntity(); assertEquals(2, duVolRes.getCount()); List duData = duVolRes.getDuData(); @@ -489,7 +489,7 @@ public void testDiskUsageVolume() throws Exception { public void testDiskUsageBucket() throws Exception { // bucket level DU Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false); + false, false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(1, duBucketResponse.getCount()); DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); @@ -501,7 +501,7 @@ public void testDiskUsageBucket() throws Exception { public void testDiskUsageDir() throws Exception { // dir level DU Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, false); + false, false, false); DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); assertEquals(3, duDirReponse.getCount()); List duSubDir = duDirReponse.getDuData(); @@ -524,7 +524,7 @@ public void testDiskUsageDir() throws Exception { public void testDiskUsageKey() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false); + false, false, false); DUResponse keyObj = (DUResponse) keyResponse.getEntity(); assertEquals(0, keyObj.getCount()); assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); @@ -534,7 +534,7 @@ public void testDiskUsageKey() throws Exception { public void testDiskUsageUnknown() throws Exception { // invalid path check Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false); + false, false, false); DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); @@ -544,7 +544,7 @@ public void testDiskUsageUnknown() throws Exception { public void testDiskUsageWithReplication() throws Exception { setUpMultiBlockKey(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, @@ -556,7 +556,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); // withReplica is true Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, @@ -570,7 +570,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { public void testDataSizeUnderVolWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, @@ -583,7 +583,7 @@ public void testDataSizeUnderVolWithReplication() throws IOException { public void testDataSizeUnderBucketWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, @@ -602,7 +602,7 @@ public void testDataSizeUnderBucketWithReplication() throws IOException { public void testDataSizeUnderDirWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, @@ -615,7 +615,7 @@ public void testDataSizeUnderDirWithReplication() throws IOException { public void testDataSizeUnderKeyWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java index 8d8299aefc1..ce8aa729635 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java @@ -552,7 +552,7 @@ public void testGetBasicInfoKey() throws Exception { public void testDiskUsageRoot() throws Exception { // root level DU Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false); + false, false, false); DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); assertEquals(2, duRootRes.getCount()); List duRootData = duRootRes.getDuData(); @@ -571,7 +571,7 @@ public void testDiskUsageRoot() throws Exception { public void testDiskUsageVolume() throws Exception { // volume level DU Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false); + false, false, false); DUResponse duVolRes = (DUResponse) volResponse.getEntity(); assertEquals(2, duVolRes.getCount()); List duData = duVolRes.getDuData(); @@ -590,7 +590,7 @@ public void testDiskUsageVolume() throws Exception { public void testDiskUsageVolTwo() throws Exception { // volume level DU Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_TWO_PATH, - false, false); + false, false, false); DUResponse duVolRes = (DUResponse) volResponse.getEntity(); assertEquals(2, duVolRes.getCount()); List duData = duVolRes.getDuData(); @@ -608,13 +608,13 @@ public void testDiskUsageVolTwo() throws Exception { public void testDiskUsageBucketOne() throws Exception { // bucket level DU Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false); + false, false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); // There are no sub-paths under this OBS bucket. assertEquals(0, duBucketResponse.getCount()); Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_ONE_PATH, true, false); + BUCKET_ONE_PATH, true, false, false); DUResponse duBucketResponseWithFiles = (DUResponse) bucketResponseWithSubpath.getEntity(); assertEquals(3, duBucketResponseWithFiles.getCount()); @@ -626,13 +626,13 @@ public void testDiskUsageBucketOne() throws Exception { public void testDiskUsageBucketTwo() throws Exception { // bucket level DU Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_TWO_PATH, - false, false); + false, false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); // There are no sub-paths under this OBS bucket. assertEquals(0, duBucketResponse.getCount()); Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_TWO_PATH, true, false); + BUCKET_TWO_PATH, true, false, false); DUResponse duBucketResponseWithFiles = (DUResponse) bucketResponseWithSubpath.getEntity(); assertEquals(2, duBucketResponseWithFiles.getCount()); @@ -644,13 +644,13 @@ public void testDiskUsageBucketTwo() throws Exception { public void testDiskUsageBucketThree() throws Exception { // bucket level DU Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_THREE_PATH, - false, false); + false, false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); // There are no sub-paths under this Legacy bucket. assertEquals(0, duBucketResponse.getCount()); Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_THREE_PATH, true, false); + BUCKET_THREE_PATH, true, false, false); DUResponse duBucketResponseWithFiles = (DUResponse) bucketResponseWithSubpath.getEntity(); assertEquals(3, duBucketResponseWithFiles.getCount()); @@ -662,7 +662,7 @@ public void testDiskUsageBucketThree() throws Exception { public void testDiskUsageKey1() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_ONE_PATH, - false, false); + false, false, false); DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); assertEquals(0, duKeyResponse.getCount()); assertEquals(FILE_ONE_SIZE, duKeyResponse.getSize()); @@ -672,7 +672,7 @@ public void testDiskUsageKey1() throws Exception { public void testDiskUsageKey2() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_TWO_PATH, - false, false); + false, false, false); DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); assertEquals(0, duKeyResponse.getCount()); assertEquals(FILE_TWO_SIZE, duKeyResponse.getSize()); @@ -682,7 +682,7 @@ public void testDiskUsageKey2() throws Exception { public void testDiskUsageKey4() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY4_PATH, - true, false); + true, false, false); DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); assertEquals(0, duKeyResponse.getCount()); assertEquals(FILE_FOUR_SIZE, duKeyResponse.getSize()); @@ -692,7 +692,7 @@ public void testDiskUsageKey4() throws Exception { public void testDiskUsageKey5() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_FIVE_PATH, - false, false); + false, false, false); DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); assertEquals(0, duKeyResponse.getCount()); assertEquals(FILE_FIVE_SIZE, duKeyResponse.getSize()); @@ -702,7 +702,7 @@ public void testDiskUsageKey5() throws Exception { public void testDiskUsageKey8() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_EIGHT_PATH, - false, false); + false, false, false); DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); assertEquals(0, duKeyResponse.getCount()); assertEquals(FILE_EIGHT_SIZE, duKeyResponse.getSize()); @@ -712,7 +712,7 @@ public void testDiskUsageKey8() throws Exception { public void testDiskUsageKey11() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_ELEVEN_PATH, - false, false); + false, false, false); DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); assertEquals(0, duKeyResponse.getCount()); assertEquals(FILE_ELEVEN_SIZE, duKeyResponse.getSize()); @@ -722,7 +722,7 @@ public void testDiskUsageKey11() throws Exception { public void testDiskUsageUnknown() throws Exception { // invalid path check Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false); + false, false, false); DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); @@ -732,7 +732,7 @@ public void testDiskUsageUnknown() throws Exception { public void testDiskUsageWithReplication() throws Exception { setUpMultiBlockKey(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, @@ -744,7 +744,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); // withReplica is true Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, @@ -758,7 +758,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { public void testDataSizeUnderVolWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, @@ -771,7 +771,7 @@ public void testDataSizeUnderVolWithReplication() throws IOException { public void testDataSizeUnderBucketOneWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, @@ -782,7 +782,7 @@ public void testDataSizeUnderBucketOneWithReplication() throws IOException { public void testDataSizeUnderBucketThreeWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_THREE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET3, @@ -793,7 +793,7 @@ public void testDataSizeUnderBucketThreeWithReplication() throws IOException { public void testDataSizeUnderKeyWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY4_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY,