From 7e8bb75f2aec2e88764f8531ff283784a703bbca Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 3 Mar 2024 01:21:53 +0530 Subject: [PATCH 01/10] HDDS-10452. Improve Recon Disk Usage to fetch and display Top N records based on size. --- .../ozone/recon/TestNSSummaryEndpoint.java | 304 ++++++++++++++++++ .../api/handlers/BucketEntityHandler.java | 26 +- .../api/handlers/DirectoryEntityHandler.java | 26 +- .../recon/api/handlers/RootEntityHandler.java | 23 ++ .../api/handlers/VolumeEntityHandler.java | 5 + 5 files changed, 380 insertions(+), 4 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java new file mode 100644 index 00000000000..6eb7f85c919 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java @@ -0,0 +1,304 @@ +package org.apache.hadoop.ozone.recon; + +import static org.apache.hadoop.ozone.OzoneConfigKeys.*; +import static org.apache.hadoop.ozone.om.OMConfigKeys.*; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Comparator; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; + +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.api.NSSummaryEndpoint; +import org.apache.hadoop.ozone.recon.api.OMDBInsightEndpoint; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; +import org.apache.ozone.test.GenericTestUtils; +import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.logging.Logger; +import org.junit.platform.commons.logging.LoggerFactory; + +import javax.ws.rs.core.Response; + +/** + * Integration test for verifying the correctness of NSSummaryEndpoint. + */ +public class TestNSSummaryEndpoint { + + private static Logger LOG = + LoggerFactory.getLogger(TestNSSummaryEndpoint.class); + private static boolean omRatisEnabled = true; + + private static MiniOzoneCluster cluster; + private static FileSystem fs; + private static String volumeName; + private static String bucketName; + private static OzoneClient client; + + private static final String VOLUME_A = "vola"; + private static final String VOLUME_B = "volb"; + private static final String BUCKET_A1 = "bucketa1"; + private static final String BUCKET_A2 = "bucketa2"; + private static final String BUCKET_A3 = "bucketa3"; + private static final String BUCKET_B1 = "bucketb1"; + + @BeforeAll + public static void init() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setInt(OZONE_DIR_DELETING_SERVICE_INTERVAL, 1000000); + conf.setInt(OZONE_PATH_DELETING_LIMIT_PER_TASK, 0); + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 10000000, + TimeUnit.MILLISECONDS); + conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled); + conf.setBoolean(OZONE_ACL_ENABLED, true); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) + .includeRecon(true) + .build(); + cluster.waitForClusterToBeReady(); + client = cluster.newClient(); + + // create a volume and a bucket to be used by OzoneFileSystem + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, + getFSOBucketLayout()); + volumeName = bucket.getVolumeName(); + bucketName = bucket.getName(); + + String rootPath = String.format("%s://%s.%s/", + OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName); + + // Set the fs.defaultFS and start the filesystem + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); + // Set the number of keys to be processed during batch operate. + conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); + + fs = FileSystem.get(conf); + } + + @AfterAll + public static void teardown() throws IOException { + if (client != null) { + client.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + + @AfterEach + public void cleanup() { + assertDoesNotThrow(() -> { + Path root = new Path("/"); + FileStatus[] fileStatuses = fs.listStatus(root); + for (FileStatus fileStatus : fileStatuses) { + fs.delete(fileStatus.getPath(), true); + } + }); + } + + /** + * Tests the NSSummaryEndpoint for a given volume, bucket, and directory structure. + * The test setup mimics the following filesystem structure with specified sizes: + * + * root + * ├── volA + * │ ├── bucketA1 + * │ │ ├── fileA1 (Size: 600KB) + * │ │ ├── fileA2 (Size: 800KB) + * │ │ ├── dirA1 (Total Size: 1500KB) + * │ │ ├── dirA2 (Total Size: 1700KB) + * │ │ └── dirA3 (Total Size: 1300KB) + * │ ├── bucketA2 + * │ │ ├── fileA3 (Size: 200KB) + * │ │ ├── fileA4 (Size: 400KB) + * │ │ ├── dirA4 (Total Size: 1100KB) + * │ │ ├── dirA5 (Total Size: 1900KB) + * │ │ └── dirA6 (Total Size: 2100KB) + * │ └── bucketA3 + * │ ├── fileA5 (Size: 500KB) + * │ ├── fileA6 (Size: 700KB) + * │ ├── dirA7 (Total Size: 1200KB) + * │ ├── dirA8 (Total Size: 1600KB) + * │ └── dirA9 (Total Size: 1800KB) + * └── volB + * └── bucketB1 + * ├── fileB1 (Size: 300KB) + * ├── fileB2 (Size: 500KB) + * ├── dirB1 (Total Size: 1400KB) + * ├── dirB2 (Total Size: 1800KB) + * └── dirB3 (Total Size: 2200KB) + * + * @throws Exception + */ + @Test + public void testDiskUsageOrderingBySubpathSize() throws Exception { + // Setup test data and sync data from OM to Recon + setupTestData(); + syncDataFromOM(); + + OzoneStorageContainerManager reconSCM = + cluster.getReconServer().getReconStorageContainerManager(); + ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager = + (ReconNamespaceSummaryManagerImpl) cluster.getReconServer() + .getReconNamespaceSummaryManager(); + ReconOMMetadataManager reconOmMetadataManagerInstance = + (ReconOMMetadataManager) cluster.getReconServer() + .getOzoneManagerServiceProvider().getOMMetadataManagerInstance(); + + NSSummaryEndpoint nsSummaryEndpoint = + new NSSummaryEndpoint(reconNamespaceSummaryManager, + reconOmMetadataManagerInstance, reconSCM); + + // Verify the ordering of subpaths under the root + verifyOrdering(nsSummaryEndpoint, "/"); + + // Verify the ordering of subpaths under each volume + verifyOrdering(nsSummaryEndpoint, VOLUME_A); + verifyOrdering(nsSummaryEndpoint, VOLUME_B); + + // Verify the ordering of subpaths under each bucket + verifyOrdering(nsSummaryEndpoint, VOLUME_A + "/" + BUCKET_A1); + verifyOrdering(nsSummaryEndpoint, VOLUME_A + "/" + BUCKET_A2); + verifyOrdering(nsSummaryEndpoint, VOLUME_A + "/" + BUCKET_A3); + verifyOrdering(nsSummaryEndpoint, VOLUME_B + "/" + BUCKET_B1); + } + + private void verifyOrdering(NSSummaryEndpoint nsSummaryEndpoint, String Path) + throws IOException { + Response response = + nsSummaryEndpoint.getDiskUsage(Path, true, false); + DUResponse duRes = (DUResponse) response.getEntity(); + List duData = duRes.getDuData(); + List sortedDuData = new ArrayList<>(duData); + // Sort the DU data by size in descending order to compare with the original. + sortedDuData.sort( + Comparator.comparingLong(DUResponse.DiskUsage::getSize).reversed()); + + for (int i = 0; i < duData.size(); i++) { + assertEquals(sortedDuData.get(i).getSubpath(), + duData.get(i).getSubpath(), + "DU-Sub-Path under " + Path + + " should be sorted by descending order of size"); + } + } + + public void setupTestData() throws IOException { + // Helper method to write data to a file + BiConsumer writeFile = (filePath, size) -> { + try (FSDataOutputStream outputStream = fs.create(new Path(filePath))) { + byte[] data = new byte[size.intValue()]; + new Random().nextBytes(data); // Fill with random data + outputStream.write(data); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + + // Create volumes and buckets. + client.getObjectStore().createVolume(VOLUME_A); + OzoneVolume volumeA = client.getObjectStore().getVolume(VOLUME_A); + volumeA.createBucket(BUCKET_A1); + volumeA.createBucket(BUCKET_A2); + volumeA.createBucket(BUCKET_A3); + + client.getObjectStore().createVolume(VOLUME_B); + OzoneVolume volumeB = client.getObjectStore().getVolume(VOLUME_B); + volumeB.createBucket(BUCKET_B1); + + // Define the structure and size in KB for the filesystem + Map fileSystemStructure = + new LinkedHashMap() {{ + put("vola/bucketa1/fileA1", 600); + put("vola/bucketa1/fileA2", 800); + put("vola/bucketa1/dirA1", 1500); + put("vola/bucketa1/dirA2", 1700); + put("vola/bucketa1/dirA3", 1300); + put("vola/bucketa2/fileA3", 200); + put("vola/bucketa2/fileA4", 400); + put("vola/bucketa2/dirA4", 1100); + put("vola/bucketa2/dirA5", 1900); + put("vola/bucketa2/dirA6", 2100); + put("vola/bucketa3/fileA5", 500); + put("vola/bucketa3/fileA6", 700); + put("vola/bucketa3/dirA7", 1200); + put("vola/bucketa3/dirA8", 1600); + put("vola/bucketa3/dirA9", 1800); + put("volb/bucketb1/fileB1", 300); + put("volb/bucketb1/fileB2", 500); + put("volb/bucketb1/dirB1", 1400); + put("volb/bucketb1/dirB2", 1800); + put("volb/bucketb1/dirB3", 2200); + }}; + + // Create files and directories + for (Map.Entry entry : fileSystemStructure.entrySet()) { + String[] pathParts = entry.getKey().split("/"); + String itemName = pathParts[2]; + int sizeInKB = entry.getValue(); + // Calculate the size in bytes + long sizeInBytes = sizeInKB * 1024L; + + if (itemName.startsWith("file")) { + // Create a file with the specified size + String filePath = "/" + volumeName + "/" + bucketName + "/" + itemName; + writeFile.accept(filePath, sizeInBytes); + } else { + // Create a directory + String dirPath = "/" + volumeName + "/" + bucketName + "/" + itemName; + fs.mkdirs(new Path(dirPath)); + + // Create a file inside the directory to achieve the total specified size + String innerFilePath = dirPath + "/innerFile"; + writeFile.accept(innerFilePath, sizeInBytes); + } + } + } + + private void syncDataFromOM() { + // Sync data from Ozone Manager to Recon. + OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) + cluster.getReconServer().getOzoneManagerServiceProvider(); + impl.syncDataFromOM(); + } + + private static BucketLayout getFSOBucketLayout() { + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + } + +} + diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java index 7ad961195ee..27229e8a430 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java @@ -35,6 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.Map; +import java.util.HashMap; +import java.util.Comparator; +import java.util.stream.Collectors; /** * Class for handling bucket entity type. @@ -105,10 +109,24 @@ public DUResponse getDuResponse( // get object IDs for all its subdirectories Set bucketSubdirs = bucketNSSummary.getChildDir(); duResponse.setKeySize(bucketNSSummary.getSizeOfFiles()); + + // Map to hold sizes for each subdir for sorting + Map subdirSizes = new HashMap<>(); + for (long subdirObjectId : bucketSubdirs) { + long dataSize = getTotalSize(subdirObjectId); + subdirSizes.put(subdirObjectId, dataSize); + } + + // Sort subdirs based on their size in descending order + List sortedSubdirObjectIds = bucketSubdirs.stream().sorted( + Comparator.comparingLong( + subdirObjectId -> subdirSizes.get(subdirObjectId)).reversed()) + .collect(Collectors.toList()); + List dirDUData = new ArrayList<>(); long bucketDataSize = duResponse.getKeySize(); long bucketDataSizeWithReplica = 0L; - for (long subdirObjectId: bucketSubdirs) { + for (long subdirObjectId : sortedSubdirObjectIds) { NSSummary subdirNSSummary = getReconNamespaceSummaryManager() .getNSSummary(subdirObjectId); @@ -119,7 +137,8 @@ public DUResponse getDuResponse( // format with leading slash and without trailing slash DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); diskUsage.setSubpath(subpath); - long dataSize = getTotalSize(subdirObjectId); + long dataSize = subdirSizes.get(subdirObjectId); + bucketDataSize += dataSize; if (withReplica) { @@ -136,7 +155,10 @@ public DUResponse getDuResponse( bucketDataSizeWithReplica += getBucketHandler() .handleDirectKeys(bucketObjectId, withReplica, listFile, dirDUData, getNormalizedPath()); + // Sort dirDUData by size in descending order after adding files + dirDUData.sort((du1, du2) -> Long.compare(du2.getSize(), du1.getSize())); } + if (withReplica) { duResponse.setSizeWithReplica(bucketDataSizeWithReplica); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index fc7022e2dab..1a19bd79b65 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -35,10 +35,15 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Comparator; import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; + /** * Class for handling directory entity type. */ @@ -100,9 +105,23 @@ public DUResponse getDuResponse( duResponse.setKeySize(dirNSSummary.getSizeOfFiles()); long dirDataSize = duResponse.getKeySize(); long dirDataSizeWithReplica = 0L; + + // Map to hold sizes for each subdir for sorting + Map subdirSizes = new HashMap<>(); + for (long subdirObjectId : subdirs) { + long dataSize = getTotalSize(subdirObjectId); + subdirSizes.put(subdirObjectId, dataSize); + } + + // Sort subdirs based on their size in descending order + List sortedSubdirObjectIds = subdirs.stream() + .sorted(Comparator.comparingLong( + subdirObjectId -> subdirSizes.get(subdirObjectId)).reversed()) + .collect(Collectors.toList()); + List subdirDUData = new ArrayList<>(); // iterate all subdirectories to get disk usage data - for (long subdirObjectId: subdirs) { + for (long subdirObjectId : sortedSubdirObjectIds) { NSSummary subdirNSSummary = getReconNamespaceSummaryManager().getNSSummary(subdirObjectId); // for the subdirName we need the subdir filename, not the key name @@ -128,7 +147,7 @@ public DUResponse getDuResponse( DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); // reformat the response diskUsage.setSubpath(subpath); - long dataSize = getTotalSize(subdirObjectId); + long dataSize = subdirSizes.get(subdirObjectId); dirDataSize += dataSize; if (withReplica) { @@ -147,6 +166,9 @@ public DUResponse getDuResponse( dirDataSizeWithReplica += getBucketHandler() .handleDirectKeys(dirObjectId, withReplica, listFile, subdirDUData, getNormalizedPath()); + // Sort dirDUData by size in descending order after adding files + subdirDUData.sort( + (du1, du2) -> Long.compare(du2.getSize(), du1.getSize())); } if (withReplica) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index fd0e58f191a..4626145736b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -34,6 +34,12 @@ import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import java.io.IOException; import java.util.ArrayList; @@ -96,6 +102,23 @@ public DUResponse getDuResponse( List volumes = getOmMetadataManager().listVolumes(); duResponse.setCount(volumes.size()); + // Map to hold total size for each volume + Map volumeSizes = new HashMap<>(); + for (OmVolumeArgs volume : volumes) { + long totalVolumeSize = 0; + List buckets = + omMetadataManager.listBucketsUnderVolume(volume.getVolume()); + for (OmBucketInfo bucket : buckets) { + long bucketObjectID = bucket.getObjectID(); + totalVolumeSize += getTotalSize(bucketObjectID); + } + volumeSizes.put(volume.getVolume(), totalVolumeSize); + } + + // Sort volumes based on the total size in descending order + volumes.sort((v1, v2) -> volumeSizes.get(v2.getVolume()) + .compareTo(volumeSizes.get(v1.getVolume()))); + List volumeDuData = new ArrayList<>(); long totalDataSize = 0L; long totalDataSizeWithReplica = 0L; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java index fae508a99c9..056fa89fd9a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java @@ -100,6 +100,11 @@ public DUResponse getDuResponse( String volName = names[0]; List buckets = getOmMetadataManager(). listBucketsUnderVolume(volName); + + // Sort buckets in descending order by size + buckets.sort( + (b1, b2) -> Long.compare(b2.getUsedBytes(), b1.getUsedBytes())); + duResponse.setCount(buckets.size()); // List of DiskUsage data for all buckets From 5a03fba3460b028e8bbd983d3de9c5e79c58f5f2 Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 3 Mar 2024 02:23:35 +0530 Subject: [PATCH 02/10] Fixed checkstyle and bugs --- .../ozone/recon/TestNSSummaryEndpoint.java | 80 +++++++------------ .../recon/api/handlers/RootEntityHandler.java | 4 - 2 files changed, 27 insertions(+), 57 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java index 6eb7f85c919..26bad94faa9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java @@ -1,9 +1,8 @@ package org.apache.hadoop.ozone.recon; -import static org.apache.hadoop.ozone.OzoneConfigKeys.*; -import static org.apache.hadoop.ozone.om.OMConfigKeys.*; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.mock; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import java.io.IOException; import java.util.ArrayList; @@ -12,9 +11,6 @@ import java.util.Map; import java.util.Random; import java.util.Comparator; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -22,36 +18,24 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.recon.api.NSSummaryEndpoint; -import org.apache.hadoop.ozone.recon.api.OMDBInsightEndpoint; import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; -import org.apache.ozone.test.GenericTestUtils; -import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.platform.commons.logging.Logger; -import org.junit.platform.commons.logging.LoggerFactory; import javax.ws.rs.core.Response; @@ -60,8 +44,6 @@ */ public class TestNSSummaryEndpoint { - private static Logger LOG = - LoggerFactory.getLogger(TestNSSummaryEndpoint.class); private static boolean omRatisEnabled = true; private static MiniOzoneCluster cluster; @@ -80,12 +62,6 @@ public class TestNSSummaryEndpoint { @BeforeAll public static void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OZONE_DIR_DELETING_SERVICE_INTERVAL, 1000000); - conf.setInt(OZONE_PATH_DELETING_LIMIT_PER_TASK, 0); - conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 10000000, - TimeUnit.MILLISECONDS); - conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled); - conf.setBoolean(OZONE_ACL_ENABLED, true); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .includeRecon(true) @@ -198,10 +174,10 @@ public void testDiskUsageOrderingBySubpathSize() throws Exception { verifyOrdering(nsSummaryEndpoint, VOLUME_B + "/" + BUCKET_B1); } - private void verifyOrdering(NSSummaryEndpoint nsSummaryEndpoint, String Path) + private void verifyOrdering(NSSummaryEndpoint nsSummaryEndpoint, String path) throws IOException { Response response = - nsSummaryEndpoint.getDiskUsage(Path, true, false); + nsSummaryEndpoint.getDiskUsage(path, true, false); DUResponse duRes = (DUResponse) response.getEntity(); List duData = duRes.getDuData(); List sortedDuData = new ArrayList<>(duData); @@ -212,7 +188,7 @@ private void verifyOrdering(NSSummaryEndpoint nsSummaryEndpoint, String Path) for (int i = 0; i < duData.size(); i++) { assertEquals(sortedDuData.get(i).getSubpath(), duData.get(i).getSubpath(), - "DU-Sub-Path under " + Path + + "DU-Sub-Path under " + path + " should be sorted by descending order of size"); } } @@ -241,29 +217,27 @@ public void setupTestData() throws IOException { volumeB.createBucket(BUCKET_B1); // Define the structure and size in KB for the filesystem - Map fileSystemStructure = - new LinkedHashMap() {{ - put("vola/bucketa1/fileA1", 600); - put("vola/bucketa1/fileA2", 800); - put("vola/bucketa1/dirA1", 1500); - put("vola/bucketa1/dirA2", 1700); - put("vola/bucketa1/dirA3", 1300); - put("vola/bucketa2/fileA3", 200); - put("vola/bucketa2/fileA4", 400); - put("vola/bucketa2/dirA4", 1100); - put("vola/bucketa2/dirA5", 1900); - put("vola/bucketa2/dirA6", 2100); - put("vola/bucketa3/fileA5", 500); - put("vola/bucketa3/fileA6", 700); - put("vola/bucketa3/dirA7", 1200); - put("vola/bucketa3/dirA8", 1600); - put("vola/bucketa3/dirA9", 1800); - put("volb/bucketb1/fileB1", 300); - put("volb/bucketb1/fileB2", 500); - put("volb/bucketb1/dirB1", 1400); - put("volb/bucketb1/dirB2", 1800); - put("volb/bucketb1/dirB3", 2200); - }}; + Map fileSystemStructure = new LinkedHashMap<>(); + fileSystemStructure.put("vola/bucketa1/fileA1", 600); + fileSystemStructure.put("vola/bucketa1/fileA2", 800); + fileSystemStructure.put("vola/bucketa1/dirA1", 1500); + fileSystemStructure.put("vola/bucketa1/dirA2", 1700); + fileSystemStructure.put("vola/bucketa1/dirA3", 1300); + fileSystemStructure.put("vola/bucketa2/fileA3", 200); + fileSystemStructure.put("vola/bucketa2/fileA4", 400); + fileSystemStructure.put("vola/bucketa2/dirA4", 1100); + fileSystemStructure.put("vola/bucketa2/dirA5", 1900); + fileSystemStructure.put("vola/bucketa2/dirA6", 2100); + fileSystemStructure.put("vola/bucketa3/fileA5", 500); + fileSystemStructure.put("vola/bucketa3/fileA6", 700); + fileSystemStructure.put("vola/bucketa3/dirA7", 1200); + fileSystemStructure.put("vola/bucketa3/dirA8", 1600); + fileSystemStructure.put("vola/bucketa3/dirA9", 1800); + fileSystemStructure.put("volb/bucketb1/fileB1", 300); + fileSystemStructure.put("volb/bucketb1/fileB2", 500); + fileSystemStructure.put("volb/bucketb1/dirB1", 1400); + fileSystemStructure.put("volb/bucketb1/dirB2", 1800); + fileSystemStructure.put("volb/bucketb1/dirB3", 2200); // Create files and directories for (Map.Entry entry : fileSystemStructure.entrySet()) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index 4626145736b..ef1a93cc43a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -34,16 +34,12 @@ import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import java.io.IOException; import java.util.ArrayList; -import java.util.List; /** * Class for handling root entity type. From a11e44fb748a471e7955451ad7925598aa7ef046 Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 3 Mar 2024 02:34:05 +0530 Subject: [PATCH 03/10] Added licence --- .../ozone/recon/TestNSSummaryEndpoint.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java index 26bad94faa9..83dd2dc94c4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.recon; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; From 65b3053b6178e0e9443aaab77ef3298eb2c5ecac Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 5 Mar 2024 03:22:28 +0530 Subject: [PATCH 04/10] Fixed errors and bugs --- .../ozone/recon/TestNSSummaryEndpoint.java | 296 ------------- .../api/TestNSSummaryDiskUsageOrdering.java | 407 ++++++++++++++++++ 2 files changed, 407 insertions(+), 296 deletions(-) delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java deleted file mode 100644 index 83dd2dc94c4..00000000000 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryEndpoint.java +++ /dev/null @@ -1,296 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon; - -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Comparator; -import java.util.function.BiConsumer; - -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.recon.api.NSSummaryEndpoint; -import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import javax.ws.rs.core.Response; - -/** - * Integration test for verifying the correctness of NSSummaryEndpoint. - */ -public class TestNSSummaryEndpoint { - - private static boolean omRatisEnabled = true; - - private static MiniOzoneCluster cluster; - private static FileSystem fs; - private static String volumeName; - private static String bucketName; - private static OzoneClient client; - - private static final String VOLUME_A = "vola"; - private static final String VOLUME_B = "volb"; - private static final String BUCKET_A1 = "bucketa1"; - private static final String BUCKET_A2 = "bucketa2"; - private static final String BUCKET_A3 = "bucketa3"; - private static final String BUCKET_B1 = "bucketb1"; - - @BeforeAll - public static void init() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .includeRecon(true) - .build(); - cluster.waitForClusterToBeReady(); - client = cluster.newClient(); - - // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, - getFSOBucketLayout()); - volumeName = bucket.getVolumeName(); - bucketName = bucket.getName(); - - String rootPath = String.format("%s://%s.%s/", - OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName); - - // Set the fs.defaultFS and start the filesystem - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); - // Set the number of keys to be processed during batch operate. - conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); - - fs = FileSystem.get(conf); - } - - @AfterAll - public static void teardown() throws IOException { - if (client != null) { - client.close(); - } - if (cluster != null) { - cluster.shutdown(); - } - } - - @AfterEach - public void cleanup() { - assertDoesNotThrow(() -> { - Path root = new Path("/"); - FileStatus[] fileStatuses = fs.listStatus(root); - for (FileStatus fileStatus : fileStatuses) { - fs.delete(fileStatus.getPath(), true); - } - }); - } - - /** - * Tests the NSSummaryEndpoint for a given volume, bucket, and directory structure. - * The test setup mimics the following filesystem structure with specified sizes: - * - * root - * ├── volA - * │ ├── bucketA1 - * │ │ ├── fileA1 (Size: 600KB) - * │ │ ├── fileA2 (Size: 800KB) - * │ │ ├── dirA1 (Total Size: 1500KB) - * │ │ ├── dirA2 (Total Size: 1700KB) - * │ │ └── dirA3 (Total Size: 1300KB) - * │ ├── bucketA2 - * │ │ ├── fileA3 (Size: 200KB) - * │ │ ├── fileA4 (Size: 400KB) - * │ │ ├── dirA4 (Total Size: 1100KB) - * │ │ ├── dirA5 (Total Size: 1900KB) - * │ │ └── dirA6 (Total Size: 2100KB) - * │ └── bucketA3 - * │ ├── fileA5 (Size: 500KB) - * │ ├── fileA6 (Size: 700KB) - * │ ├── dirA7 (Total Size: 1200KB) - * │ ├── dirA8 (Total Size: 1600KB) - * │ └── dirA9 (Total Size: 1800KB) - * └── volB - * └── bucketB1 - * ├── fileB1 (Size: 300KB) - * ├── fileB2 (Size: 500KB) - * ├── dirB1 (Total Size: 1400KB) - * ├── dirB2 (Total Size: 1800KB) - * └── dirB3 (Total Size: 2200KB) - * - * @throws Exception - */ - @Test - public void testDiskUsageOrderingBySubpathSize() throws Exception { - // Setup test data and sync data from OM to Recon - setupTestData(); - syncDataFromOM(); - - OzoneStorageContainerManager reconSCM = - cluster.getReconServer().getReconStorageContainerManager(); - ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager = - (ReconNamespaceSummaryManagerImpl) cluster.getReconServer() - .getReconNamespaceSummaryManager(); - ReconOMMetadataManager reconOmMetadataManagerInstance = - (ReconOMMetadataManager) cluster.getReconServer() - .getOzoneManagerServiceProvider().getOMMetadataManagerInstance(); - - NSSummaryEndpoint nsSummaryEndpoint = - new NSSummaryEndpoint(reconNamespaceSummaryManager, - reconOmMetadataManagerInstance, reconSCM); - - // Verify the ordering of subpaths under the root - verifyOrdering(nsSummaryEndpoint, "/"); - - // Verify the ordering of subpaths under each volume - verifyOrdering(nsSummaryEndpoint, VOLUME_A); - verifyOrdering(nsSummaryEndpoint, VOLUME_B); - - // Verify the ordering of subpaths under each bucket - verifyOrdering(nsSummaryEndpoint, VOLUME_A + "/" + BUCKET_A1); - verifyOrdering(nsSummaryEndpoint, VOLUME_A + "/" + BUCKET_A2); - verifyOrdering(nsSummaryEndpoint, VOLUME_A + "/" + BUCKET_A3); - verifyOrdering(nsSummaryEndpoint, VOLUME_B + "/" + BUCKET_B1); - } - - private void verifyOrdering(NSSummaryEndpoint nsSummaryEndpoint, String path) - throws IOException { - Response response = - nsSummaryEndpoint.getDiskUsage(path, true, false); - DUResponse duRes = (DUResponse) response.getEntity(); - List duData = duRes.getDuData(); - List sortedDuData = new ArrayList<>(duData); - // Sort the DU data by size in descending order to compare with the original. - sortedDuData.sort( - Comparator.comparingLong(DUResponse.DiskUsage::getSize).reversed()); - - for (int i = 0; i < duData.size(); i++) { - assertEquals(sortedDuData.get(i).getSubpath(), - duData.get(i).getSubpath(), - "DU-Sub-Path under " + path + - " should be sorted by descending order of size"); - } - } - - public void setupTestData() throws IOException { - // Helper method to write data to a file - BiConsumer writeFile = (filePath, size) -> { - try (FSDataOutputStream outputStream = fs.create(new Path(filePath))) { - byte[] data = new byte[size.intValue()]; - new Random().nextBytes(data); // Fill with random data - outputStream.write(data); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - - // Create volumes and buckets. - client.getObjectStore().createVolume(VOLUME_A); - OzoneVolume volumeA = client.getObjectStore().getVolume(VOLUME_A); - volumeA.createBucket(BUCKET_A1); - volumeA.createBucket(BUCKET_A2); - volumeA.createBucket(BUCKET_A3); - - client.getObjectStore().createVolume(VOLUME_B); - OzoneVolume volumeB = client.getObjectStore().getVolume(VOLUME_B); - volumeB.createBucket(BUCKET_B1); - - // Define the structure and size in KB for the filesystem - Map fileSystemStructure = new LinkedHashMap<>(); - fileSystemStructure.put("vola/bucketa1/fileA1", 600); - fileSystemStructure.put("vola/bucketa1/fileA2", 800); - fileSystemStructure.put("vola/bucketa1/dirA1", 1500); - fileSystemStructure.put("vola/bucketa1/dirA2", 1700); - fileSystemStructure.put("vola/bucketa1/dirA3", 1300); - fileSystemStructure.put("vola/bucketa2/fileA3", 200); - fileSystemStructure.put("vola/bucketa2/fileA4", 400); - fileSystemStructure.put("vola/bucketa2/dirA4", 1100); - fileSystemStructure.put("vola/bucketa2/dirA5", 1900); - fileSystemStructure.put("vola/bucketa2/dirA6", 2100); - fileSystemStructure.put("vola/bucketa3/fileA5", 500); - fileSystemStructure.put("vola/bucketa3/fileA6", 700); - fileSystemStructure.put("vola/bucketa3/dirA7", 1200); - fileSystemStructure.put("vola/bucketa3/dirA8", 1600); - fileSystemStructure.put("vola/bucketa3/dirA9", 1800); - fileSystemStructure.put("volb/bucketb1/fileB1", 300); - fileSystemStructure.put("volb/bucketb1/fileB2", 500); - fileSystemStructure.put("volb/bucketb1/dirB1", 1400); - fileSystemStructure.put("volb/bucketb1/dirB2", 1800); - fileSystemStructure.put("volb/bucketb1/dirB3", 2200); - - // Create files and directories - for (Map.Entry entry : fileSystemStructure.entrySet()) { - String[] pathParts = entry.getKey().split("/"); - String itemName = pathParts[2]; - int sizeInKB = entry.getValue(); - // Calculate the size in bytes - long sizeInBytes = sizeInKB * 1024L; - - if (itemName.startsWith("file")) { - // Create a file with the specified size - String filePath = "/" + volumeName + "/" + bucketName + "/" + itemName; - writeFile.accept(filePath, sizeInBytes); - } else { - // Create a directory - String dirPath = "/" + volumeName + "/" + bucketName + "/" + itemName; - fs.mkdirs(new Path(dirPath)); - - // Create a file inside the directory to achieve the total specified size - String innerFilePath = dirPath + "/innerFile"; - writeFile.accept(innerFilePath, sizeInBytes); - } - } - } - - private void syncDataFromOM() { - // Sync data from Ozone Manager to Recon. - OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); - impl.syncDataFromOM(); - } - - private static BucketLayout getFSOBucketLayout() { - return BucketLayout.FILE_SYSTEM_OPTIMIZED; - } - -} - diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java new file mode 100644 index 00000000000..bfc086ac118 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java @@ -0,0 +1,407 @@ +package org.apache.hadoop.ozone.recon.api; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; + +import org.apache.hadoop.ozone.recon.common.CommonUtils; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; +import org.junit.jupiter.api.BeforeEach; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; +import static org.aspectj.weaver.tools.cache.SimpleCacheFactory.path; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.ws.rs.core.Response; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.mockito.Mockito.when; + +/** + * Test NSSummary Disk Usage subpath ordering. + */ +public class TestNSSummaryDiskUsageOrdering { + + @TempDir + private Path temporaryFolder; + + private ReconOMMetadataManager reconOMMetadataManager; + private NSSummaryEndpoint nsSummaryEndpoint; + private OzoneConfiguration ozoneConfiguration; + private CommonUtils commonUtils; + private static final String ROOT_PATH = "/"; + private static final String TEST_USER = "TestUser"; + private OMMetadataManager omMetadataManager; + @BeforeEach + public void setUp() throws Exception { + ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, + 100); + omMetadataManager = initializeNewOmMetadataManager( + Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")) + .toFile()); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + getMockOzoneManagerServiceProviderWithFSO(); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .addBinding(OzoneStorageContainerManager.class, + getMockReconSCM()) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(NSSummaryEndpoint.class) + .build(); + ReconNamespaceSummaryManager reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); + + // populate OM DB and reprocess into Recon RocksDB + populateOMDB(); + NSSummaryTaskWithFSO nSSummaryTaskWithFso = + new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, + reconOMMetadataManager, ozoneConfiguration); + nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); + commonUtils = new CommonUtils(); + } + + /** + * Create a new OM Metadata manager instance with one user, one vol, and two + * buckets. + * @throws IOException ioEx + */ + private static OMMetadataManager initializeNewOmMetadataManager( + File omDbDir) + throws IOException { + OzoneConfiguration omConfiguration = new OzoneConfiguration(); + omConfiguration.set(OZONE_OM_DB_DIRS, + omDbDir.getAbsolutePath()); + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( + omConfiguration, null); + return omMetadataManager; + } + + @Test + public void testDiskUsageOrderingForRoot() throws Exception { + // root level DU + // Verify the ordering of subpaths under the root + verifyOrdering(ROOT_PATH); + } + + @Test + public void testDiskUsageOrderingForVolume() throws Exception { + // volume level DU + // Verify the ordering of subpaths under the volume + verifyOrdering("/volA"); + verifyOrdering("/volB"); + } + + @Test + public void testDiskUsageOrderingForBucket() throws Exception { + // bucket level DU + // Verify the ordering of subpaths under the bucket + verifyOrdering("/volA/bucketA1"); + verifyOrdering("/volA/bucketA2"); + verifyOrdering("/volA/bucketA3"); + verifyOrdering("/volB/bucketB1"); + } + + private void verifyOrdering(String path) + throws IOException { + Response response = + nsSummaryEndpoint.getDiskUsage(path, true, false); + DUResponse duRes = (DUResponse) response.getEntity(); + List duData = duRes.getDuData(); + List sortedDuData = new ArrayList<>(duData); + // Sort the DU data by size in descending order to compare with the original. + sortedDuData.sort( + Comparator.comparingLong(DUResponse.DiskUsage::getSize).reversed()); + + for (int i = 0; i < duData.size(); i++) { + assertEquals(sortedDuData.get(i).getSubpath(), + duData.get(i).getSubpath(), + "DU-Sub-Path under " + path + + " should be sorted by descending order of size"); + } + } + + /** + * Tests the NSSummaryEndpoint for a given volume, bucket, and directory structure. + * The test setup mimics the following filesystem structure with specified sizes: + * + * root + * ├── volA + * │ ├── bucketA1 + * │ │ ├── fileA1 (Size: 600KB) + * │ │ ├── fileA2 (Size: 800KB) + * │ │ ├── dirA1 (Total Size: 1500KB) + * │ │ ├── dirA2 (Total Size: 1700KB) + * │ │ └── dirA3 (Total Size: 1300KB) + * │ ├── bucketA2 + * │ │ ├── fileA3 (Size: 200KB) + * │ │ ├── fileA4 (Size: 400KB) + * │ │ ├── dirA4 (Total Size: 1100KB) + * │ │ ├── dirA5 (Total Size: 1900KB) + * │ │ └── dirA6 (Total Size: 2100KB) + * │ └── bucketA3 + * │ ├── fileA5 (Size: 500KB) + * │ ├── fileA6 (Size: 700KB) + * │ ├── dirA7 (Total Size: 1200KB) + * │ ├── dirA8 (Total Size: 1600KB) + * │ └── dirA9 (Total Size: 1800KB) + * └── volB + * └── bucketB1 + * ├── fileB1 (Size: 300KB) + * ├── fileB2 (Size: 500KB) + * ├── dirB1 (Total Size: 1400KB) + * ├── dirB2 (Total Size: 1800KB) + * └── dirB3 (Total Size: 2200KB) + * + * @throws Exception + */ + private void populateOMDB() throws Exception { + // Create Volumes + long volAObjectId = createVolume("volA"); + long volBObjectId = createVolume("volB"); + + // Create Buckets in volA + long bucketA1ObjectId = + createBucket("volA", "bucketA1", 600 + 800 + 1500 + 1700 + 1300); + long bucketA2ObjectId = + createBucket("volA", "bucketA2", 200 + 400 + 1100 + 1900 + 2100); + long bucketA3ObjectId = + createBucket("volA", "bucketA3", 500 + 700 + 1200 + 1600 + 1800); + + // Create Bucket in volB + long bucketB1ObjectId = + createBucket("volB", "bucketB1", 300 + 500 + 1400 + 1800 + 2200); + + // Create Directories and Files under bucketA1 + long dirA1ObjectId = + createDirectory(bucketA1ObjectId, bucketA1ObjectId, volAObjectId, + "dirA1"); + long dirA2ObjectId = + createDirectory(bucketA1ObjectId, bucketA1ObjectId, volAObjectId, + "dirA2"); + long dirA3ObjectId = + createDirectory(bucketA1ObjectId, bucketA1ObjectId, volAObjectId, + "dirA3"); + + // Files directly under bucketA1 + createFile("fileA1", "bucketA1", "volA", "fileA1", bucketA1ObjectId, + bucketA1ObjectId, volAObjectId, 600 * 1024); + createFile("fileA2", "bucketA1", "volA", "fileA2", bucketA1ObjectId, + bucketA1ObjectId, volAObjectId, 800 * 1024); + + // Create Directories and Files under bucketA2 + long dirA4ObjectId = + createDirectory(bucketA2ObjectId, bucketA2ObjectId, volAObjectId, + "dirA4"); + long dirA5ObjectId = + createDirectory(bucketA2ObjectId, bucketA2ObjectId, volAObjectId, + "dirA5"); + long dirA6ObjectId = + createDirectory(bucketA2ObjectId, bucketA2ObjectId, volAObjectId, + "dirA6"); + + // Files directly under bucketA2 + createFile("fileA3", "bucketA2", "volA", "fileA3", bucketA2ObjectId, + bucketA2ObjectId, volAObjectId, 200 * 1024); + createFile("fileA4", "bucketA2", "volA", "fileA4", bucketA2ObjectId, + bucketA2ObjectId, volAObjectId, 400 * 1024); + + // Create Directories and Files under bucketA3 + long dirA7ObjectId = + createDirectory(bucketA3ObjectId, bucketA3ObjectId, volAObjectId, + "dirA7"); + long dirA8ObjectId = + createDirectory(bucketA3ObjectId, bucketA3ObjectId, volAObjectId, + "dirA8"); + long dirA9ObjectId = + createDirectory(bucketA3ObjectId, bucketA3ObjectId, volAObjectId, + "dirA9"); + + // Files directly under bucketA3 + createFile("fileA5", "bucketA3", "volA", "fileA5", bucketA3ObjectId, + bucketA3ObjectId, volAObjectId, 500 * 1024); + createFile("fileA6", "bucketA3", "volA", "fileA6", bucketA3ObjectId, + bucketA3ObjectId, volAObjectId, 700 * 1024); + + // Create Directories and Files under bucketB1 + long dirB1ObjectId = + createDirectory(bucketB1ObjectId, bucketB1ObjectId, volBObjectId, + "dirB1"); + long dirB2ObjectId = + createDirectory(bucketB1ObjectId, bucketB1ObjectId, volBObjectId, + "dirB2"); + long dirB3ObjectId = + createDirectory(bucketB1ObjectId, bucketB1ObjectId, volBObjectId, + "dirB3"); + + // Files directly under bucketB1 + createFile("fileB1", "bucketB1", "volB", "fileB1", bucketB1ObjectId, + bucketB1ObjectId, volBObjectId, 300 * 1024); + createFile("fileB2", "bucketB1", "volB", "fileB2", bucketB1ObjectId, + bucketB1ObjectId, volBObjectId, 500 * 1024); + + // Create Inner files under directories + createFile("dirA1/innerFile", "bucketA1", "volA", "innerFile", + dirA1ObjectId, bucketA1ObjectId, volAObjectId, 1500 * 1024); + createFile("dirA2/innerFile", "bucketA1", "volA", "innerFile", + dirA2ObjectId, bucketA1ObjectId, volAObjectId, 1700 * 1024); + createFile("dirA3/innerFile", "bucketA1", "volA", "innerFile", + dirA3ObjectId, bucketA1ObjectId, volAObjectId, 1300 * 1024); + createFile("dirA4/innerFile", "bucketA2", "volA", "innerFile", + dirA4ObjectId, bucketA2ObjectId, volAObjectId, 1100 * 1024); + createFile("dirA5/innerFile", "bucketA2", "volA", "innerFile", + dirA5ObjectId, bucketA2ObjectId, volAObjectId, 1900 * 1024); + createFile("dirA6/innerFile", "bucketA2", "volA", "innerFile", + dirA6ObjectId, bucketA2ObjectId, volAObjectId, 2100 * 1024); + createFile("dirA7/innerFile", "bucketA3", "volA", "innerFile", + dirA7ObjectId, bucketA3ObjectId, volAObjectId, 1200 * 1024); + createFile("dirA8/innerFile", "bucketA3", "volA", "innerFile", + dirA8ObjectId, bucketA3ObjectId, volAObjectId, 1600 * 1024); + createFile("dirA9/innerFile", "bucketA3", "volA", "innerFile", + dirA9ObjectId, bucketA3ObjectId, volAObjectId, 1800 * 1024); + createFile("dirB1/innerFile", "bucketB1", "volB", "innerFile", + dirB1ObjectId, bucketB1ObjectId, volBObjectId, 1400 * 1024); + createFile("dirB2/innerFile", "bucketB1", "volB", "innerFile", + dirB2ObjectId, bucketB1ObjectId, volBObjectId, 1800 * 1024); + createFile("dirB3/innerFile", "bucketB1", "volB", "innerFile", + dirB3ObjectId, bucketB1ObjectId, volBObjectId, 2200 * 1024); + } + + /** + * Create a volume and add it to the Volume Table. + * @return volume Object ID + * @throws IOException + */ + private long createVolume(String volumeName) throws Exception { + String volumeKey = reconOMMetadataManager.getVolumeKey(volumeName); + long volumeId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Generate positive ID + OmVolumeArgs args = OmVolumeArgs.newBuilder() + .setObjectID(volumeId) + .setVolume(volumeName) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .build(); + + reconOMMetadataManager.getVolumeTable().put(volumeKey, args); + return volumeId; + } + + /** + * Create a bucket and add it to the Bucket Table. + * @return bucket Object ID + * @throws IOException + */ + private long createBucket(String volumeName, String bucketName, long dataSize) + throws Exception { + String bucketKey = + reconOMMetadataManager.getBucketKey(volumeName, bucketName); + long bucketId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Generate positive ID + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setObjectID(bucketId) + .setBucketLayout(getBucketLayout()) + .setUsedBytes(dataSize) + .build(); + + reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); + return bucketId; + } + + /** + * Create a directory and add it to the Directory Table. + * @return directory Object ID + * @throws IOException + */ + private long createDirectory(long parentObjectId, + long bucketObjectId, + long volumeObjectId, + String dirName) throws IOException { + long objectId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Ensure positive ID + writeDirToOm(reconOMMetadataManager, objectId, parentObjectId, + bucketObjectId, + volumeObjectId, dirName); + return objectId; + } + + /** + * Create a file and add it to the File Table. + * @return file Object ID + * @throws IOException + */ + @SuppressWarnings("checkstyle:ParameterNumber") + private long createFile(String key, + String bucket, + String volume, + String fileName, + long parentObjectId, + long bucketObjectId, + long volumeObjectId, + long dataSize) throws IOException { + long objectId = UUID.randomUUID().getMostSignificantBits() & + Long.MAX_VALUE; // Ensure positive ID + writeKeyToOm(reconOMMetadataManager, key, bucket, volume, fileName, + objectId, + parentObjectId, bucketObjectId, volumeObjectId, dataSize, + getBucketLayout()); + return objectId; + } + + private static ReconStorageContainerManagerFacade getMockReconSCM() + throws ContainerNotFoundException { + ReconStorageContainerManagerFacade reconSCM = + mock(ReconStorageContainerManagerFacade.class); + ContainerManager containerManager = mock(ContainerManager.class); + + when(reconSCM.getContainerManager()).thenReturn(containerManager); + ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); + when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); + return reconSCM; + } + + private static BucketLayout getBucketLayout() { + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + } +} From 32625dab0a310cba6a1455433e5d3bc46c339977 Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 5 Mar 2024 10:21:38 +0530 Subject: [PATCH 05/10] Added licence --- .../api/TestNSSummaryDiskUsageOrdering.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java index bfc086ac118..ad883239c11 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.recon.api; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -58,7 +76,6 @@ public class TestNSSummaryDiskUsageOrdering { private ReconOMMetadataManager reconOMMetadataManager; private NSSummaryEndpoint nsSummaryEndpoint; private OzoneConfiguration ozoneConfiguration; - private CommonUtils commonUtils; private static final String ROOT_PATH = "/"; private static final String TEST_USER = "TestUser"; private OMMetadataManager omMetadataManager; @@ -97,7 +114,6 @@ public void setUp() throws Exception { new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, reconOMMetadataManager, ozoneConfiguration); nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); - commonUtils = new CommonUtils(); } /** From 1c292f101a584f67910a0daf2cd46e0630661dfb Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 5 Mar 2024 11:15:13 +0530 Subject: [PATCH 06/10] Fixed checkstyle isssues --- .../hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java index ad883239c11..ea003a435ce 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java @@ -25,7 +25,6 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.common.CommonUtils; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; @@ -35,7 +34,6 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; -import static org.aspectj.weaver.tools.cache.SimpleCacheFactory.path; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import org.apache.hadoop.hdds.scm.container.ContainerManager; From 994a9e5fbf6b8040227aeb3fb68ab6377a9d20ef Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 8 Mar 2024 21:55:07 +0530 Subject: [PATCH 07/10] Added a flag to enable/disable sorting --- .../ozone/recon/api/NSSummaryEndpoint.java | 11 +- .../api/handlers/BucketEntityHandler.java | 57 +++++-- .../api/handlers/DirectoryEntityHandler.java | 41 +++-- .../recon/api/handlers/EntityHandler.java | 2 +- .../recon/api/handlers/KeyEntityHandler.java | 2 +- .../recon/api/handlers/RootEntityHandler.java | 40 ++++- .../api/handlers/UnknownEntityHandler.java | 2 +- .../api/handlers/VolumeEntityHandler.java | 10 +- .../ozone/recon/heatmap/HeatMapUtil.java | 2 +- .../api/TestNSSummaryDiskUsageOrdering.java | 35 +++-- .../api/TestNSSummaryEndpointWithFSO.java | 147 ++++++++++-------- .../api/TestNSSummaryEndpointWithLegacy.java | 28 ++-- .../api/TestNSSummaryEndpointWithOBS.java | 20 +-- 13 files changed, 236 insertions(+), 161 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java index 5b104c46115..e2b75658629 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java @@ -101,6 +101,7 @@ public Response getBasicInfo( * @param path request path * @param listFile show subpath/disk usage for each key * @param withReplica count actual DU with replication + * @param sort whether to sort the subpaths by their sizes in descending order * @return DU response * @throws IOException */ @@ -108,10 +109,9 @@ public Response getBasicInfo( @Path("/du") @SuppressWarnings("methodlength") public Response getDiskUsage(@QueryParam("path") String path, - @DefaultValue("false") - @QueryParam("files") boolean listFile, - @DefaultValue("false") - @QueryParam("replica") boolean withReplica) + @DefaultValue("false") @QueryParam("files") boolean listFile, + @DefaultValue("false") @QueryParam("replica") boolean withReplica, + @DefaultValue("true") @QueryParam("sortSubPaths") boolean sortSubpaths) throws IOException { if (path == null || path.length() == 0) { return Response.status(Response.Status.BAD_REQUEST).build(); @@ -127,8 +127,7 @@ public Response getDiskUsage(@QueryParam("path") String path, reconNamespaceSummaryManager, omMetadataManager, reconSCM, path); - duResponse = handler.getDuResponse( - listFile, withReplica); + duResponse = handler.getDuResponse(listFile, withReplica, sortSubpaths); return Response.ok(duResponse).build(); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java index 27229e8a430..904d7c36c20 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.ArrayList; @@ -91,7 +92,7 @@ private BucketObjectDBInfo getBucketObjDbInfo(String[] names) @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sortSubpaths) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); @@ -110,23 +111,12 @@ public DUResponse getDuResponse( Set bucketSubdirs = bucketNSSummary.getChildDir(); duResponse.setKeySize(bucketNSSummary.getSizeOfFiles()); - // Map to hold sizes for each subdir for sorting - Map subdirSizes = new HashMap<>(); - for (long subdirObjectId : bucketSubdirs) { - long dataSize = getTotalSize(subdirObjectId); - subdirSizes.put(subdirObjectId, dataSize); - } - - // Sort subdirs based on their size in descending order - List sortedSubdirObjectIds = bucketSubdirs.stream().sorted( - Comparator.comparingLong( - subdirObjectId -> subdirSizes.get(subdirObjectId)).reversed()) - .collect(Collectors.toList()); + SortedResult result = getSortedResult(bucketSubdirs, sortSubpaths); List dirDUData = new ArrayList<>(); long bucketDataSize = duResponse.getKeySize(); long bucketDataSizeWithReplica = 0L; - for (long subdirObjectId : sortedSubdirObjectIds) { + for (long subdirObjectId : result.sortedSubdirObjectIds) { NSSummary subdirNSSummary = getReconNamespaceSummaryManager() .getNSSummary(subdirObjectId); @@ -137,7 +127,7 @@ public DUResponse getDuResponse( // format with leading slash and without trailing slash DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); diskUsage.setSubpath(subpath); - long dataSize = subdirSizes.get(subdirObjectId); + long dataSize = result.subdirSizes.get(subdirObjectId); bucketDataSize += dataSize; @@ -195,4 +185,41 @@ public FileSizeDistributionResponse getDistResponse() return distResponse; } + @NotNull + private SortedResult getSortedResult(Set bucketSubdirs, boolean sortSubpaths) + throws IOException { + // Map to hold sizes for each subdir + Map subdirSizes = new HashMap<>(); + for (long subdirObjectId : bucketSubdirs) { + long dataSize = getTotalSize(subdirObjectId); + subdirSizes.put(subdirObjectId, dataSize); + } + + List subdirObjectIds; + if (sortSubpaths) { + // Sort subdirs based on their size in descending order if sortSubpaths is true + subdirObjectIds = bucketSubdirs.stream().sorted( + Comparator.comparingLong( + subdirObjectId -> subdirSizes.get(subdirObjectId)).reversed()) + .collect(Collectors.toList()); + } else { + // If sortSubpaths is false, return the subdirs as is, without sorting + subdirObjectIds = new ArrayList<>(bucketSubdirs); + } + return new SortedResult(subdirSizes, subdirObjectIds); + } + + + private static class SortedResult { + public final Map subdirSizes; + public final List sortedSubdirObjectIds; + + public SortedResult(Map subdirSizes, + List sortedSubdirObjectIds) { + this.subdirSizes = subdirSizes; + this.sortedSubdirObjectIds = sortedSubdirObjectIds; + } + } + + } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index 1a19bd79b65..5d31cf1cdf8 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -85,7 +85,7 @@ private ObjectDBInfo getDirectoryObjDbInfo(String[] names) @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sortSubPaths) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); @@ -106,22 +106,11 @@ public DUResponse getDuResponse( long dirDataSize = duResponse.getKeySize(); long dirDataSizeWithReplica = 0L; - // Map to hold sizes for each subdir for sorting - Map subdirSizes = new HashMap<>(); - for (long subdirObjectId : subdirs) { - long dataSize = getTotalSize(subdirObjectId); - subdirSizes.put(subdirObjectId, dataSize); - } - - // Sort subdirs based on their size in descending order - List sortedSubdirObjectIds = subdirs.stream() - .sorted(Comparator.comparingLong( - subdirObjectId -> subdirSizes.get(subdirObjectId)).reversed()) - .collect(Collectors.toList()); + SortedSubdirsResult sortedResult = new SortedSubdirsResult(subdirs, sortSubPaths); List subdirDUData = new ArrayList<>(); // iterate all subdirectories to get disk usage data - for (long subdirObjectId : sortedSubdirObjectIds) { + for (long subdirObjectId : sortedResult.sortedSubdirObjectIds) { NSSummary subdirNSSummary = getReconNamespaceSummaryManager().getNSSummary(subdirObjectId); // for the subdirName we need the subdir filename, not the key name @@ -147,7 +136,7 @@ public DUResponse getDuResponse( DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); // reformat the response diskUsage.setSubpath(subpath); - long dataSize = subdirSizes.get(subdirObjectId); + long dataSize = sortedResult.subdirSizes.get(subdirObjectId); dirDataSize += dataSize; if (withReplica) { @@ -201,4 +190,26 @@ public FileSizeDistributionResponse getDistResponse() return distResponse; } + private class SortedSubdirsResult { + public final List sortedSubdirObjectIds; + public final Map subdirSizes; + + public SortedSubdirsResult(Set subdirs, boolean sortSubPaths) throws IOException { + this.subdirSizes = new HashMap<>(); + for (long subdirObjectId : subdirs) { + long dataSize = getTotalSize(subdirObjectId); + this.subdirSizes.put(subdirObjectId, dataSize); + } + + if (sortSubPaths) { + this.sortedSubdirObjectIds = subdirs.stream() + .sorted(Comparator.comparingLong(subdirObjectId -> this.subdirSizes.get(subdirObjectId)).reversed()) + .collect(Collectors.toList()); + } else { + this.sortedSubdirObjectIds = new ArrayList<>(subdirs); + } + } + } + + } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java index d12c7b6545a..585385c852b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java @@ -69,7 +69,7 @@ public abstract NamespaceSummaryResponse getSummaryResponse() throws IOException; public abstract DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sort) throws IOException; public abstract QuotaUsageResponse getQuotaResponse() diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java index a687bf3d0bd..8ea26fd2846 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java @@ -71,7 +71,7 @@ private ObjectDBInfo getKeyDbObjectInfo(String[] names) @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sort) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index ef1a93cc43a..7e08f6dcb4e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -37,6 +37,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import java.io.IOException; import java.util.ArrayList; @@ -90,7 +91,7 @@ private ObjectDBInfo getPrefixObjDbInfo() @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sortSubPaths) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); @@ -111,14 +112,13 @@ public DUResponse getDuResponse( volumeSizes.put(volume.getVolume(), totalVolumeSize); } - // Sort volumes based on the total size in descending order - volumes.sort((v1, v2) -> volumeSizes.get(v2.getVolume()) - .compareTo(volumeSizes.get(v1.getVolume()))); - + // Apply sorting based on sortSubPaths flag + SortedResult sortedResult = getSortedResult(volumes, volumeSizes, sortSubPaths); List volumeDuData = new ArrayList<>(); long totalDataSize = 0L; long totalDataSizeWithReplica = 0L; - for (OmVolumeArgs volume: volumes) { + + for (OmVolumeArgs volume : sortedResult.sortedVolumes) { String volumeName = volume.getVolume(); String subpath = omMetadataManager.getVolumeKey(volumeName); DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); @@ -167,7 +167,7 @@ public QuotaUsageResponse getQuotaResponse() QuotaUsageResponse quotaUsageResponse = new QuotaUsageResponse(); SCMNodeStat stats = getReconSCM().getScmNodeManager().getStats(); long quotaInBytes = stats.getCapacity().get(); - long quotaUsedInBytes = getDuResponse(true, true).getSizeWithReplica(); + long quotaUsedInBytes = getDuResponse(true, true, false).getSizeWithReplica(); quotaUsageResponse.setQuota(quotaInBytes); quotaUsageResponse.setQuotaUsed(quotaUsedInBytes); return quotaUsageResponse; @@ -195,4 +195,30 @@ public FileSizeDistributionResponse getDistResponse() return distResponse; } + private SortedResult getSortedResult(List volumes, + Map volumeSizes, + boolean sortSubPaths) { + if (sortSubPaths) { + // Sort volumes based on the total size in descending order + List sortedVolumes = volumes.stream() + .sorted((v1, v2) -> volumeSizes.get(v2.getVolume()) + .compareTo(volumeSizes.get(v1.getVolume()))) + .collect(Collectors.toList()); + return new SortedResult(sortedVolumes, volumeSizes); + } else { + // Return volumes as is without sorting + return new SortedResult(new ArrayList<>(volumes), volumeSizes); + } + } + + private static class SortedResult { + public final List sortedVolumes; + public final Map volumeSizes; + + public SortedResult(List sortedVolumes, Map volumeSizes) { + this.sortedVolumes = sortedVolumes; + this.volumeSizes = volumeSizes; + } + } + } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java index b5a5bd9a0be..ab61ec38e8b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java @@ -51,7 +51,7 @@ public NamespaceSummaryResponse getSummaryResponse() @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sort) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setStatus(ResponseStatus.PATH_NOT_FOUND); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java index 056fa89fd9a..8ae7b6fe763 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java @@ -92,7 +92,7 @@ private VolumeObjectDBInfo getVolumeObjDbInfo(String[] names) @Override public DUResponse getDuResponse( - boolean listFile, boolean withReplica) + boolean listFile, boolean withReplica, boolean sortSubPaths) throws IOException { DUResponse duResponse = new DUResponse(); duResponse.setPath(getNormalizedPath()); @@ -101,9 +101,11 @@ public DUResponse getDuResponse( List buckets = getOmMetadataManager(). listBucketsUnderVolume(volName); - // Sort buckets in descending order by size - buckets.sort( - (b1, b2) -> Long.compare(b2.getUsedBytes(), b1.getUsedBytes())); + if (sortSubPaths) { + // Sort buckets in descending order by size if sortSubPaths is true + buckets.sort( + (b1, b2) -> Long.compare(b2.getUsedBytes(), b1.getUsedBytes())); + } duResponse.setCount(buckets.size()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java index 2f3de1debcd..57f7686263f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java @@ -71,7 +71,7 @@ private long getEntitySize(String path) throws IOException { EntityHandler.getEntityHandler(reconNamespaceSummaryManager, omMetadataManager, reconSCM, path); if (null != entityHandler) { - DUResponse duResponse = entityHandler.getDuResponse(false, false); + DUResponse duResponse = entityHandler.getDuResponse(false, false, false); if (null != duResponse && duResponse.getStatus() == ResponseStatus.OK) { return duResponse.getSize(); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java index ea003a435ce..1adf7bfccf7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; @@ -158,7 +159,7 @@ public void testDiskUsageOrderingForBucket() throws Exception { private void verifyOrdering(String path) throws IOException { Response response = - nsSummaryEndpoint.getDiskUsage(path, true, false); + nsSummaryEndpoint.getDiskUsage(path, true, false,true); DUResponse duRes = (DUResponse) response.getEntity(); List duData = duRes.getDuData(); List sortedDuData = new ArrayList<>(duData); @@ -182,27 +183,27 @@ private void verifyOrdering(String path) * ├── volA * │ ├── bucketA1 * │ │ ├── fileA1 (Size: 600KB) - * │ │ ├── fileA2 (Size: 800KB) + * │ │ ├── fileA2 (Size: 80KB) * │ │ ├── dirA1 (Total Size: 1500KB) * │ │ ├── dirA2 (Total Size: 1700KB) * │ │ └── dirA3 (Total Size: 1300KB) * │ ├── bucketA2 * │ │ ├── fileA3 (Size: 200KB) - * │ │ ├── fileA4 (Size: 400KB) + * │ │ ├── fileA4 (Size: 4000KB) * │ │ ├── dirA4 (Total Size: 1100KB) * │ │ ├── dirA5 (Total Size: 1900KB) - * │ │ └── dirA6 (Total Size: 2100KB) + * │ │ └── dirA6 (Total Size: 210KB) * │ └── bucketA3 - * │ ├── fileA5 (Size: 500KB) + * │ ├── fileA5 (Size: 5000KB) * │ ├── fileA6 (Size: 700KB) * │ ├── dirA7 (Total Size: 1200KB) * │ ├── dirA8 (Total Size: 1600KB) - * │ └── dirA9 (Total Size: 1800KB) + * │ └── dirA9 (Total Size: 180KB) * └── volB * └── bucketB1 * ├── fileB1 (Size: 300KB) * ├── fileB2 (Size: 500KB) - * ├── dirB1 (Total Size: 1400KB) + * ├── dirB1 (Total Size: 14000KB) * ├── dirB2 (Total Size: 1800KB) * └── dirB3 (Total Size: 2200KB) * @@ -215,15 +216,15 @@ private void populateOMDB() throws Exception { // Create Buckets in volA long bucketA1ObjectId = - createBucket("volA", "bucketA1", 600 + 800 + 1500 + 1700 + 1300); + createBucket("volA", "bucketA1", 600 + 80 + 1500 + 1700 + 1300); long bucketA2ObjectId = - createBucket("volA", "bucketA2", 200 + 400 + 1100 + 1900 + 2100); + createBucket("volA", "bucketA2", 200 + 4000 + 1100 + 1900 + 210); long bucketA3ObjectId = - createBucket("volA", "bucketA3", 500 + 700 + 1200 + 1600 + 1800); + createBucket("volA", "bucketA3", 5000 + 700 + 1200 + 1600 + 180); // Create Bucket in volB long bucketB1ObjectId = - createBucket("volB", "bucketB1", 300 + 500 + 1400 + 1800 + 2200); + createBucket("volB", "bucketB1", 300 + 500 + 14000 + 1800 + 2200); // Create Directories and Files under bucketA1 long dirA1ObjectId = @@ -240,7 +241,7 @@ private void populateOMDB() throws Exception { createFile("fileA1", "bucketA1", "volA", "fileA1", bucketA1ObjectId, bucketA1ObjectId, volAObjectId, 600 * 1024); createFile("fileA2", "bucketA1", "volA", "fileA2", bucketA1ObjectId, - bucketA1ObjectId, volAObjectId, 800 * 1024); + bucketA1ObjectId, volAObjectId, 80 * 1024); // Create Directories and Files under bucketA2 long dirA4ObjectId = @@ -257,7 +258,7 @@ private void populateOMDB() throws Exception { createFile("fileA3", "bucketA2", "volA", "fileA3", bucketA2ObjectId, bucketA2ObjectId, volAObjectId, 200 * 1024); createFile("fileA4", "bucketA2", "volA", "fileA4", bucketA2ObjectId, - bucketA2ObjectId, volAObjectId, 400 * 1024); + bucketA2ObjectId, volAObjectId, 4000 * 1024); // Create Directories and Files under bucketA3 long dirA7ObjectId = @@ -272,7 +273,7 @@ private void populateOMDB() throws Exception { // Files directly under bucketA3 createFile("fileA5", "bucketA3", "volA", "fileA5", bucketA3ObjectId, - bucketA3ObjectId, volAObjectId, 500 * 1024); + bucketA3ObjectId, volAObjectId, 5000 * 1024); createFile("fileA6", "bucketA3", "volA", "fileA6", bucketA3ObjectId, bucketA3ObjectId, volAObjectId, 700 * 1024); @@ -305,15 +306,15 @@ private void populateOMDB() throws Exception { createFile("dirA5/innerFile", "bucketA2", "volA", "innerFile", dirA5ObjectId, bucketA2ObjectId, volAObjectId, 1900 * 1024); createFile("dirA6/innerFile", "bucketA2", "volA", "innerFile", - dirA6ObjectId, bucketA2ObjectId, volAObjectId, 2100 * 1024); + dirA6ObjectId, bucketA2ObjectId, volAObjectId, 210 * 1024); createFile("dirA7/innerFile", "bucketA3", "volA", "innerFile", dirA7ObjectId, bucketA3ObjectId, volAObjectId, 1200 * 1024); createFile("dirA8/innerFile", "bucketA3", "volA", "innerFile", dirA8ObjectId, bucketA3ObjectId, volAObjectId, 1600 * 1024); createFile("dirA9/innerFile", "bucketA3", "volA", "innerFile", - dirA9ObjectId, bucketA3ObjectId, volAObjectId, 1800 * 1024); + dirA9ObjectId, bucketA3ObjectId, volAObjectId, 180 * 1024); createFile("dirB1/innerFile", "bucketB1", "volB", "innerFile", - dirB1ObjectId, bucketB1ObjectId, volBObjectId, 1400 * 1024); + dirB1ObjectId, bucketB1ObjectId, volBObjectId, 14000 * 1024); createFile("dirB2/innerFile", "bucketB1", "volB", "innerFile", dirB2ObjectId, bucketB1ObjectId, volBObjectId, 1800 * 1024); createFile("dirB3/innerFile", "bucketB1", "volB", "innerFile", diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index cbe850b918f..326ed035a71 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -89,23 +89,23 @@ /** * Test for NSSummary REST APIs with FSO. * We tested on a mini file system with the following setting: - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ - * dir2 dir3 dir4 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 + * vol + * / \ + * bucket1 bucket2 + * / \ / \ + * file1 dir1 file4 file5 + * / \ \ + * dir2 dir3 dir4 + * / \ \ + * file2 file3 file6 + * ---------------------------------------- + * vol2 + * / \ + * bucket3 bucket4 + * / \ / + * file8 dir5 file11 + * / \ + * file9 file10 * This is a test for the Rest APIs only. We have tested NSSummaryTask before, * so there is no need to test process() on DB's updates */ @@ -119,10 +119,10 @@ public class TestNSSummaryEndpointWithFSO { private CommonUtils commonUtils; private static final String TEST_PATH_UTILITY = - "/vol1/buck1/a/b/c/d/e/file1.txt"; + "/vol1/buck1/a/b/c/d/e/file1.txt"; private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; private static final String[] TEST_NAMES = - new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; + new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; // Object names in FSO-enabled format @@ -197,12 +197,12 @@ public class TestNSSummaryEndpointWithFSO { private static final long CONTAINER_SIX_ID = 6L; // replication factors - private static final int CONTAINER_ONE_REPLICA_COUNT = 3; - private static final int CONTAINER_TWO_REPLICA_COUNT = 2; - private static final int CONTAINER_THREE_REPLICA_COUNT = 4; - private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; - private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; - private static final int CONTAINER_SIX_REPLICA_COUNT = 3; + private static final int CONTAINER_ONE_REPLICA_COUNT = 3; + private static final int CONTAINER_TWO_REPLICA_COUNT = 2; + private static final int CONTAINER_THREE_REPLICA_COUNT = 4; + private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; + private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; + private static final int CONTAINER_SIX_REPLICA_COUNT = 3; // block lengths private static final long BLOCK_ONE_LENGTH = 1000L; @@ -227,39 +227,39 @@ public class TestNSSummaryEndpointWithFSO { private static final long FILE1_SIZE_WITH_REPLICA = getReplicatedSize(KEY_ONE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE2_SIZE_WITH_REPLICA = getReplicatedSize(KEY_TWO_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE3_SIZE_WITH_REPLICA = getReplicatedSize(KEY_THREE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE4_SIZE_WITH_REPLICA = getReplicatedSize(KEY_FOUR_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE5_SIZE_WITH_REPLICA = getReplicatedSize(KEY_FIVE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE6_SIZE_WITH_REPLICA = getReplicatedSize(KEY_SIX_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE7_SIZE_WITH_REPLICA = getReplicatedSize(KEY_SEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE8_SIZE_WITH_REPLICA = getReplicatedSize(KEY_EIGHT_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE9_SIZE_WITH_REPLICA = getReplicatedSize(KEY_NINE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE10_SIZE_WITH_REPLICA = getReplicatedSize(KEY_TEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE11_SIZE_WITH_REPLICA = getReplicatedSize(KEY_ELEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA - = FILE7_SIZE_WITH_REPLICA; + = FILE7_SIZE_WITH_REPLICA; private static final long MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT = FILE1_SIZE_WITH_REPLICA @@ -336,19 +336,19 @@ public class TestNSSummaryEndpointWithFSO { KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; private static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE; + KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE; private static final long VOL_TWO_DATA_SIZE = KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; private static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE; + KEY_THREE_SIZE + KEY_SIX_SIZE; private static final long BUCKET_TWO_DATA_SIZE = - KEY_FOUR_SIZE + KEY_FIVE_SIZE; + KEY_FOUR_SIZE + KEY_FIVE_SIZE; private static final long DIR_ONE_DATA_SIZE = KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE; + KEY_THREE_SIZE + KEY_SIX_SIZE; @BeforeEach public void setUp() throws Exception { @@ -364,17 +364,17 @@ public void setUp() throws Exception { Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(ozoneManagerServiceProvider) - .withReconSqlDb() - .withContainerDB() - .addBinding(OzoneStorageContainerManager.class, - getMockReconSCM()) - .addBinding(StorageContainerServiceProvider.class, - mock(StorageContainerServiceProviderImpl.class)) - .addBinding(NSSummaryEndpoint.class) - .build(); + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .addBinding(OzoneStorageContainerManager.class, + getMockReconSCM()) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(NSSummaryEndpoint.class) + .build(); ReconNamespaceSummaryManager reconNamespaceSummaryManager = reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); @@ -449,7 +449,7 @@ public void testGetBasicInfoKey() throws Exception { public void testDiskUsageRoot() throws Exception { // root level DU Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false); + false, false, false); DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); assertEquals(2, duRootRes.getCount()); List duRootData = duRootRes.getDuData(); @@ -463,17 +463,18 @@ public void testDiskUsageRoot() throws Exception { assertEquals(VOL_DATA_SIZE, duVol1.getSize()); assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); } + @Test public void testDiskUsageVolume() throws Exception { // volume level DU Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false); + false, false, false); DUResponse duVolRes = (DUResponse) volResponse.getEntity(); assertEquals(2, duVolRes.getCount()); List duData = duVolRes.getDuData(); // sort based on subpath Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); DUResponse.DiskUsage duBucket1 = duData.get(0); DUResponse.DiskUsage duBucket2 = duData.get(1); assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); @@ -482,11 +483,12 @@ public void testDiskUsageVolume() throws Exception { assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); } + @Test public void testDiskUsageBucket() throws Exception { // bucket level DU Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false); + false, false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(1, duBucketResponse.getCount()); DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); @@ -494,11 +496,12 @@ public void testDiskUsageBucket() throws Exception { assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); } + @Test public void testDiskUsageDir() throws Exception { // dir level DU Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, false); + false, false, false); DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); assertEquals(3, duDirReponse.getCount()); List duSubDir = duDirReponse.getDuData(); @@ -517,35 +520,37 @@ public void testDiskUsageDir() throws Exception { assertEquals(KEY_SIX_SIZE, duDir4.getSize()); } + @Test public void testDiskUsageKey() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false); + false, false, false); DUResponse keyObj = (DUResponse) keyResponse.getEntity(); assertEquals(0, keyObj.getCount()); assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); } + @Test public void testDiskUsageUnknown() throws Exception { // invalid path check Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false); + false, false, false); DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); + invalidObj.getStatus()); } @Test public void testDiskUsageWithReplication() throws Exception { setUpMultiBlockKey(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, - replicaDUResponse.getSizeWithReplica()); + replicaDUResponse.getSizeWithReplica()); } @Test @@ -553,7 +558,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); // withReplica is true Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, @@ -567,7 +572,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { public void testDataSizeUnderVolWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, @@ -580,7 +585,7 @@ public void testDataSizeUnderVolWithReplication() throws IOException { public void testDataSizeUnderBucketWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, @@ -593,13 +598,14 @@ public void testDataSizeUnderBucketWithReplication() throws IOException { * When calculating DU under dir1 * there are 3 keys, file2, file3, file6. * There is one direct key, file7. + * * @throws IOException */ @Test public void testDataSizeUnderDirWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, @@ -612,7 +618,7 @@ public void testDataSizeUnderDirWithReplication() throws IOException { public void testDataSizeUnderKeyWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, @@ -677,10 +683,10 @@ public void testFileSizeDist() throws Exception { } public void checkFileSizeDist(String path, int bin0, - int bin1, int bin2, int bin3) throws Exception { + int bin1, int bin2, int bin3) throws Exception { Response res = nsSummaryEndpoint.getFileSizeDistribution(path); FileSizeDistributionResponse fileSizeDistResObj = - (FileSizeDistributionResponse) res.getEntity(); + (FileSizeDistributionResponse) res.getEntity(); int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); assertEquals(bin0, fileSizeDist[0]); assertEquals(bin1, fileSizeDist[1]); @@ -693,6 +699,7 @@ public void checkFileSizeDist(String path, int bin0, /** * Write directories and keys info into OM DB. + * * @throws Exception */ private void populateOMDB() throws Exception { @@ -829,6 +836,7 @@ private void populateOMDB() throws Exception { /** * Create a new OM Metadata manager instance with one user, one vol, and two * buckets. + * * @throws IOException ioEx */ private static OMMetadataManager initializeNewOmMetadataManager( @@ -1168,6 +1176,7 @@ private void setUpMultiBlockReplicatedKeys() throws IOException { /** * Generate a set of mock container replica with a size of * replication factor for container. + * * @param replicationFactor number of replica * @param containerID the container replicated based upon * @return a set of container replica for testing diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java index ba00f843f44..c68045b0c36 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java @@ -243,10 +243,10 @@ public class TestNSSummaryEndpointWithLegacy { StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE6_SIZE_WITH_REPLICA = getReplicatedSize(KEY_SIX_SIZE, - StandaloneReplicationConfig.getInstance(ONE));; + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE7_SIZE_WITH_REPLICA = getReplicatedSize(KEY_SEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE));; + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE8_SIZE_WITH_REPLICA = getReplicatedSize(KEY_EIGHT_SIZE, StandaloneReplicationConfig.getInstance(ONE)); @@ -449,7 +449,7 @@ public void testGetBasicInfoKey() throws Exception { public void testDiskUsageRoot() throws Exception { // root level DU Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false); + false, false, false); DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); assertEquals(2, duRootRes.getCount()); List duRootData = duRootRes.getDuData(); @@ -468,7 +468,7 @@ public void testDiskUsageRoot() throws Exception { public void testDiskUsageVolume() throws Exception { // volume level DU Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false); + false, false, false); DUResponse duVolRes = (DUResponse) volResponse.getEntity(); assertEquals(2, duVolRes.getCount()); List duData = duVolRes.getDuData(); @@ -487,7 +487,7 @@ public void testDiskUsageVolume() throws Exception { public void testDiskUsageBucket() throws Exception { // bucket level DU Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false); + false, false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(1, duBucketResponse.getCount()); DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); @@ -499,7 +499,7 @@ public void testDiskUsageBucket() throws Exception { public void testDiskUsageDir() throws Exception { // dir level DU Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, false); + false, false, false); DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); assertEquals(3, duDirReponse.getCount()); List duSubDir = duDirReponse.getDuData(); @@ -522,7 +522,7 @@ public void testDiskUsageDir() throws Exception { public void testDiskUsageKey() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false); + false, false, false); DUResponse keyObj = (DUResponse) keyResponse.getEntity(); assertEquals(0, keyObj.getCount()); assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); @@ -532,7 +532,7 @@ public void testDiskUsageKey() throws Exception { public void testDiskUsageUnknown() throws Exception { // invalid path check Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false); + false, false, false); DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); @@ -542,7 +542,7 @@ public void testDiskUsageUnknown() throws Exception { public void testDiskUsageWithReplication() throws Exception { setUpMultiBlockKey(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, @@ -554,7 +554,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); // withReplica is true Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, @@ -568,7 +568,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { public void testDataSizeUnderVolWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, @@ -581,7 +581,7 @@ public void testDataSizeUnderVolWithReplication() throws IOException { public void testDataSizeUnderBucketWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, @@ -600,7 +600,7 @@ public void testDataSizeUnderBucketWithReplication() throws IOException { public void testDataSizeUnderDirWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, @@ -613,7 +613,7 @@ public void testDataSizeUnderDirWithReplication() throws IOException { public void testDataSizeUnderKeyWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBS.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBS.java index ac8dee5f093..5e58bf29b1c 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBS.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBS.java @@ -426,7 +426,7 @@ public void testGetBasicInfoKey() throws Exception { public void testDiskUsageRoot() throws Exception { // root level DU Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false); + false, false, false); DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); assertEquals(2, duRootRes.getCount()); List duRootData = duRootRes.getDuData(); @@ -445,7 +445,7 @@ public void testDiskUsageRoot() throws Exception { public void testDiskUsageVolume() throws Exception { // volume level DU Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false); + false, false, false); DUResponse duVolRes = (DUResponse) volResponse.getEntity(); assertEquals(2, duVolRes.getCount()); List duData = duVolRes.getDuData(); @@ -464,7 +464,7 @@ public void testDiskUsageVolume() throws Exception { public void testDiskUsageBucket() throws Exception { // bucket level DU Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false); + false, false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); // There are no sub-paths under this OBS bucket. assertEquals(0, duBucketResponse.getCount()); @@ -475,7 +475,7 @@ public void testDiskUsageBucket() throws Exception { public void testDiskUsageKey() throws Exception { // key level DU Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false); + false, false, false); DUResponse keyObj = (DUResponse) keyResponse.getEntity(); assertEquals(0, keyObj.getCount()); assertEquals(FILE_FOUR_SIZE, keyObj.getSize()); @@ -485,7 +485,7 @@ public void testDiskUsageKey() throws Exception { public void testDiskUsageUnknown() throws Exception { // invalid path check Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false); + false, false, false); DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); @@ -495,7 +495,7 @@ public void testDiskUsageUnknown() throws Exception { public void testDiskUsageWithReplication() throws Exception { setUpMultiBlockKey(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, @@ -507,7 +507,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); // withReplica is true Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, @@ -521,7 +521,7 @@ public void testDataSizeUnderRootWithReplication() throws IOException { public void testDataSizeUnderVolWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, @@ -534,7 +534,7 @@ public void testDataSizeUnderVolWithReplication() throws IOException { public void testDataSizeUnderBucketWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, @@ -545,7 +545,7 @@ public void testDataSizeUnderBucketWithReplication() throws IOException { public void testDataSizeUnderKeyWithReplication() throws IOException { setUpMultiBlockReplicatedKeys(); Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true); + false, true, false); DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, From 0876f35ec98923b582d474c8b176af9f7aeef5f0 Mon Sep 17 00:00:00 2001 From: arafat Date: Sat, 9 Mar 2024 00:23:05 +0530 Subject: [PATCH 08/10] Changed the sorting algorithm to parallel sorting --- .../hadoop/ozone/recon/ReconConstants.java | 5 ++ .../apache/hadoop/ozone/recon/ReconUtils.java | 28 ++++++++ .../api/handlers/BucketEntityHandler.java | 64 ++++--------------- .../api/handlers/DirectoryEntityHandler.java | 45 ++++--------- .../recon/api/handlers/RootEntityHandler.java | 62 ++++-------------- .../api/handlers/VolumeEntityHandler.java | 18 ++++-- 6 files changed, 82 insertions(+), 140 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 134092146e5..9c79a869c41 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -36,6 +36,11 @@ private ReconConstants() { public static final String RECON_SCM_SNAPSHOT_DB = "scm.snapshot.db"; // By default, limit the number of results returned + + /** + * The maximum number of top disk usage records to return in a /du response. + */ + public static final int DISK_USAGE_TOP_RECORDS_LIMIT = 30; public static final String DEFAULT_OPEN_KEY_INCLUDE_NON_FSO = "false"; public static final String DEFAULT_OPEN_KEY_INCLUDE_FSO = "false"; public static final String DEFAULT_FETCH_COUNT = "1000"; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 39d091ee03c..cfa37da5f93 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; +import java.util.stream.Collectors; import com.google.common.base.Preconditions; import com.google.inject.Singleton; @@ -59,6 +60,7 @@ import static org.jooq.impl.DSL.select; import static org.jooq.impl.DSL.using; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.scm.ReconContainerReportQueue; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; @@ -322,6 +324,32 @@ public static void upsertGlobalStatsTable(Configuration sqlConfiguration, } } + /** + * Sorts a list of DiskUsage objects in descending order by size using parallel sorting and + * returns the top N records as specified by the limit. + * + * This method is optimized for large datasets and utilizes parallel processing to efficiently + * sort and retrieve the top N largest records by size. It's especially useful for reducing + * processing time and memory usage when only a subset of sorted records is needed. + * + * Advantages of this approach include: + * - Efficient handling of large datasets by leveraging multi-core processors. + * - Reduction in memory usage and improvement in processing time by limiting the + * number of returned records. + * - Scalability and easy integration with existing systems. + * + * @param diskUsageList the list of DiskUsage objects to be sorted. + * @param limit the maximum number of DiskUsage objects to return. + * @return a list of the top N DiskUsage objects sorted in descending order by size, + * where N is the specified limit. + */ + public static List sortDiskUsageDescendingWithLimit(List diskUsageList, int limit) { + return diskUsageList.parallelStream() + .sorted((du1, du2) -> Long.compare(du2.getSize(), du1.getSize())) + .limit(limit) + .collect(Collectors.toList()); + } + public static long getFileSizeUpperBound(long fileSize) { if (fileSize >= ReconConstants.MAX_FILE_SIZE_UPPER_BOUND) { return Long.MAX_VALUE; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java index 904d7c36c20..03cb4e381c1 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java @@ -30,16 +30,14 @@ import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; -import java.util.Map; -import java.util.HashMap; -import java.util.Comparator; -import java.util.stream.Collectors; + +import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit; /** * Class for handling bucket entity type. @@ -110,13 +108,10 @@ public DUResponse getDuResponse( // get object IDs for all its subdirectories Set bucketSubdirs = bucketNSSummary.getChildDir(); duResponse.setKeySize(bucketNSSummary.getSizeOfFiles()); - - SortedResult result = getSortedResult(bucketSubdirs, sortSubpaths); - List dirDUData = new ArrayList<>(); long bucketDataSize = duResponse.getKeySize(); long bucketDataSizeWithReplica = 0L; - for (long subdirObjectId : result.sortedSubdirObjectIds) { + for (long subdirObjectId: bucketSubdirs) { NSSummary subdirNSSummary = getReconNamespaceSummaryManager() .getNSSummary(subdirObjectId); @@ -127,8 +122,7 @@ public DUResponse getDuResponse( // format with leading slash and without trailing slash DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); diskUsage.setSubpath(subpath); - long dataSize = result.subdirSizes.get(subdirObjectId); - + long dataSize = getTotalSize(subdirObjectId); bucketDataSize += dataSize; if (withReplica) { @@ -145,16 +139,21 @@ public DUResponse getDuResponse( bucketDataSizeWithReplica += getBucketHandler() .handleDirectKeys(bucketObjectId, withReplica, listFile, dirDUData, getNormalizedPath()); - // Sort dirDUData by size in descending order after adding files - dirDUData.sort((du1, du2) -> Long.compare(du2.getSize(), du1.getSize())); } - if (withReplica) { duResponse.setSizeWithReplica(bucketDataSizeWithReplica); } duResponse.setCount(dirDUData.size()); duResponse.setSize(bucketDataSize); + + if (sortSubpaths) { + // Parallel sort directory/files DU data in descending order of size + dirDUData = sortDiskUsageDescendingWithLimit(dirDUData, + DISK_USAGE_TOP_RECORDS_LIMIT); + } + duResponse.setDuData(dirDUData); + return duResponse; } @@ -185,41 +184,4 @@ public FileSizeDistributionResponse getDistResponse() return distResponse; } - @NotNull - private SortedResult getSortedResult(Set bucketSubdirs, boolean sortSubpaths) - throws IOException { - // Map to hold sizes for each subdir - Map subdirSizes = new HashMap<>(); - for (long subdirObjectId : bucketSubdirs) { - long dataSize = getTotalSize(subdirObjectId); - subdirSizes.put(subdirObjectId, dataSize); - } - - List subdirObjectIds; - if (sortSubpaths) { - // Sort subdirs based on their size in descending order if sortSubpaths is true - subdirObjectIds = bucketSubdirs.stream().sorted( - Comparator.comparingLong( - subdirObjectId -> subdirSizes.get(subdirObjectId)).reversed()) - .collect(Collectors.toList()); - } else { - // If sortSubpaths is false, return the subdirs as is, without sorting - subdirObjectIds = new ArrayList<>(bucketSubdirs); - } - return new SortedResult(subdirSizes, subdirObjectIds); - } - - - private static class SortedResult { - public final Map subdirSizes; - public final List sortedSubdirObjectIds; - - public SortedResult(Map subdirSizes, - List sortedSubdirObjectIds) { - this.subdirSizes = subdirSizes; - this.sortedSubdirObjectIds = sortedSubdirObjectIds; - } - } - - } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index 5d31cf1cdf8..a33c49f35fc 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -35,14 +35,12 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; -import java.util.Comparator; import java.util.ArrayList; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; +import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit; /** * Class for handling directory entity type. @@ -105,12 +103,9 @@ public DUResponse getDuResponse( duResponse.setKeySize(dirNSSummary.getSizeOfFiles()); long dirDataSize = duResponse.getKeySize(); long dirDataSizeWithReplica = 0L; - - SortedSubdirsResult sortedResult = new SortedSubdirsResult(subdirs, sortSubPaths); - List subdirDUData = new ArrayList<>(); // iterate all subdirectories to get disk usage data - for (long subdirObjectId : sortedResult.sortedSubdirObjectIds) { + for (long subdirObjectId: subdirs) { NSSummary subdirNSSummary = getReconNamespaceSummaryManager().getNSSummary(subdirObjectId); // for the subdirName we need the subdir filename, not the key name @@ -136,7 +131,7 @@ public DUResponse getDuResponse( DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); // reformat the response diskUsage.setSubpath(subpath); - long dataSize = sortedResult.subdirSizes.get(subdirObjectId); + long dataSize = getTotalSize(subdirObjectId); dirDataSize += dataSize; if (withReplica) { @@ -155,9 +150,6 @@ public DUResponse getDuResponse( dirDataSizeWithReplica += getBucketHandler() .handleDirectKeys(dirObjectId, withReplica, listFile, subdirDUData, getNormalizedPath()); - // Sort dirDUData by size in descending order after adding files - subdirDUData.sort( - (du1, du2) -> Long.compare(du2.getSize(), du1.getSize())); } if (withReplica) { @@ -165,6 +157,13 @@ public DUResponse getDuResponse( } duResponse.setCount(subdirDUData.size()); duResponse.setSize(dirDataSize); + + if (sortSubPaths) { + // Parallel sort subdirDUData in descending order of size + subdirDUData = sortDiskUsageDescendingWithLimit(subdirDUData, + DISK_USAGE_TOP_RECORDS_LIMIT); + } + duResponse.setDuData(subdirDUData); return duResponse; @@ -190,26 +189,4 @@ public FileSizeDistributionResponse getDistResponse() return distResponse; } - private class SortedSubdirsResult { - public final List sortedSubdirObjectIds; - public final Map subdirSizes; - - public SortedSubdirsResult(Set subdirs, boolean sortSubPaths) throws IOException { - this.subdirSizes = new HashMap<>(); - for (long subdirObjectId : subdirs) { - long dataSize = getTotalSize(subdirObjectId); - this.subdirSizes.put(subdirObjectId, dataSize); - } - - if (sortSubPaths) { - this.sortedSubdirObjectIds = subdirs.stream() - .sorted(Comparator.comparingLong(subdirObjectId -> this.subdirSizes.get(subdirObjectId)).reversed()) - .collect(Collectors.toList()); - } else { - this.sortedSubdirObjectIds = new ArrayList<>(subdirs); - } - } - } - - } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index 7e08f6dcb4e..e5c97234535 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -34,13 +34,13 @@ import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import java.io.IOException; import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit; /** * Class for handling root entity type. @@ -99,26 +99,10 @@ public DUResponse getDuResponse( List volumes = getOmMetadataManager().listVolumes(); duResponse.setCount(volumes.size()); - // Map to hold total size for each volume - Map volumeSizes = new HashMap<>(); - for (OmVolumeArgs volume : volumes) { - long totalVolumeSize = 0; - List buckets = - omMetadataManager.listBucketsUnderVolume(volume.getVolume()); - for (OmBucketInfo bucket : buckets) { - long bucketObjectID = bucket.getObjectID(); - totalVolumeSize += getTotalSize(bucketObjectID); - } - volumeSizes.put(volume.getVolume(), totalVolumeSize); - } - - // Apply sorting based on sortSubPaths flag - SortedResult sortedResult = getSortedResult(volumes, volumeSizes, sortSubPaths); List volumeDuData = new ArrayList<>(); long totalDataSize = 0L; long totalDataSizeWithReplica = 0L; - - for (OmVolumeArgs volume : sortedResult.sortedVolumes) { + for (OmVolumeArgs volume: volumes) { String volumeName = volume.getVolume(); String subpath = omMetadataManager.getVolumeKey(volumeName); DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); @@ -156,6 +140,13 @@ public DUResponse getDuResponse( duResponse.setSizeWithReplica(totalDataSizeWithReplica); } duResponse.setSize(totalDataSize); + + if (sortSubPaths) { + // Parallel sort volumeDuData in descending order of size + volumeDuData = sortDiskUsageDescendingWithLimit(volumeDuData, + DISK_USAGE_TOP_RECORDS_LIMIT); + } + duResponse.setDuData(volumeDuData); return duResponse; @@ -167,7 +158,8 @@ public QuotaUsageResponse getQuotaResponse() QuotaUsageResponse quotaUsageResponse = new QuotaUsageResponse(); SCMNodeStat stats = getReconSCM().getScmNodeManager().getStats(); long quotaInBytes = stats.getCapacity().get(); - long quotaUsedInBytes = getDuResponse(true, true, false).getSizeWithReplica(); + long quotaUsedInBytes = + getDuResponse(true, true, false).getSizeWithReplica(); quotaUsageResponse.setQuota(quotaInBytes); quotaUsageResponse.setQuotaUsed(quotaUsedInBytes); return quotaUsageResponse; @@ -195,30 +187,4 @@ public FileSizeDistributionResponse getDistResponse() return distResponse; } - private SortedResult getSortedResult(List volumes, - Map volumeSizes, - boolean sortSubPaths) { - if (sortSubPaths) { - // Sort volumes based on the total size in descending order - List sortedVolumes = volumes.stream() - .sorted((v1, v2) -> volumeSizes.get(v2.getVolume()) - .compareTo(volumeSizes.get(v1.getVolume()))) - .collect(Collectors.toList()); - return new SortedResult(sortedVolumes, volumeSizes); - } else { - // Return volumes as is without sorting - return new SortedResult(new ArrayList<>(volumes), volumeSizes); - } - } - - private static class SortedResult { - public final List sortedVolumes; - public final Map volumeSizes; - - public SortedResult(List sortedVolumes, Map volumeSizes) { - this.sortedVolumes = sortedVolumes; - this.volumeSizes = volumeSizes; - } - } - } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java index 8ae7b6fe763..7aa74728dc5 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java @@ -36,6 +36,10 @@ import java.util.ArrayList; import java.util.List; + +import static org.apache.hadoop.ozone.recon.ReconConstants.DISK_USAGE_TOP_RECORDS_LIMIT; +import static org.apache.hadoop.ozone.recon.ReconUtils.sortDiskUsageDescendingWithLimit; + /** * Class for handling volume entity type. */ @@ -100,13 +104,6 @@ public DUResponse getDuResponse( String volName = names[0]; List buckets = getOmMetadataManager(). listBucketsUnderVolume(volName); - - if (sortSubPaths) { - // Sort buckets in descending order by size if sortSubPaths is true - buckets.sort( - (b1, b2) -> Long.compare(b2.getUsedBytes(), b1.getUsedBytes())); - } - duResponse.setCount(buckets.size()); // List of DiskUsage data for all buckets @@ -138,6 +135,13 @@ public DUResponse getDuResponse( duResponse.setSizeWithReplica(volDataSizeWithReplica); } duResponse.setSize(volDataSize); + + if (sortSubPaths) { + // Parallel sort bucketDuData in descending order of size + bucketDuData = sortDiskUsageDescendingWithLimit(bucketDuData, + DISK_USAGE_TOP_RECORDS_LIMIT); + } + duResponse.setDuData(bucketDuData); return duResponse; } From aeeabce2859d224edba095f3dc7fd5fd8ab0bec5 Mon Sep 17 00:00:00 2001 From: arafat Date: Sat, 9 Mar 2024 00:47:49 +0530 Subject: [PATCH 09/10] Fixed checkstyle issues --- .../apache/hadoop/ozone/recon/ReconUtils.java | 3 +- .../api/TestNSSummaryDiskUsageOrdering.java | 3 +- .../api/TestNSSummaryEndpointWithFSO.java | 118 +++++++++--------- 3 files changed, 60 insertions(+), 64 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index cfa37da5f93..f154f024fbd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -343,7 +343,8 @@ public static void upsertGlobalStatsTable(Configuration sqlConfiguration, * @return a list of the top N DiskUsage objects sorted in descending order by size, * where N is the specified limit. */ - public static List sortDiskUsageDescendingWithLimit(List diskUsageList, int limit) { + public static List sortDiskUsageDescendingWithLimit( + List diskUsageList, int limit) { return diskUsageList.parallelStream() .sorted((du1, du2) -> Long.compare(du2.getSize(), du1.getSize())) .limit(limit) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java index 1adf7bfccf7..a244e4ff2ce 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryDiskUsageOrdering.java @@ -30,7 +30,6 @@ import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; @@ -159,7 +158,7 @@ public void testDiskUsageOrderingForBucket() throws Exception { private void verifyOrdering(String path) throws IOException { Response response = - nsSummaryEndpoint.getDiskUsage(path, true, false,true); + nsSummaryEndpoint.getDiskUsage(path, true, false, true); DUResponse duRes = (DUResponse) response.getEntity(); List duData = duRes.getDuData(); List sortedDuData = new ArrayList<>(duData); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index 326ed035a71..a88064d565b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -89,23 +89,23 @@ /** * Test for NSSummary REST APIs with FSO. * We tested on a mini file system with the following setting: - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ - * dir2 dir3 dir4 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 + * vol + * / \ + * bucket1 bucket2 + * / \ / \ + * file1 dir1 file4 file5 + * / \ \ + * dir2 dir3 dir4 + * / \ \ + * file2 file3 file6 + * ---------------------------------------- + * vol2 + * / \ + * bucket3 bucket4 + * / \ / + * file8 dir5 file11 + * / \ + * file9 file10 * This is a test for the Rest APIs only. We have tested NSSummaryTask before, * so there is no need to test process() on DB's updates */ @@ -119,10 +119,10 @@ public class TestNSSummaryEndpointWithFSO { private CommonUtils commonUtils; private static final String TEST_PATH_UTILITY = - "/vol1/buck1/a/b/c/d/e/file1.txt"; + "/vol1/buck1/a/b/c/d/e/file1.txt"; private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; private static final String[] TEST_NAMES = - new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; + new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; // Object names in FSO-enabled format @@ -197,12 +197,12 @@ public class TestNSSummaryEndpointWithFSO { private static final long CONTAINER_SIX_ID = 6L; // replication factors - private static final int CONTAINER_ONE_REPLICA_COUNT = 3; - private static final int CONTAINER_TWO_REPLICA_COUNT = 2; - private static final int CONTAINER_THREE_REPLICA_COUNT = 4; - private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; - private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; - private static final int CONTAINER_SIX_REPLICA_COUNT = 3; + private static final int CONTAINER_ONE_REPLICA_COUNT = 3; + private static final int CONTAINER_TWO_REPLICA_COUNT = 2; + private static final int CONTAINER_THREE_REPLICA_COUNT = 4; + private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; + private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; + private static final int CONTAINER_SIX_REPLICA_COUNT = 3; // block lengths private static final long BLOCK_ONE_LENGTH = 1000L; @@ -227,39 +227,39 @@ public class TestNSSummaryEndpointWithFSO { private static final long FILE1_SIZE_WITH_REPLICA = getReplicatedSize(KEY_ONE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE2_SIZE_WITH_REPLICA = getReplicatedSize(KEY_TWO_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE3_SIZE_WITH_REPLICA = getReplicatedSize(KEY_THREE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE4_SIZE_WITH_REPLICA = getReplicatedSize(KEY_FOUR_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE5_SIZE_WITH_REPLICA = getReplicatedSize(KEY_FIVE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE6_SIZE_WITH_REPLICA = getReplicatedSize(KEY_SIX_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE7_SIZE_WITH_REPLICA = getReplicatedSize(KEY_SEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE8_SIZE_WITH_REPLICA = getReplicatedSize(KEY_EIGHT_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE9_SIZE_WITH_REPLICA = getReplicatedSize(KEY_NINE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE10_SIZE_WITH_REPLICA = getReplicatedSize(KEY_TEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long FILE11_SIZE_WITH_REPLICA = getReplicatedSize(KEY_ELEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); + StandaloneReplicationConfig.getInstance(ONE)); private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA - = FILE7_SIZE_WITH_REPLICA; + = FILE7_SIZE_WITH_REPLICA; private static final long MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT = FILE1_SIZE_WITH_REPLICA @@ -336,19 +336,19 @@ public class TestNSSummaryEndpointWithFSO { KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; private static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE; + KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE; private static final long VOL_TWO_DATA_SIZE = KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; private static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE; + KEY_THREE_SIZE + KEY_SIX_SIZE; private static final long BUCKET_TWO_DATA_SIZE = - KEY_FOUR_SIZE + KEY_FIVE_SIZE; + KEY_FOUR_SIZE + KEY_FIVE_SIZE; private static final long DIR_ONE_DATA_SIZE = KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE; + KEY_THREE_SIZE + KEY_SIX_SIZE; @BeforeEach public void setUp() throws Exception { @@ -364,17 +364,17 @@ public void setUp() throws Exception { Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(ozoneManagerServiceProvider) - .withReconSqlDb() - .withContainerDB() - .addBinding(OzoneStorageContainerManager.class, - getMockReconSCM()) - .addBinding(StorageContainerServiceProvider.class, - mock(StorageContainerServiceProviderImpl.class)) - .addBinding(NSSummaryEndpoint.class) - .build(); + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .addBinding(OzoneStorageContainerManager.class, + getMockReconSCM()) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(NSSummaryEndpoint.class) + .build(); ReconNamespaceSummaryManager reconNamespaceSummaryManager = reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); @@ -474,7 +474,7 @@ public void testDiskUsageVolume() throws Exception { List duData = duVolRes.getDuData(); // sort based on subpath Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); DUResponse.DiskUsage duBucket1 = duData.get(0); DUResponse.DiskUsage duBucket2 = duData.get(1); assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); @@ -539,7 +539,7 @@ public void testDiskUsageUnknown() throws Exception { false, false, false); DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); + invalidObj.getStatus()); } @Test @@ -550,7 +550,7 @@ public void testDiskUsageWithReplication() throws Exception { DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, - replicaDUResponse.getSizeWithReplica()); + replicaDUResponse.getSizeWithReplica()); } @Test @@ -598,7 +598,6 @@ public void testDataSizeUnderBucketWithReplication() throws IOException { * When calculating DU under dir1 * there are 3 keys, file2, file3, file6. * There is one direct key, file7. - * * @throws IOException */ @Test @@ -683,10 +682,10 @@ public void testFileSizeDist() throws Exception { } public void checkFileSizeDist(String path, int bin0, - int bin1, int bin2, int bin3) throws Exception { + int bin1, int bin2, int bin3) throws Exception { Response res = nsSummaryEndpoint.getFileSizeDistribution(path); FileSizeDistributionResponse fileSizeDistResObj = - (FileSizeDistributionResponse) res.getEntity(); + (FileSizeDistributionResponse) res.getEntity(); int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); assertEquals(bin0, fileSizeDist[0]); assertEquals(bin1, fileSizeDist[1]); @@ -699,7 +698,6 @@ public void checkFileSizeDist(String path, int bin0, /** * Write directories and keys info into OM DB. - * * @throws Exception */ private void populateOMDB() throws Exception { @@ -836,7 +834,6 @@ private void populateOMDB() throws Exception { /** * Create a new OM Metadata manager instance with one user, one vol, and two * buckets. - * * @throws IOException ioEx */ private static OMMetadataManager initializeNewOmMetadataManager( @@ -1176,7 +1173,6 @@ private void setUpMultiBlockReplicatedKeys() throws IOException { /** * Generate a set of mock container replica with a size of * replication factor for container. - * * @param replicationFactor number of replica * @param containerID the container replicated based upon * @return a set of container replica for testing From 07de9c25587b27f5a71c446c47e6fb83a0f80171 Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 12 Mar 2024 11:25:22 +0530 Subject: [PATCH 10/10] Made review comments --- .../org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java | 3 ++- .../hadoop/ozone/recon/api/handlers/BucketEntityHandler.java | 2 +- .../ozone/recon/api/handlers/DirectoryEntityHandler.java | 2 +- .../hadoop/ozone/recon/api/handlers/RootEntityHandler.java | 2 +- .../hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java index e2b75658629..71040b9fdf6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java @@ -101,7 +101,8 @@ public Response getBasicInfo( * @param path request path * @param listFile show subpath/disk usage for each key * @param withReplica count actual DU with replication - * @param sort whether to sort the subpaths by their sizes in descending order + * @param sortSubpaths determines whether to sort the subpaths by their sizes in descending order + * and returns the N largest subpaths based on the configuration value DISK_USAGE_TOP_RECORDS_LIMIT. * @return DU response * @throws IOException */ diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java index 03cb4e381c1..00cd9617b5d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java @@ -147,7 +147,7 @@ public DUResponse getDuResponse( duResponse.setSize(bucketDataSize); if (sortSubpaths) { - // Parallel sort directory/files DU data in descending order of size + // Parallel sort directory/files DU data in descending order of size and returns the top N elements. dirDUData = sortDiskUsageDescendingWithLimit(dirDUData, DISK_USAGE_TOP_RECORDS_LIMIT); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index a33c49f35fc..ae7181af70b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -159,7 +159,7 @@ public DUResponse getDuResponse( duResponse.setSize(dirDataSize); if (sortSubPaths) { - // Parallel sort subdirDUData in descending order of size + // Parallel sort subdirDUData in descending order of size and returns the top N elements. subdirDUData = sortDiskUsageDescendingWithLimit(subdirDUData, DISK_USAGE_TOP_RECORDS_LIMIT); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index e5c97234535..b67703257ac 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -142,7 +142,7 @@ public DUResponse getDuResponse( duResponse.setSize(totalDataSize); if (sortSubPaths) { - // Parallel sort volumeDuData in descending order of size + // Parallel sort volumeDuData in descending order of size and returns the top N elements. volumeDuData = sortDiskUsageDescendingWithLimit(volumeDuData, DISK_USAGE_TOP_RECORDS_LIMIT); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java index 7aa74728dc5..2ca9c352ce7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java @@ -137,7 +137,7 @@ public DUResponse getDuResponse( duResponse.setSize(volDataSize); if (sortSubPaths) { - // Parallel sort bucketDuData in descending order of size + // Parallel sort bucketDuData in descending order of size and returns the top N elements. bucketDuData = sortDiskUsageDescendingWithLimit(bucketDuData, DISK_USAGE_TOP_RECORDS_LIMIT); }