diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 5d8eb432741..3ddbb410087 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -387,6 +387,7 @@ private OzoneConsts() { /** Metadata stored in OmKeyInfo. */ public static final String HSYNC_CLIENT_ID = "hsyncClientId"; public static final String LEASE_RECOVERY = "leaseRecovery"; + public static final String DELETED_HSYNC_KEY = "deletedHsyncKey"; public static final String FORCE_LEASE_RECOVERY_ENV = "OZONE.CLIENT.RECOVER.LEASE.FORCE"; //GDPR diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index daa433f68f8..cdc9487a057 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -25,7 +25,9 @@ import java.nio.charset.StandardCharsets; import java.security.GeneralSecurityException; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -84,6 +86,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.ozone.test.GenericTestUtils; @@ -113,6 +116,11 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LEASE_HARD_LIMIT; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -144,6 +152,10 @@ public class TestHSync { private static final int FLUSH_SIZE = 2 * CHUNK_SIZE; private static final int MAX_FLUSH_SIZE = 2 * FLUSH_SIZE; private static final int BLOCK_SIZE = 2 * MAX_FLUSH_SIZE; + private static final int SERVICE_INTERVAL = 100; + private static final int EXPIRE_THRESHOLD_MS = 140; + + private static OpenKeyCleanupService openKeyCleanupService; @BeforeAll public static void init() throws Exception { @@ -155,9 +167,18 @@ public static void init() throws Exception { CONF.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10); // Reduce KeyDeletingService interval CONF.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); + CONF.setTimeDuration(OZONE_DIR_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); CONF.setBoolean("ozone.client.incremental.chunk.list", true); CONF.setBoolean("ozone.client.stream.putblock.piggybacking", true); CONF.setBoolean(OZONE_CHUNK_LIST_INCREMENTAL, true); + CONF.setTimeDuration(OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL, + SERVICE_INTERVAL, TimeUnit.MILLISECONDS); + CONF.setTimeDuration(OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD, + EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS); + CONF.setTimeDuration(OZONE_OM_LEASE_HARD_LIMIT, + EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS); + CONF.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s"); + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) @@ -183,6 +204,10 @@ public static void init() throws Exception { GenericTestUtils.setLogLevel(BlockOutputStream.LOG, Level.DEBUG); GenericTestUtils.setLogLevel(BlockInputStream.LOG, Level.DEBUG); GenericTestUtils.setLogLevel(KeyValueHandler.LOG, Level.DEBUG); + + openKeyCleanupService = + (OpenKeyCleanupService) cluster.getOzoneManager().getKeyManager().getOpenKeyCleanupService(); + openKeyCleanupService.suspend(); } @AfterAll @@ -393,6 +418,65 @@ public void testHSyncDeletedKey() throws Exception { } } + @Test + public void testHSyncOpenKeyDeletionWhileDeleteDirectory() throws Exception { + // Verify that when directory is deleted recursively hsync related openKeys should be deleted, + + // Set the fs.defaultFS + final String rootPath = String.format("%s://%s/", + OZONE_OFS_URI_SCHEME, CONF.get(OZONE_OM_ADDRESS_KEY)); + CONF.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); + + final String dir = OZONE_ROOT + bucket.getVolumeName() + + OZONE_URI_DELIMITER + bucket.getName() + OZONE_URI_DELIMITER + "dir1/dir2"; + final Path key1 = new Path(dir, "hsync-key"); + + try (FileSystem fs = FileSystem.get(CONF)) { + // Create key1 + try (FSDataOutputStream os = fs.create(key1, true)) { + os.write(1); + os.hsync(); + // There should be 1 key in openFileTable + assertThat(1 == getOpenKeyInfo(BucketLayout.FILE_SYSTEM_OPTIMIZED).size()); + // Delete directory recursively + fs.delete(new Path(OZONE_ROOT + bucket.getVolumeName() + OZONE_URI_DELIMITER + + bucket.getName() + OZONE_URI_DELIMITER + "dir1/"), true); + + // Verify if DELETED_HSYNC_KEY metadata is added to openKey + GenericTestUtils.waitFor(() -> { + List omKeyInfo = getOpenKeyInfo(BucketLayout.FILE_SYSTEM_OPTIMIZED); + return omKeyInfo.size() > 0 && omKeyInfo.get(0).getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY); + }, 1000, 12000); + + // Resume openKeyCleanupService + openKeyCleanupService.resume(); + + // Verify entry from openKey gets deleted eventually + GenericTestUtils.waitFor(() -> + 0 == getOpenKeyInfo(BucketLayout.FILE_SYSTEM_OPTIMIZED).size(), 1000, 12000); + } catch (OMException ex) { + assertEquals(OMException.ResultCodes.DIRECTORY_NOT_FOUND, ex.getResult()); + } finally { + openKeyCleanupService.suspend(); + } + } + } + + private List getOpenKeyInfo(BucketLayout bucketLayout) { + List omKeyInfo = new ArrayList<>(); + + Table openFileTable = + cluster.getOzoneManager().getMetadataManager().getOpenKeyTable(bucketLayout); + try (TableIterator> + iterator = openFileTable.iterator()) { + while (iterator.hasNext()) { + omKeyInfo.add(iterator.next().getValue()); + } + } catch (Exception e) { + } + return omKeyInfo; + } + @Test public void testUncommittedBlocks() throws Exception { // Set the fs.defaultFS diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index afc9eae859a..5e683591933 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1838,8 +1838,10 @@ public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, .filter(id -> id.equals(clientIdString)) .isPresent(); - if (!isHsync && openKeyInfo.getCreationTime() <= expiredCreationTimestamp) { + if ((!isHsync && openKeyInfo.getCreationTime() <= expiredCreationTimestamp) || + (openKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY))) { // add non-hsync'ed keys + // also add hsync keys which are already deleted from keyTable expiredKeys.addOpenKey(openKeyInfo, dbOpenKeyName); num++; } else if (isHsync && openKeyInfo.getModificationTime() <= expiredLeaseTimestamp && diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java index 6116ed81e87..be12886a689 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java @@ -221,6 +221,10 @@ private RecoverLeaseResponse doWork(OzoneManager ozoneManager, throw new OMException("Open Key " + dbOpenFileKey + " not found in openKeyTable", KEY_NOT_FOUND); } + if (openKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY)) { + throw new OMException("Open Key " + keyName + " is already deleted", + KEY_NOT_FOUND); + } long openKeyModificationTime = openKeyInfo.getModificationTime(); if (openKeyInfo.getMetadata().containsKey(OzoneConsts.LEASE_RECOVERY)) { LOG.debug("Key: " + keyName + " is already under recovery"); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index d47b2b7a57e..2e3b63d6e4e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -208,6 +208,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn throw new OMException("Open Key " + openKeyName + " is under lease recovery", KEY_UNDER_LEASE_RECOVERY); } + if (openKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY)) { + throw new OMException("Open Key " + openKeyName + " is already deleted", + KEY_NOT_FOUND); + } List newLocationList = Collections.singletonList( OmKeyLocationInfo.getFromProtobuf(blockLocation)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 505b6287307..d6935ed683c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -21,9 +21,11 @@ import java.io.IOException; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -39,8 +41,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import java.util.List; - +import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; /** @@ -66,6 +67,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn Set> lockSet = new HashSet<>(); Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + Map openKeyInfoMap = new HashMap<>(); OMMetrics omMetrics = ozoneManager.getMetrics(); try { @@ -110,6 +112,21 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn volumeName, bucketName); lockSet.add(volBucketPair); } + + // If omKeyInfo has hsync metadata, delete its corresponding open key as well + String dbOpenKey; + String hsyncClientId = keyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); + if (hsyncClientId != null) { + long parentId = keyInfo.getParentObjectID(); + dbOpenKey = omMetadataManager.getOpenFileName(path.getVolumeId(), path.getBucketId(), + parentId, keyInfo.getFileName(), hsyncClientId); + OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKey); + if (openKeyInfo != null) { + openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyInfoMap.put(dbOpenKey, openKeyInfo); + } + } + omMetrics.decNumKeys(); OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); @@ -142,7 +159,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn getOmRequest()); OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( omResponse.build(), purgeRequests, ozoneManager.isRatisEnabled(), - getBucketLayout(), volBucketInfoMap, fromSnapshotInfo); + getBucketLayout(), volBucketInfoMap, fromSnapshotInfo, openKeyInfoMap); return omClientResponse; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 830ab70827c..2b238342ee6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -253,6 +253,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn throw new OMException("Failed to " + action + " key, as " + dbOpenKey + " entry is not found in the OpenKey table", KEY_NOT_FOUND); } + if (omKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY)) { + throw new OMException("Open Key " + keyName + " is already deleted", + KEY_NOT_FOUND); + } if (omKeyInfo.getMetadata().containsKey(OzoneConsts.LEASE_RECOVERY) && omKeyInfo.getMetadata().containsKey(OzoneConsts.HSYNC_CLIENT_ID)) { if (!isRecovery) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 61e5976f805..14dc8242454 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; @@ -161,6 +162,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn long quotaReleased = sumBlockLengths(omKeyInfo); omBucketInfo.incrUsedBytes(-quotaReleased); omBucketInfo.incrUsedNamespace(-1L); + OmKeyInfo deletedOpenKeyInfo = null; // If omKeyInfo has hsync metadata, delete its corresponding open key as well String dbOpenKey = null; @@ -170,8 +172,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, hsyncClientId); OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - // Remove the open key by putting a tombstone entry - openKeyTable.addCacheEntry(dbOpenKey, trxnLogIndex); + openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); + deletedOpenKeyInfo = openKeyInfo; } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } @@ -180,7 +183,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn omClientResponse = new OMKeyDeleteResponse( omResponse.setDeleteKeyResponse(DeleteKeyResponse.newBuilder()) .build(), omKeyInfo, ozoneManager.isRatisEnabled(), - omBucketInfo.copyObject(), dbOpenKey); + omBucketInfo.copyObject(), deletedOpenKeyInfo); result = Result.SUCCESS; } catch (IOException | InvalidPathException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java index a0b2cfcbb15..b8b3efbc24e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java @@ -51,6 +51,7 @@ import java.nio.file.InvalidPathException; import java.util.Map; +import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_EMPTY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; @@ -129,6 +130,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn String ozonePathKey = omMetadataManager.getOzonePathKey(volumeId, bucketId, omKeyInfo.getParentObjectID(), omKeyInfo.getFileName()); + OmKeyInfo deletedOpenKeyInfo = null; if (keyStatus.isDirectory()) { // Check if there are any sub path exists under the user requested path @@ -165,8 +167,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId, parentId, fileName, hsyncClientId); OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - // Remove the open key by putting a tombstone entry - openKeyTable.addCacheEntry(dbOpenKey, trxnLogIndex); + openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); + deletedOpenKeyInfo = openKeyInfo; } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } @@ -175,7 +178,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn omClientResponse = new OMKeyDeleteResponseWithFSO(omResponse .setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(), keyName, omKeyInfo, ozoneManager.isRatisEnabled(), - omBucketInfo.copyObject(), keyStatus.isDirectory(), volumeId, dbOpenKey); + omBucketInfo.copyObject(), keyStatus.isDirectory(), volumeId, deletedOpenKeyInfo); result = Result.SUCCESS; } catch (IOException | InvalidPathException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index be89da369cd..61ed5ffb1c8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -57,11 +57,13 @@ import java.io.IOException; import java.nio.file.InvalidPathException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import static org.apache.hadoop.ozone.OzoneConsts.BUCKET; +import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.OzoneConsts.DELETED_KEYS_LIST; import static org.apache.hadoop.ozone.OzoneConsts.UNDELETED_KEYS_LIST; import static org.apache.hadoop.ozone.OzoneConsts.VOLUME; @@ -174,18 +176,18 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); - List dbOpenKeys = new ArrayList<>(); + Map openKeyInfoMap = new HashMap<>(); // Mark all keys which can be deleted, in cache as deleted. quotaReleased = markKeysAsDeletedInCache(ozoneManager, trxnLogIndex, omKeyInfoList, - dirList, omMetadataManager, quotaReleased, dbOpenKeys); + dirList, omMetadataManager, quotaReleased, openKeyInfoMap); omBucketInfo.incrUsedBytes(-quotaReleased); omBucketInfo.incrUsedNamespace(-1L * omKeyInfoList.size()); final long volumeId = omMetadataManager.getVolumeId(volumeName); omClientResponse = getOmClientResponse(ozoneManager, omKeyInfoList, dirList, omResponse, - unDeletedKeys, deleteStatus, omBucketInfo, volumeId, dbOpenKeys); + unDeletedKeys, deleteStatus, omBucketInfo, volumeId, openKeyInfoMap); result = Result.SUCCESS; @@ -260,7 +262,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, List omKeyInfoList, List dirList, OMResponse.Builder omResponse, OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys, - boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, List dbOpenKeys) { + boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap) { OMClientResponse omClientResponse; omClientResponse = new OMKeysDeleteResponse(omResponse .setDeleteKeysResponse( @@ -268,13 +270,13 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, .setUnDeletedKeys(unDeletedKeys)) .setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus) .build(), omKeyInfoList, ozoneManager.isRatisEnabled(), - omBucketInfo.copyObject(), dbOpenKeys); + omBucketInfo.copyObject(), openKeyInfoMap); return omClientResponse; } protected long markKeysAsDeletedInCache(OzoneManager ozoneManager, long trxnLogIndex, List omKeyInfoList, List dirList, - OMMetadataManager omMetadataManager, long quotaReleased, List dbOpenKeys) + OMMetadataManager omMetadataManager, long quotaReleased, Map openKeyInfoMap) throws IOException { for (OmKeyInfo omKeyInfo : omKeyInfoList) { String volumeName = omKeyInfo.getVolumeName(); @@ -294,10 +296,10 @@ protected long markKeysAsDeletedInCache(OzoneManager ozoneManager, String dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, hsyncClientId); OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - // Remove the open key by putting a tombstone entry - openKeyTable.addCacheEntry(dbOpenKey, trxnLogIndex); - // Append to the list of open keys to be deleted. The list is not expected to be large. - dbOpenKeys.add(dbOpenKey); + openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); + // Add to the map of open keys to be deleted. + openKeyInfoMap.put(dbOpenKey, openKeyInfo); } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index b90fd15b267..85d9c8ff4cb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -37,7 +37,9 @@ import java.io.IOException; import java.util.List; +import java.util.Map; +import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_DELETE; @@ -88,7 +90,7 @@ protected long markKeysAsDeletedInCache( OzoneManager ozoneManager, long trxnLogIndex, List omKeyInfoList, List dirList, OMMetadataManager omMetadataManager, - long quotaReleased, List dbOpenKeys) throws IOException { + long quotaReleased, Map openKeyInfoMap) throws IOException { // Mark all keys which can be deleted, in cache as deleted. for (OmKeyInfo omKeyInfo : omKeyInfoList) { @@ -113,10 +115,10 @@ protected long markKeysAsDeletedInCache( String dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId, parentId, fileName, hsyncClientId); OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - // Remove the open key by putting a tombstone entry - openKeyTable.addCacheEntry(dbOpenKey, trxnLogIndex); - // Append to the list of open keys to be deleted. The list is not expected to be large. - dbOpenKeys.add(dbOpenKey); + openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); + // Add to the map of open keys to be deleted. + openKeyInfoMap.put(dbOpenKey, openKeyInfo); } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } @@ -146,7 +148,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, List omKeyInfoList, List dirList, OzoneManagerProtocolProtos.OMResponse.Builder omResponse, OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys, - boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, List dbOpenKeys) { + boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap) { OMClientResponse omClientResponse; omClientResponse = new OMKeysDeleteResponseWithFSO(omResponse .setDeleteKeysResponse( @@ -154,7 +156,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, .setStatus(deleteStatus).setUnDeletedKeys(unDeletedKeys)) .setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus) .build(), omKeyInfoList, dirList, ozoneManager.isRatisEnabled(), - omBucketInfo.copyObject(), volumeId, dbOpenKeys); + omBucketInfo.copyObject(), volumeId, openKeyInfoMap); return omClientResponse; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 848c5c30890..edb13f8cf98 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -62,17 +62,19 @@ public class OMDirectoriesPurgeResponseWithFSO extends OmKeyResponse { private boolean isRatisEnabled; private Map, OmBucketInfo> volBucketInfoMap; private SnapshotInfo fromSnapshotInfo; + private Map openKeyInfoMap; public OMDirectoriesPurgeResponseWithFSO(@Nonnull OMResponse omResponse, @Nonnull List paths, boolean isRatisEnabled, @Nonnull BucketLayout bucketLayout, Map, OmBucketInfo> volBucketInfoMap, - SnapshotInfo fromSnapshotInfo) { + SnapshotInfo fromSnapshotInfo, Map openKeyInfoMap) { super(omResponse, bucketLayout); this.paths = paths; this.isRatisEnabled = isRatisEnabled; this.volBucketInfoMap = volBucketInfoMap; this.fromSnapshotInfo = fromSnapshotInfo; + this.openKeyInfoMap = openKeyInfoMap; } @Override @@ -165,6 +167,13 @@ public void processPaths(OMMetadataManager omMetadataManager, deletedKey, repeatedOmKeyInfo); } + if (!openKeyInfoMap.isEmpty()) { + for (Map.Entry entry : openKeyInfoMap.entrySet()) { + omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( + batchOperation, entry.getKey(), entry.getValue()); + } + } + // Delete the visited directory from deleted directory table if (path.hasDeletedDir()) { omMetadataManager.getDeletedDirTable().deleteWithBatch(batchOperation, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java index 0cb0d745d19..7cdd2dc36b2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.response.key; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -45,15 +46,15 @@ public class OMKeyDeleteResponse extends AbstractOMKeyDeleteResponse { private OmKeyInfo omKeyInfo; private OmBucketInfo omBucketInfo; // If not null, this key will be deleted from OpenKeyTable - private String dbOpenKey; + private OmKeyInfo deletedOpenKeyInfo; public OMKeyDeleteResponse(@Nonnull OMResponse omResponse, @Nonnull OmKeyInfo omKeyInfo, boolean isRatisEnabled, - @Nonnull OmBucketInfo omBucketInfo, String dbOpenKey) { + @Nonnull OmBucketInfo omBucketInfo, OmKeyInfo deletedOpenKeyInfo) { super(omResponse, isRatisEnabled, omBucketInfo.getBucketLayout()); this.omKeyInfo = omKeyInfo; this.omBucketInfo = omBucketInfo; - this.dbOpenKey = dbOpenKey; + this.deletedOpenKeyInfo = deletedOpenKeyInfo; } /** @@ -83,10 +84,16 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(), omBucketInfo.getBucketName()), omBucketInfo); - // Remove open key (necessary when the file is hsync'ed but not committed) - if (dbOpenKey != null) { - omMetadataManager.getOpenKeyTable(getBucketLayout()).deleteWithBatch( - batchOperation, dbOpenKey); + // necessary when the file is hsync'ed but not committed + // Update metadata which will be used to cleanup openKey in openKeyCleanupService + if (deletedOpenKeyInfo != null) { + String hsyncClientId = deletedOpenKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); + if (hsyncClientId != null) { + String dbOpenKey = omMetadataManager.getOpenKey(deletedOpenKeyInfo.getVolumeName(), + deletedOpenKeyInfo.getBucketName(), deletedOpenKeyInfo.getKeyName(), hsyncClientId); + omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( + batchOperation, dbOpenKey, deletedOpenKeyInfo); + } } } @@ -94,11 +101,12 @@ protected OmKeyInfo getOmKeyInfo() { return omKeyInfo; } + protected OmKeyInfo getDeletedOpenKeyInfo() { + return deletedOpenKeyInfo; + } + protected OmBucketInfo getOmBucketInfo() { return omBucketInfo; } - public String getDbOpenKey() { - return dbOpenKey; - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java index f52ea1b4ce0..e428684a169 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -52,8 +53,8 @@ public class OMKeyDeleteResponseWithFSO extends OMKeyDeleteResponse { public OMKeyDeleteResponseWithFSO(@Nonnull OMResponse omResponse, @Nonnull String keyName, @Nonnull OmKeyInfo omKeyInfo, boolean isRatisEnabled, @Nonnull OmBucketInfo omBucketInfo, - @Nonnull boolean isDeleteDirectory, @Nonnull long volumeId, String dbOpenKey) { - super(omResponse, omKeyInfo, isRatisEnabled, omBucketInfo, dbOpenKey); + @Nonnull boolean isDeleteDirectory, @Nonnull long volumeId, OmKeyInfo deletedOpenKeyInfo) { + super(omResponse, omKeyInfo, isRatisEnabled, omBucketInfo, deletedOpenKeyInfo); this.keyName = keyName; this.isDeleteDirectory = isDeleteDirectory; this.volumeId = volumeId; @@ -110,9 +111,16 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getBucketKey(getOmBucketInfo().getVolumeName(), getOmBucketInfo().getBucketName()), getOmBucketInfo()); - if (getDbOpenKey() != null) { - omMetadataManager.getOpenKeyTable(getBucketLayout()).deleteWithBatch( - batchOperation, getDbOpenKey()); + // Update metadata which will be used to cleanup openKey in openKeyCleanupService + OmKeyInfo deletedOpenKeyInfo = getDeletedOpenKeyInfo(); + if (deletedOpenKeyInfo != null) { + String hsyncClientId = getDeletedOpenKeyInfo().getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); + if (hsyncClientId != null) { + String dbOpenKey = omMetadataManager.getOpenKey(deletedOpenKeyInfo.getVolumeName(), + deletedOpenKeyInfo.getBucketName(), deletedOpenKeyInfo.getKeyName(), hsyncClientId); + omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( + batchOperation, dbOpenKey, deletedOpenKeyInfo); + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java index 23f3acd45ac..8cf7f6b9260 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java @@ -29,7 +29,9 @@ import jakarta.annotation.Nonnull; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; @@ -45,16 +47,16 @@ public class OMKeysDeleteResponse extends AbstractOMKeyDeleteResponse { private List omKeyInfoList; private OmBucketInfo omBucketInfo; - private List dbOpenKeys; + private Map openKeyInfoMap = new HashMap<>(); public OMKeysDeleteResponse(@Nonnull OMResponse omResponse, @Nonnull List keyDeleteList, boolean isRatisEnabled, @Nonnull OmBucketInfo omBucketInfo, - @Nonnull List dbOpenKeys) { + @Nonnull Map openKeyInfoMap) { super(omResponse, isRatisEnabled); this.omKeyInfoList = keyDeleteList; this.omBucketInfo = omBucketInfo; - this.dbOpenKeys = dbOpenKeys; + this.openKeyInfoMap = openKeyInfoMap; } /** @@ -100,9 +102,11 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(), omBucketInfo.getBucketName()), omBucketInfo); - for (String dbOpenKey : dbOpenKeys) { - omMetadataManager.getOpenKeyTable(getBucketLayout()).deleteWithBatch( - batchOperation, dbOpenKey); + if (!openKeyInfoMap.isEmpty()) { + for (Map.Entry entry : openKeyInfoMap.entrySet()) { + omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( + batchOperation, entry.getKey(), entry.getValue()); + } } } @@ -114,7 +118,7 @@ public OmBucketInfo getOmBucketInfo() { return omBucketInfo; } - public List getDbOpenKeys() { - return dbOpenKeys; + protected Map getOpenKeyInfoMap() { + return openKeyInfoMap; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java index 3a662727b02..861204c3938 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java @@ -29,6 +29,7 @@ import jakarta.annotation.Nonnull; import java.io.IOException; import java.util.List; +import java.util.Map; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE; @@ -52,8 +53,8 @@ public OMKeysDeleteResponseWithFSO( @Nonnull List keyDeleteList, @Nonnull List dirDeleteList, boolean isRatisEnabled, @Nonnull OmBucketInfo omBucketInfo, @Nonnull long volId, - @Nonnull List dbOpenKeys) { - super(omResponse, keyDeleteList, isRatisEnabled, omBucketInfo, dbOpenKeys); + @Nonnull Map openKeyInfoMap) { + super(omResponse, keyDeleteList, isRatisEnabled, omBucketInfo, openKeyInfoMap); this.dirsList = dirDeleteList; this.volumeId = volId; } @@ -95,9 +96,11 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getBucketKey(getOmBucketInfo().getVolumeName(), getOmBucketInfo().getBucketName()), getOmBucketInfo()); - for (String dbOpenKey : getDbOpenKeys()) { - omMetadataManager.getOpenKeyTable(getBucketLayout()).deleteWithBatch( - batchOperation, dbOpenKey); + if (!getOpenKeyInfoMap().isEmpty()) { + for (Map.Entry entry : getOpenKeyInfoMap().entrySet()) { + omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( + batchOperation, entry.getKey(), entry.getValue()); + } } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java index 8d178bcd47b..e7689e52ed7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java @@ -104,7 +104,7 @@ public void testKeysDeleteResponse() throws Exception { protected OMClientResponse getOmKeysDeleteResponse(OMResponse omResponse, OmBucketInfo omBucketInfo) { return new OMKeysDeleteResponse( - omResponse, omKeyInfoList, true, omBucketInfo, Collections.emptyList()); + omResponse, omKeyInfoList, true, omBucketInfo, Collections.emptyMap()); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java index 98522814de7..13e706757f9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java @@ -112,7 +112,7 @@ protected OMClientResponse getOmKeysDeleteResponse(OMResponse omResponse, OmBucketInfo omBucketInfo) { return new OMKeysDeleteResponseWithFSO( omResponse, getOmKeyInfoList(), dirDeleteList, true, omBucketInfo, - volId, Collections.emptyList()); + volId, Collections.emptyMap()); } @Test