Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Backport 2.x] Make IndicesRequestCacheCleanupIT more robust #14498

Merged
merged 1 commit into from
Jun 21, 2024
Merged

[Backport 2.x] Make IndicesRequestCacheCleanupIT more robust #14498

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 17 additions & 31 deletions server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false)
public class IndicesRequestCacheCleanupIT extends OpenSearchIntegTestCase {

private static final long MAX_ITERATIONS = 5;

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(InternalSettingsPlugin.class);
Expand All @@ -74,23 +76,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
public void testCacheWithInvalidation() throws Exception {
Client client = client();
String index = "index";
assertAcked(
client.admin()
.indices()
.prepareCreate(index)
.setMapping("k", "type=keyword")
.setSettings(
Settings.builder()
.put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put("index.refresh_interval", -1)
// Disable index refreshing to avoid cache being invalidated mid-test
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
)
.get()
);
indexRandom(false, client.prepareIndex(index).setSource("k", "hello"));
setupIndex(client, index);
ensureSearchable(index);
// Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
forceMerge(client, index);
Expand Down Expand Up @@ -125,8 +111,8 @@ public void testCacheClearAPIRemovesStaleKeysWhenStalenessThresholdIsLow() throw
.put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
.put(
IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
// setting intentionally high to avoid cache cleaner interfering
TimeValue.timeValueMillis(300)
// Set interval much larger than test timeout to effectively disable it
TimeValue.timeValueDays(1)
)
);
Client client = client(node);
Expand Down Expand Up @@ -210,7 +196,7 @@ public void testStaleKeysCleanupWithLowThreshold() throws Exception {
assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
// cache cleaner should NOT have cleaned from index 1
assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
// sleep until cache cleaner would have cleaned up the stale key from index 2
}

Expand Down Expand Up @@ -260,7 +246,7 @@ public void testCacheCleanupOnEqualStalenessAndThreshold() throws Exception {
assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
// cache cleaner should NOT have cleaned from index 1
assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

// when staleness threshold is higher than staleness, it should NOT clean the cache
Expand Down Expand Up @@ -308,7 +294,7 @@ public void testCacheCleanupSkipsWithHighStalenessThreshold() throws Exception {
assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
// cache cleaner should NOT have cleaned from index 1
assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

// when staleness threshold is explicitly set to 0, cache cleaner regularly cleans up stale keys.
Expand Down Expand Up @@ -356,7 +342,7 @@ public void testCacheCleanupOnZeroStalenessThreshold() throws Exception {
assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
// cache cleaner should NOT have cleaned from index 1
assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

// when staleness threshold is not explicitly set, cache cleaner regularly cleans up stale keys
Expand Down Expand Up @@ -403,7 +389,7 @@ public void testStaleKeysRemovalWithoutExplicitThreshold() throws Exception {
assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
// cache cleaner should NOT have cleaned from index 1
assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

// when cache cleaner interval setting is not set, cache cleaner is configured appropriately with the fall-back setting
Expand Down Expand Up @@ -447,7 +433,7 @@ public void testCacheCleanupWithDefaultSettings() throws Exception {
assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
// cache cleaner should NOT have cleaned from index 1
assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

// staleness threshold updates flows through to the cache cleaner
Expand Down Expand Up @@ -490,7 +476,7 @@ public void testDynamicStalenessThresholdUpdate() throws Exception {
assertBusy(() -> {
// cache cleaner should NOT have cleaned up the stale key from index 2
assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);

// Update indices.requests.cache.cleanup.staleness_threshold to "10%"
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
Expand All @@ -505,7 +491,7 @@ public void testDynamicStalenessThresholdUpdate() throws Exception {
assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
// cache cleaner should NOT have cleaned from index 1
assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

// staleness threshold dynamic updates should throw exceptions on invalid input
Expand Down Expand Up @@ -557,7 +543,7 @@ public void testCacheClearanceAfterIndexClosure() throws Exception {
assertBusy(() -> {
// cache cleaner should have cleaned up the stale keys from index
assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

// deleting the Index after caching will clean up from Indices Request Cache
Expand Down Expand Up @@ -598,7 +584,7 @@ public void testCacheCleanupAfterIndexDeletion() throws Exception {
assertBusy(() -> {
// cache cleaner should have cleaned up the stale keys from index
assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

// when staleness threshold is lower than staleness, it should clean the cache from all indices having stale keys
Expand Down Expand Up @@ -645,7 +631,7 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
// Assert cache is cleared up
assertBusy(
() -> { assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes()); },
cacheCleanIntervalInMillis * 2,
cacheCleanIntervalInMillis * MAX_ITERATIONS,
TimeUnit.MILLISECONDS
);

Expand All @@ -667,7 +653,7 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
long currentMemorySizeInBytesForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
// assert the memory size of index1 to only contain 1 entry added after flushAndRefresh
assertEquals(memorySizeForIndex1With1Entries, currentMemorySizeInBytesForIndex1);
}, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
}, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS);
}

private void setupIndex(Client client, String index) throws Exception {
Expand Down
Loading