Add indexRandomForConcurrentSearch to tests (#11259)

* Update Concurrent search related tests to use slice count > 1. Signed-off-by: Harish Bhakuni <[email protected]> Signed-off-by: Jay Deng <[email protected]> * Add indexRandomForConcurrentSearch to tests: * CardinalityWithRequestBreakerIT.java * TopHitsIT.java * SearchFieldsIT.java * DecayFunctionScoreIT.java * FunctionScoreFieldValueIT.java * FunctionScoreIT.java * FunctionScorePluginIT.java * QueryRescorerIT.java Signed-off-by: Jay Deng <[email protected]> --------- Signed-off-by: Harish Bhakuni <[email protected]> Signed-off-by: Jay Deng <[email protected]> Co-authored-by: Harish Bhakuni <[email protected]> (cherry picked from commit edbc4e2) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Nov 19, 2023 · 3af5093 · 3af5093
1 parent f2cdd9f
commit 3af5093
Show file tree

Hide file tree

Showing 7 changed files with 48 additions and 8 deletions.
diff --git a/...Test/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java b/...Test/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
@@ -99,6 +99,7 @@ public void testRequestBreaker() throws Exception {
             )
             .get();
 
+        indexRandomForConcurrentSearch("test");
         try {
             client().prepareSearch("test")
                 .addAggregation(

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
@@ -251,6 +251,7 @@ public void testStoredFields() throws Exception {
             .get();
 
         client().admin().indices().prepareRefresh().get();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field1").get();
         assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
@@ -358,6 +359,7 @@ public void testScriptDocAndFields() throws Exception {
             )
             .get();
         client().admin().indices().refresh(refreshRequest()).actionGet();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("running doc['num1'].value");
         SearchResponse response = client().prepareSearch()
@@ -458,6 +460,7 @@ public void testScriptWithUnsignedLong() throws Exception {
             )
             .get();
         client().admin().indices().refresh(refreshRequest()).actionGet();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -547,6 +550,7 @@ public void testScriptFieldWithNanos() throws Exception {
                 .setSource(jsonBuilder().startObject().field("date", "1970-01-01T00:00:00.000Z").endObject()),
             client().prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("date", date).endObject())
         );
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -632,6 +636,7 @@ public void testScriptFieldUsingSource() throws Exception {
             )
             .get();
         client().admin().indices().refresh(refreshRequest()).actionGet();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -674,6 +679,7 @@ public void testScriptFieldUsingSource() throws Exception {
     public void testScriptFieldsForNullReturn() throws Exception {
         client().prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy("true").get();
 
+        indexRandomForConcurrentSearch("test");
         SearchResponse response = client().prepareSearch()
             .setQuery(matchAllQuery())
             .addScriptField("test_script_1", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return null", Collections.emptyMap()))
@@ -795,6 +801,7 @@ public void testStoredFieldsWithoutSource() throws Exception {
             .get();
 
         client().admin().indices().prepareRefresh().get();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -852,6 +859,7 @@ public void testSearchFieldsMetadata() throws Exception {
             .setSource(jsonBuilder().startObject().field("field1", "value").endObject())
             .setRefreshPolicy(IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("my-index");
 
         SearchResponse searchResponse = client().prepareSearch("my-index").addStoredField("field1").addStoredField("_routing").get();
 
@@ -866,6 +874,7 @@ public void testSearchFieldsNonLeafField() throws Exception {
             .setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject())
             .setRefreshPolicy(IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("my-index");
 
         assertFailures(
             client().prepareSearch("my-index").addStoredField("field1"),
@@ -932,6 +941,7 @@ public void testGetFieldsComplexField() throws Exception {
         );
 
         client().prepareIndex("my-index").setId("1").setRefreshPolicy(IMMEDIATE).setSource(source, MediaTypeRegistry.JSON).get();
+        indexRandomForConcurrentSearch("my-index");
 
         String field = "field1.field2.field3.field4";
 
@@ -1039,6 +1049,7 @@ public void testDocValueFields() throws Exception {
             .get();
 
         client().admin().indices().prepareRefresh().get();
+        indexRandomForConcurrentSearch("test");
 
         SearchRequestBuilder builder = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -1271,6 +1282,7 @@ public void testScriptFields() throws Exception {
             );
         }
         indexRandom(true, reqs);
+        indexRandomForConcurrentSearch("index");
         ensureSearchable();
         SearchRequestBuilder req = client().prepareSearch("index");
         for (String field : Arrays.asList("s", "ms", "l", "ml", "d", "md")) {
@@ -1326,6 +1338,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception {
 
         index("test", MapperService.SINGLE_MAPPING_NAME, "1", "text_field", "foo", "date_field", formatter.print(date));
         refresh("test");
+        indexRandomForConcurrentSearch("test");
 
         SearchRequestBuilder builder = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -1387,6 +1400,7 @@ public void testWildcardDocValueFieldsWithFieldAlias() throws Exception {
 
         index("test", MapperService.SINGLE_MAPPING_NAME, "1", "text_field", "foo", "date_field", formatter.print(date));
         refresh("test");
+        indexRandomForConcurrentSearch("test");
 
         SearchRequestBuilder builder = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -1440,6 +1454,7 @@ public void testStoredFieldsWithFieldAlias() throws Exception {
 
         index("test", MapperService.SINGLE_MAPPING_NAME, "1", "field1", "value1", "field2", "value2");
         refresh("test");
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -1482,6 +1497,7 @@ public void testWildcardStoredFieldsWithFieldAlias() throws Exception {
 
         index("test", MapperService.SINGLE_MAPPING_NAME, "1", "field1", "value1", "field2", "value2");
         refresh("test");
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field*").get();
         assertHitCount(searchResponse, 1L);

diff --git a/...rc/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java b/...rc/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java
@@ -398,6 +398,7 @@ public void testBoostModeSettingWorks() throws Exception {
                 )
         );
         indexRandom(true, false, indexBuilders); // force no dummy docs
+        indexRandomForConcurrentSearch("test");
 
         // Test Gauss
         List<Float> lonlat = new ArrayList<>();
@@ -482,6 +483,7 @@ public void testParseGeoPoint() throws Exception {
             constantScoreQuery(termQuery("test", "value")),
             ScoreFunctionBuilders.weightFactorFunction(randomIntBetween(1, 10))
         );
+        indexRandomForConcurrentSearch("test");
         GeoPoint point = new GeoPoint(20, 11);
         ActionFuture<SearchResponse> response = client().search(
             searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
@@ -535,6 +537,7 @@ public void testCombineModes() throws Exception {
             .setRefreshPolicy(IMMEDIATE)
             .setSource(jsonBuilder().startObject().field("test", "value value").field("num", 1.0).endObject())
             .get();
+        indexRandomForConcurrentSearch("test");
         FunctionScoreQueryBuilder baseQuery = functionScoreQuery(
             constantScoreQuery(termQuery("test", "value")),
             ScoreFunctionBuilders.weightFactorFunction(2)
@@ -654,6 +657,7 @@ public void testCombineModesExplain() throws Exception {
             constantScoreQuery(termQuery("test", "value")).queryName("query1"),
             ScoreFunctionBuilders.weightFactorFunction(2, "weight1")
         );
+        indexRandomForConcurrentSearch("test");
         // decay score should return 0.5 for this function and baseQuery should return 2.0f as it's score
         ActionFuture<SearchResponse> response = client().search(
             searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
@@ -762,6 +766,7 @@ public void testParseDateMath() throws Exception {
         ).actionGet();
         refresh();
 
+        indexRandomForConcurrentSearch("test");
         SearchResponse sr = client().search(
             searchRequest().source(
                 searchSource().query(functionScoreQuery(termQuery("test", "value"), gaussDecayFunction("num1", "now", "2d")))
@@ -817,6 +822,7 @@ public void testValueMissingLin() throws Exception {
         ).actionGet();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         ActionFuture<SearchResponse> response = client().search(
             searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
@@ -893,6 +899,7 @@ public void testDateWithoutOrigin() throws Exception {
         ).actionGet();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         ActionFuture<SearchResponse> response = client().search(
             searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
@@ -974,6 +981,7 @@ public void testManyDocsLin() throws Exception {
         List<Float> lonlat = new ArrayList<>();
         lonlat.add(100f);
         lonlat.add(110f);
+        indexRandomForConcurrentSearch("test");
         ActionFuture<SearchResponse> response = client().search(
             searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
                 .source(
@@ -1107,6 +1115,7 @@ public void testNoQueryGiven() throws Exception {
         client().index(indexRequest("test").source(jsonBuilder().startObject().field("test", "value").field("num", 1.0).endObject()))
             .actionGet();
         refresh();
+        indexRandomForConcurrentSearch("test");
         // so, we indexed a string field, but now we try to score a num field
         ActionFuture<SearchResponse> response = client().search(
             searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
@@ -1171,6 +1180,7 @@ public void testMultiFieldOptions() throws Exception {
             );
 
         indexRandom(true, doc1, doc2);
+        indexRandomForConcurrentSearch("test");
 
         ActionFuture<SearchResponse> response = client().search(searchRequest().source(searchSource().query(baseQuery)));
         SearchResponse sr = response.actionGet();

diff --git a/...ternalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java b/...ternalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java
@@ -80,7 +80,7 @@ protected Settings featureFlagSettings() {
         return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
     }
 
-    public void testFieldValueFactor() throws IOException {
+    public void testFieldValueFactor() throws IOException, InterruptedException {
         assertAcked(
             prepareCreate("test").setMapping(
                 jsonBuilder().startObject()
@@ -99,8 +99,8 @@ public void testFieldValueFactor() throws IOException {
         client().prepareIndex("test").setId("1").setSource("test", 5, "body", "foo").get();
         client().prepareIndex("test").setId("2").setSource("test", 17, "body", "foo").get();
         client().prepareIndex("test").setId("3").setSource("body", "bar").get();
-
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // document 2 scores higher because 17 > 5
         SearchResponse response = client().prepareSearch("test")
@@ -189,7 +189,7 @@ public void testFieldValueFactor() throws IOException {
         }
     }
 
-    public void testFieldValueFactorExplain() throws IOException {
+    public void testFieldValueFactorExplain() throws IOException, InterruptedException {
         assertAcked(
             prepareCreate("test").setMapping(
                 jsonBuilder().startObject()
@@ -210,6 +210,7 @@ public void testFieldValueFactorExplain() throws IOException {
         client().prepareIndex("test").setId("3").setSource("body", "bar").get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // document 2 scores higher because 17 > 5
         final String functionName = "func1";

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java
@@ -126,10 +126,11 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         }
     }
 
-    public void testScriptScoresNested() throws IOException {
+    public void testScriptScoresNested() throws IOException, InterruptedException {
         createIndex(INDEX);
         index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
         refresh();
+        indexRandomForConcurrentSearch(INDEX);
 
         Script scriptOne = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1", Collections.emptyMap());
         Script scriptTwo = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());
@@ -148,10 +149,11 @@ public void testScriptScoresNested() throws IOException {
         assertThat(response.getHits().getAt(0).getScore(), equalTo(1.0f));
     }
 
-    public void testScriptScoresWithAgg() throws IOException {
+    public void testScriptScoresWithAgg() throws IOException, InterruptedException {
         createIndex(INDEX);
         index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
         refresh();
+        indexRandomForConcurrentSearch(INDEX);
 
         Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());
 
@@ -166,10 +168,11 @@ public void testScriptScoresWithAgg() throws IOException {
         assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1L));
     }
 
-    public void testScriptScoresWithAggWithExplain() throws IOException {
+    public void testScriptScoresWithAggWithExplain() throws IOException, InterruptedException {
         createIndex(INDEX);
         index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
         refresh();
+        indexRandomForConcurrentSearch(INDEX);
 
         Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());
 
@@ -195,7 +198,7 @@ public void testScriptScoresWithAggWithExplain() throws IOException {
         assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1L));
     }
 
-    public void testMinScoreFunctionScoreBasic() throws IOException {
+    public void testMinScoreFunctionScoreBasic() throws IOException, InterruptedException {
         float score = randomValueOtherThanMany((f) -> Float.compare(f, 0) < 0, OpenSearchTestCase::randomFloat);
         float minScore = randomValueOtherThanMany((f) -> Float.compare(f, 0) < 0, OpenSearchTestCase::randomFloat);
         index(
@@ -207,6 +210,7 @@ public void testMinScoreFunctionScoreBasic() throws IOException {
                 .endObject()
         );
         refresh();
+        indexRandomForConcurrentSearch(INDEX);
         ensureYellow();
 
         Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['random_score']", Collections.emptyMap());
@@ -291,6 +295,7 @@ public void testWithEmptyFunctions() throws IOException, ExecutionException, Int
         assertAcked(prepareCreate("test"));
         index("test", "testtype", "1", jsonBuilder().startObject().field("text", "test text").endObject());
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse termQuery = client().search(searchRequest().source(searchSource().explain(true).query(termQuery("text", "text"))))
             .get();

diff --git a/...c/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java b/...c/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java
@@ -122,6 +122,7 @@ public void testPlugin() throws Exception {
         ).actionGet();
 
         client().admin().indices().prepareRefresh().get();
+        indexRandomForConcurrentSearch("test");
         DecayFunctionBuilder<?> gfb = new CustomDistanceScoreBuilder("num1", "2013-05-28", "+1d");
 
         ActionFuture<SearchResponse> response = client().search(

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java
@@ -110,14 +110,15 @@ protected Settings featureFlagSettings() {
         return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
     }
 
-    public void testEnforceWindowSize() {
+    public void testEnforceWindowSize() throws InterruptedException {
         createIndex("test");
         // this
         int iters = scaledRandomIntBetween(10, 20);
         for (int i = 0; i < iters; i++) {
             client().prepareIndex("test").setId(Integer.toString(i)).setSource("f", Integer.toString(i)).get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         int numShards = getNumShards("test").numPrimaries;
         for (int j = 0; j < iters; j++) {
@@ -169,6 +170,7 @@ public void testRescorePhrase() throws Exception {
             .setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR))
             .setRescorer(
@@ -474,6 +476,7 @@ private static void assertEquivalent(String query, SearchResponse plain, SearchR
     public void testEquivalence() throws Exception {
         // no dummy docs since merges can change scores while we run queries.
         int numDocs = indexRandomNumbers("whitespace", -1, false);
+        indexRandomForConcurrentSearch("test");
 
         final int iters = scaledRandomIntBetween(50, 100);
         for (int i = 0; i < iters; i++) {
@@ -545,6 +548,7 @@ public void testExplain() throws Exception {
             .setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         {
             SearchResponse searchResponse = client().prepareSearch()
@@ -816,6 +820,7 @@ public void testFromSize() throws Exception {
             client().prepareIndex("test").setId("" + i).setSource("text", "hello world").get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchRequestBuilder request = client().prepareSearch();
         request.setQuery(QueryBuilders.termQuery("text", "hello"));
@@ -832,6 +837,7 @@ public void testRescorePhaseWithInvalidSort() throws Exception {
             client().prepareIndex("test").setId("" + i).setSource("number", 0).get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         Exception exc = expectThrows(
             Exception.class,