Support the 'fields' option in inner_hits and top_hits. (elastic#62259)

This PR adds support for the 'fields' option in the following places:
* Anytime `inner_hits` is used, for both fetching nested/ child docs and field collapsing
* The `top_hits` aggregation

Addresses elastic#61949.

docs/reference/aggregations/metrics/tophits-aggregation.asciidoc

@@ -24,6 +24,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
 * <<highlighting,Highlighting>>
 * <<request-body-search-explain,Explain>>
 * <<named-queries,Named queries>>
+* <<search-fields-param,Search fields>>
 * <<source-filtering,Source filtering>>
 * <<stored-fields,Stored fields>>
 * <<script-fields,Script fields>>
@@ -41,7 +42,7 @@ by aggregations, use a <<query-dsl-function-score-query,`function_score`>> or
 
 ==== Example
 
-In the following example we group the sales by type and per type we show the last sale. 
+In the following example we group the sales by type and per type we show the last sale.
 For each sale only the date and price fields are being included in the source.
 
 [source,console]

docs/reference/search/search-your-data/retrieve-inner-hits.asciidoc

@@ -74,6 +74,7 @@ Inner hits also supports the following per document features:
 
 * <<highlighting,Highlighting>>
 * <<request-body-search-explain,Explain>>
+* <<search-fields-param,Search fields>>
 * <<request-body-search-source-filtering,Source filtering>>
 * <<script-fields,Script fields>>
 * <<docvalue-fields,Doc value fields>>

modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java

@@ -176,7 +176,7 @@ public void testSimpleParentChild() throws Exception {
             .setQuery(
                 hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
                     new InnerHitBuilder()
-                        .addDocValueField("message")
+                        .addFetchField("message")
                         .setHighlightBuilder(new HighlightBuilder().field("message"))
                         .setExplain(true).setSize(1)
                         .addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5",
@@ -187,8 +187,18 @@ public void testSimpleParentChild() throws Exception {
         assertThat(innerHits.getHits().length, equalTo(1));
         assertThat(innerHits.getAt(0).getHighlightFields().get("message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
         assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(message:fox"));
-        assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
+        assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("fox eat quick"));
         assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
+
+        response = client().prepareSearch("articles")
+            .setQuery(
+                hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
+                    new InnerHitBuilder().addDocValueField("message").setSize(1)
+                )).get();
+        assertNoFailures(response);
+        innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
+        assertThat(innerHits.getHits().length, equalTo(1));
+        assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
     }
 
     public void testRandomParentChild() throws Exception {

rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml

@@ -6,49 +6,50 @@ setup:
           mappings:
               properties:
                 numeric_group: { type: integer }
+                tag: { type: keyword }
 
   - do:
       index:
           index:  test
           id:     1
           version_type: external
           version: 11
-          body:   { numeric_group: 1, sort: 10 }
+          body:   { numeric_group: 1, tag: A, sort: 10 }
   - do:
       index:
           index:  test
           id:     2
           version_type: external
           version: 22
-          body:   { numeric_group: 1, sort: 6 }
+          body:   { numeric_group: 1, tag: B, sort: 6 }
   - do:
       index:
           index:  test
           id:     3
           version_type: external
           version: 33
-          body:   { numeric_group: 1, sort: 24 }
+          body:   { numeric_group: 1, tag: A, sort: 24 }
   - do:
       index:
           index:  test
           id:     4
           version_type: external
           version: 44
-          body:   { numeric_group: 25, sort: 10 }
+          body:   { numeric_group: 25, tag: B, sort: 10 }
   - do:
       index:
           index:  test
           id:     5
           version_type: external
           version: 55
-          body:   { numeric_group: 25, sort: 5 }
+          body:   { numeric_group: 25, tag: A, sort: 5 }
   - do:
       index:
           index:  test
           id:     6
           version_type: external
           version: 66
-          body:   { numeric_group: 3, sort: 36 }
+          body:   { numeric_group: 3, tag: B, sort: 36 }
   - do:
       indices.refresh:
         index: test
@@ -161,6 +162,40 @@ setup:
   - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
   - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
 
+---
+"field collapsing, inner_hits, and fields":
+  - skip:
+      version: " - 7.99.99"
+      reason: the enhancement is not yet backported to 7.x
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse:
+            field: numeric_group
+            inner_hits:
+              name: sub_hits
+              size: 2
+              sort: [{ sort: asc }]
+              fields: ["tag"]
+          sort: [{ sort: desc }]
+
+  - length: { hits.hits: 3 }
+  - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0.fields.tag: ["B"]}
+  - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0.fields.tag: ["B"]}
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1.fields.tag: ["A"]}
+  - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0.fields.tag: ["A"]}
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1.fields.tag: ["B"]}
+
 ---
 "field collapsing, inner_hits and maxConcurrentGroupRequests":
 

server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java

@@ -166,6 +166,7 @@ public void setupSuiteScopeCluster() throws Exception {
                     .field(SORT_FIELD, i + 1)
                     .field("text", "some text to entertain")
                     .field("field1", 5)
+                    .field("field2", 2.71)
                     .endObject()));
         }
 
@@ -315,7 +316,7 @@ public void testBasics() throws Exception {
             assertThat((Long) hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1));
             assertThat((Long) hits.getAt(2).getSortValues()[0], equalTo(higestSortValue - 2));
 
-            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
+            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
         }
     }
 
@@ -402,7 +403,7 @@ public void testBreadthFirstWithScoreNeeded() throws Exception {
             assertThat(hits.getTotalHits().value, equalTo(10L));
             assertThat(hits.getHits().length, equalTo(3));
 
-            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
+            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
         }
     }
 
@@ -433,7 +434,7 @@ public void testBreadthFirstWithAggOrderAndScoreNeeded() throws Exception {
             assertThat(hits.getTotalHits().value, equalTo(10L));
             assertThat(hits.getHits().length, equalTo(3));
 
-            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
+            assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
             id--;
         }
     }
@@ -597,6 +598,7 @@ public void testFetchFeatures() {
                                             .explain(true)
                                             .storedField("text")
                                             .docValueField("field1")
+                                            .fetchField("field2")
                                             .scriptField("script",
                                                 new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
                                             .fetchSource("text", null)
@@ -639,13 +641,16 @@ public void testFetchFeatures() {
 
             assertThat(hit.getMatchedQueries()[0], equalTo("test"));
 
-            DocumentField field = hit.field("field1");
-            assertThat(field.getValue().toString(), equalTo("5"));
+            DocumentField field1 = hit.field("field1");
+            assertThat(field1.getValue(), equalTo(5L));
+
+            DocumentField field2 = hit.field("field2");
+            assertThat(field2.getValue(), equalTo(2.71f));
 
             assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));
 
-            field = hit.field("script");
-            assertThat(field.getValue().toString(), equalTo("5"));
+            field2 = hit.field("script");
+            assertThat(field2.getValue().toString(), equalTo("5"));
 
             assertThat(hit.getSourceAsMap().size(), equalTo(1));
             assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));

server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java

@@ -165,7 +165,7 @@ public void testSimpleNested() throws Exception {
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
                         new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message"))
                                 .setExplain(true)
-                                .addDocValueField("comments.mes*")
+                                .addFetchField("comments.mes*")
                                 .addScriptField("script",
                                         new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
                                 .setSize(1))).get();
@@ -176,8 +176,18 @@ public void testSimpleNested() throws Exception {
         assertThat(innerHits.getAt(0).getHighlightFields().get("comments.message").getFragments()[0].string(),
                 equalTo("<em>fox</em> eat quick"));
         assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(comments.message:fox in"));
-        assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
+        assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("fox eat quick"));
         assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
+
+        response = client().prepareSearch("articles")
+            .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
+                new InnerHitBuilder()
+                    .addDocValueField("comments.mes*")
+                    .setSize(1))).get();
+        assertNoFailures(response);
+        innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
+        assertThat(innerHits.getHits().length, equalTo(1));
+        assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
     }
 
     public void testRandomNested() throws Exception {

server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java

@@ -136,6 +136,9 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio
                     options.getFetchSourceContext().excludes());
             }
         }
+        if (options.getFetchFields() != null) {
+            options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format));
+        }
         if (options.getDocValueFields() != null) {
             options.getDocValueFields().forEach(ff -> groupSource.docValueField(ff.field, ff.format));
         }