Storing metadata information in DocumentField

elastic · Feb 4, 2019 · e657574 · e657574
1 parent 6968f09
commit e657574
Show file tree

Hide file tree

Showing 22 changed files with 73 additions and 54 deletions.
diff --git a/...olator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/...olator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.SearchContext;
@@ -110,7 +111,8 @@ static void innerHitsExecute(Query mainQuery, IndexSearcher indexSearcher, Searc
                     hit.fields(fields);
                 }
                 IntStream slots = convertTopDocsToSlots(topDocs, rootDocsBySlot);
-                fields.put(fieldName, new DocumentField(fieldName, slots.boxed().collect(Collectors.toList())));
+                boolean isMetadataField = MapperService.isMetadataField(fieldName);
+                fields.put(fieldName, new DocumentField(fieldName, slots.boxed().collect(Collectors.toList()), isMetadataField));
             }
         }
     }

diff --git a/server/src/main/java/org/elasticsearch/common/document/DocumentField.java b/server/src/main/java/org/elasticsearch/common/document/DocumentField.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.get.GetResult;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.search.SearchHit;
 
 import java.io.IOException;
@@ -47,14 +46,16 @@
 public class DocumentField implements Streamable, ToXContentFragment, Iterable<Object> {
 
     private String name;
+    private Boolean isMetadataField;
     private List<Object> values;
 
     private DocumentField() {
     }
 
-    public DocumentField(String name, List<Object> values) {
+    public DocumentField(String name, List<Object> values, boolean isMetadataField) {
         this.name = Objects.requireNonNull(name, "name must not be null");
         this.values = Objects.requireNonNull(values, "values must not be null");
+        this.isMetadataField = isMetadataField;
     }
 
     /**
@@ -85,7 +86,7 @@ public List<Object> getValues() {
      * @return The field is a metadata field
      */
     public boolean isMetadataField() {
-        return MapperService.isMetadataField(name);
+        return this.isMetadataField;
     }
 
     @Override
@@ -132,7 +133,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static DocumentField fromXContent(XContentParser parser) throws IOException {
+    public static DocumentField fromXContent(XContentParser parser, boolean inMetadataArea) throws IOException {
         ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
         String fieldName = parser.currentName();
         XContentParser.Token token = parser.nextToken();
@@ -141,7 +142,7 @@ public static DocumentField fromXContent(XContentParser parser) throws IOExcepti
         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
             values.add(parseFieldsValue(parser));
         }
-        return new DocumentField(fieldName, values);
+        return new DocumentField(fieldName, values, inMetadataArea);
     }
 
     @Override

diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -338,7 +338,9 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index
                 } else if (FOUND.equals(currentFieldName)) {
                     found = parser.booleanValue();
                 } else {
-                    fields.put(currentFieldName, new DocumentField(currentFieldName, Collections.singletonList(parser.objectText())));
+                    // This fields is in metadata area of the xContent, thus should be treated as metadata
+                    fields.put(currentFieldName, new DocumentField(currentFieldName, 
+                            Collections.singletonList(parser.objectText()), true));
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (SourceFieldMapper.NAME.equals(currentFieldName)) {
@@ -350,15 +352,15 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index
                     }
                 } else if (FIELDS.equals(currentFieldName)) {
                     while(parser.nextToken() != XContentParser.Token.END_OBJECT) {
-                        DocumentField getField = DocumentField.fromXContent(parser);
+                        DocumentField getField = DocumentField.fromXContent(parser, false);
                         fields.put(getField.getName(), getField);
                     }
                 } else {
                     parser.skipChildren(); // skip potential inner objects for forward compatibility
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
                 if (IgnoredFieldMapper.NAME.equals(currentFieldName)) {
-                    fields.put(currentFieldName, new DocumentField(currentFieldName, parser.list()));
+                    fields.put(currentFieldName, new DocumentField(currentFieldName, parser.list(), false));
                 } else {
                     parser.skipChildren(); // skip potential inner arrays for forward compatibility
                 }

diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java
@@ -202,7 +202,8 @@ private GetResult innerGetLoadFromStoredFields(String type, String id, String[]
                 fieldVisitor.postProcess(mapperService);
                 fields = new HashMap<>(fieldVisitor.fields().size());
                 for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
-                    fields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
+                    boolean isMetadataField = MapperService.isMetadataField(entry.getKey());
+                    fields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue(), isMetadataField));
                 }
             }
         }

diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -323,7 +323,8 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
                 seenFields.add(field.name());
             }
             String[] values = doc.getValues(field.name());
-            documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values)));
+            boolean isMetadataField = MapperService.isMetadataField(field.name());
+            documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values), isMetadataField));
         }
         return generateTermVectors(indexShard,
             XContentHelper.convertToMap(parsedDocument.source(), true, request.xContentType()).v2(), documentFields,

diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -799,7 +799,7 @@ private static void declareMetaDataFields(ObjectParser<Map<String, Object>, Void
                             @SuppressWarnings("unchecked")
                             Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
                                 v -> new HashMap<String, DocumentField>());
-                            DocumentField field = new DocumentField(metadatafield, list);
+                            DocumentField field = new DocumentField(metadatafield, list, true);
                             fieldMap.put(field.getName(), field);
                         }, (p, c) -> parseFieldsValue(p),
                         new ParseField(metadatafield));
@@ -809,7 +809,7 @@ private static void declareMetaDataFields(ObjectParser<Map<String, Object>, Void
                             Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
                                 v -> new HashMap<String, DocumentField>());
                             fieldMap.put(field.getName(), field);
-                        }, (p, c) -> new DocumentField(metadatafield, Collections.singletonList(parseFieldsValue(p))),
+                        }, (p, c) -> new DocumentField(metadatafield, Collections.singletonList(parseFieldsValue(p)), true),
                         new ParseField(metadatafield), ValueType.VALUE);
                 }
             }
@@ -819,7 +819,7 @@ private static void declareMetaDataFields(ObjectParser<Map<String, Object>, Void
     private static Map<String, DocumentField> parseFields(XContentParser parser) throws IOException {
         Map<String, DocumentField> fields = new HashMap<>();
         while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-            DocumentField field = DocumentField.fromXContent(parser);
+            DocumentField field = DocumentField.fromXContent(parser, false);
             fields.put(field.getName(), field);
         }
         return fields;

diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -243,10 +243,13 @@ private Map<String, DocumentField> getSearchFields(SearchContext context,
 
             if (storedToRequestedFields.containsKey(storedField)) {
                 for (String requestedField : storedToRequestedFields.get(storedField)) {
-                    searchFields.put(requestedField, new DocumentField(requestedField, storedValues));
+                    boolean isMetadataField = MapperService.isMetadataField(requestedField);
+
+                    searchFields.put(requestedField, new DocumentField(requestedField, storedValues, isMetadataField));
                 }
             } else {
-                searchFields.put(storedField, new DocumentField(storedField, storedValues));
+                boolean isMetadataField = MapperService.isMetadataField(storedField);
+                searchFields.put(storedField, new DocumentField(storedField, storedValues, isMetadataField));
             }
         }
         return searchFields;

diff --git a/...er/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/...er/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -124,7 +125,8 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
                     }
                     DocumentField hitField = hit.getFields().get(field);
                     if (hitField == null) {
-                        hitField = new DocumentField(field, new ArrayList<>(2));
+                        boolean isMetadataField = MapperService.isMetadataField(field);
+                        hitField = new DocumentField(field, new ArrayList<>(2), isMetadataField);
                         hit.getFields().put(field, hitField);
                     }
                     final List<Object> values = hitField.getValues();

diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsFetchSubPhase.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.ReaderUtil;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.script.FieldScript;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -84,7 +85,8 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
                     } else {
                         values = Collections.singletonList(value);
                     }
-                    hitField = new DocumentField(scriptFieldName, values);
+                    boolean isMetadataField = MapperService.isMetadataField(scriptFieldName);
+                    hitField = new DocumentField(scriptFieldName, values, isMetadataField);
                     hit.getFields().put(scriptFieldName, hitField);
                 }
             }

diff --git a/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java b/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java
@@ -68,7 +68,7 @@ protected ExplainResponse createTestInstance() {
             0, 1, randomNonNegativeLong(),
             true,
             RandomObjects.randomSource(random()),
-            singletonMap(fieldName, new DocumentField(fieldName, values)));
+            singletonMap(fieldName, new DocumentField(fieldName, values, false)));
         return new ExplainResponse(index, type, id, exist, explanation, getResult);
     }
 
@@ -85,7 +85,7 @@ public void testToXContent() throws IOException {
         Explanation explanation = Explanation.match(1.0f, "description", Collections.emptySet());
         GetResult getResult = new GetResult(null, null, null, 0, 1, -1, true, new BytesArray("{ \"field1\" : " +
             "\"value1\", \"field2\":\"value2\"}"), singletonMap("field1", new DocumentField("field1",
-            singletonList("value1"))));
+            singletonList("value1"), false)));
         ExplainResponse response = new ExplainResponse(index, type, id, exist, explanation, getResult);
 
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);

diff --git a/server/src/test/java/org/elasticsearch/action/get/GetResponseTests.java b/server/src/test/java/org/elasticsearch/action/get/GetResponseTests.java
@@ -94,7 +94,7 @@ public void testToXContent() {
         {
             GetResponse getResponse = new GetResponse(new GetResult("index", "type", "id", 0, 1, 1, true, new BytesArray("{ \"field1\" : " +
                     "\"value1\", \"field2\":\"value2\"}"), Collections.singletonMap("field1", new DocumentField("field1",
-                    Collections.singletonList("value1")))));
+                    Collections.singletonList("value1"), false))));
             String output = Strings.toString(getResponse);
             assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"_seq_no\":0,\"_primary_term\":1," +
                 "\"found\":true,\"_source\":{ \"field1\" : \"value1\", \"field2\":\"value2\"},\"fields\":{\"field1\":[\"value1\"]}}",
@@ -110,7 +110,7 @@ public void testToXContent() {
     public void testToString() {
         GetResponse getResponse = new GetResponse(new GetResult("index", "type", "id", 0, 1, 1, true,
                     new BytesArray("{ \"field1\" : " + "\"value1\", \"field2\":\"value2\"}"),
-                        Collections.singletonMap("field1", new DocumentField("field1", Collections.singletonList("value1")))));
+                        Collections.singletonMap("field1", new DocumentField("field1", Collections.singletonList("value1"), false))));
         assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"_seq_no\":0,\"_primary_term\":1," +
             "\"found\":true,\"_source\":{ \"field1\" : \"value1\", \"field2\":\"value2\"},\"fields\":{\"field1\":[\"value1\"]}}",
             getResponse.toString());

diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
@@ -104,7 +104,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
             };
 
             SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(1, "ID", new Text("type"),
-                Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))))},
+                Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue), false)))},
                 new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F);
             InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
             AtomicReference<SearchResponse> reference = new AtomicReference<>();
@@ -158,9 +158,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
         };
 
         SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(1, "ID", new Text("type"),
-            Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue)))),
+            Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue), false))),
             new SearchHit(2, "ID2", new Text("type"),
-                Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))))},
+                Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue), false)))},
             new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F);
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
         AtomicReference<SearchResponse> reference = new AtomicReference<>();
@@ -190,9 +190,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
         };
 
         SearchHits hits = new SearchHits(new SearchHit[]{new SearchHit(1, "ID", new Text("type"),
-            Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null)))),
+            Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null), false))),
             new SearchHit(2, "ID2", new Text("type"),
-                Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null))))},
+                Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null), false)))},
             new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F);
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
         AtomicReference<SearchResponse> reference = new AtomicReference<>();

diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
@@ -560,7 +560,7 @@ public void testRoutingExtraction() throws Exception {
         assertNull(UpdateHelper.calculateRouting(getResult, indexRequest));
 
         Map<String, DocumentField> fields = new HashMap<>();
-        fields.put("_routing", new DocumentField("_routing", Collections.singletonList("routing1")));
+        fields.put("_routing", new DocumentField("_routing", Collections.singletonList("routing1"), false));
 
         // Doc exists and has the parent and routing fields
         getResult = new GetResult("test", "type", "1", 0, 1, 0, true, null, fields);

diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java
@@ -69,8 +69,8 @@ public void testToXContent() throws IOException {
         {
             BytesReference source = new BytesArray("{\"title\":\"Book title\",\"isbn\":\"ABC-123\"}");
             Map<String, DocumentField> fields = new HashMap<>();
-            fields.put("title", new DocumentField("title", Collections.singletonList("Book title")));
-            fields.put("isbn", new DocumentField("isbn", Collections.singletonList("ABC-123")));
+            fields.put("title", new DocumentField("title", Collections.singletonList("Book title"), false));
+            fields.put("isbn", new DocumentField("isbn", Collections.singletonList("ABC-123"), false));
 
             UpdateResponse updateResponse = new UpdateResponse(new ReplicationResponse.ShardInfo(3, 2),
                     new ShardId("books", "books_uuid", 2), "book", "1", 7, 17, 2, UPDATED);