diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 5618e18742267..9df9623926fc5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -114,28 +114,28 @@ public class SearchIT extends ESRestHighLevelClientTestCase { public void indexDocuments() throws IOException { { Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1"); - doc1.setJsonEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}"); + doc1.setJsonEntity("{\"type\":\"type1\", \"id\":1, \"num\":10, \"num2\":50}"); client().performRequest(doc1); Request doc2 = new Request(HttpPut.METHOD_NAME, "/index/_doc/2"); - doc2.setJsonEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}"); + doc2.setJsonEntity("{\"type\":\"type1\", \"id\":2, \"num\":20, \"num2\":40}"); client().performRequest(doc2); Request doc3 = new Request(HttpPut.METHOD_NAME, "/index/_doc/3"); - doc3.setJsonEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}"); + doc3.setJsonEntity("{\"type\":\"type1\", \"id\":3, \"num\":50, \"num2\":35}"); client().performRequest(doc3); Request doc4 = new Request(HttpPut.METHOD_NAME, "/index/_doc/4"); - doc4.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}"); + doc4.setJsonEntity("{\"type\":\"type2\", \"id\":4, \"num\":100, \"num2\":10}"); client().performRequest(doc4); Request doc5 = new Request(HttpPut.METHOD_NAME, "/index/_doc/5"); - doc5.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}"); + doc5.setJsonEntity("{\"type\":\"type2\", \"id\":5, \"num\":100, \"num2\":10}"); client().performRequest(doc5); } { Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1"); - doc1.setJsonEntity("{\"field\":\"value1\", \"rating\": 7}"); + doc1.setJsonEntity("{\"id\":1, \"field\":\"value1\", \"rating\": 7}"); client().performRequest(doc1); Request doc2 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/2"); - doc2.setJsonEntity("{\"field\":\"value2\"}"); + doc2.setJsonEntity("{\"id\":2, \"field\":\"value2\"}"); client().performRequest(doc2); } @@ -153,19 +153,19 @@ public void indexDocuments() throws IOException { "}"); client().performRequest(create); Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3"); - doc3.setJsonEntity("{\"field\":\"value1\", \"rating\": \"good\"}"); + doc3.setJsonEntity("{\"id\":3, \"field\":\"value1\", \"rating\": \"good\"}"); client().performRequest(doc3); Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4"); - doc4.setJsonEntity("{\"field\":\"value2\"}"); + doc4.setJsonEntity("{\"id\":4, \"field\":\"value2\"}"); client().performRequest(doc4); } { Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5"); - doc5.setJsonEntity("{\"field\":\"value1\"}"); + doc5.setJsonEntity("{\"id\":5, \"field\":\"value1\"}"); client().performRequest(doc5); Request doc6 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/6"); - doc6.setJsonEntity("{\"field\":\"value2\"}"); + doc6.setJsonEntity("{\"id\":6, \"field\":\"value2\"}"); client().performRequest(doc6); } @@ -188,7 +188,7 @@ public void indexDocuments() throws IOException { "}"); client().performRequest(create); Request doc1 = new Request(HttpPut.METHOD_NAME, "/index4/_doc/1"); - doc1.setJsonEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}"); + doc1.setJsonEntity("{\"id\":1, \"field1\":\"value1\", \"field2\":\"value2\"}"); client().performRequest(doc1); Request createFilteredAlias = new Request(HttpPost.METHOD_NAME, "/_aliases"); @@ -225,7 +225,7 @@ public void testSearchNoQuery() throws IOException { assertEquals(1.0f, searchHit.getScore(), 0); assertEquals(-1L, searchHit.getVersion()); assertNotNull(searchHit.getSourceAsMap()); - assertEquals(3, searchHit.getSourceAsMap().size()); + assertEquals(4, searchHit.getSourceAsMap().size()); assertTrue(searchHit.getSourceAsMap().containsKey("type")); assertTrue(searchHit.getSourceAsMap().containsKey("num")); assertTrue(searchHit.getSourceAsMap().containsKey("num2")); @@ -249,7 +249,7 @@ public void testSearchMatchQuery() throws IOException { assertThat(searchHit.getScore(), greaterThan(0f)); assertEquals(-1L, searchHit.getVersion()); assertNotNull(searchHit.getSourceAsMap()); - assertEquals(3, searchHit.getSourceAsMap().size()); + assertEquals(4, searchHit.getSourceAsMap().size()); assertEquals("type1", searchHit.getSourceAsMap().get("type")); assertEquals(50, searchHit.getSourceAsMap().get("num2")); } @@ -705,13 +705,13 @@ public void testSearchScroll() throws Exception { public void testMultiSearch() throws Exception { MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); SearchRequest searchRequest1 = new SearchRequest("index1"); - searchRequest1.source().sort("_id", SortOrder.ASC); + searchRequest1.source().sort("id", SortOrder.ASC); multiSearchRequest.add(searchRequest1); SearchRequest searchRequest2 = new SearchRequest("index2"); - searchRequest2.source().sort("_id", SortOrder.ASC); + searchRequest2.source().sort("id", SortOrder.ASC); multiSearchRequest.add(searchRequest2); SearchRequest searchRequest3 = new SearchRequest("index3"); - searchRequest3.source().sort("_id", SortOrder.ASC); + searchRequest3.source().sort("id", SortOrder.ASC); multiSearchRequest.add(searchRequest3); MultiSearchResponse multiSearchResponse = @@ -1198,7 +1198,8 @@ public void testExplainWithFetchSource() throws IOException { assertTrue(explainResponse.hasExplanation()); assertThat(explainResponse.getExplanation().getValue(), equalTo(1.0f)); assertTrue(explainResponse.getGetResult().isExists()); - assertThat(explainResponse.getGetResult().getSource(), equalTo(Collections.singletonMap("field1", "value1"))); + assertEquals(2, explainResponse.getGetResult().getSource().size()); + assertThat(explainResponse.getGetResult().getSource().get("field1"), equalTo("value1")); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index 47ae97fe3573f..995a50508fcf3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -169,7 +169,7 @@ public void testSearch() throws Exception { // tag::search-source-sorting sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); // <1> - sourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC)); // <2> + sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); // <2> // end::search-source-sorting // tag::search-source-filtering-off @@ -1251,6 +1251,9 @@ private void indexSearchTestData() throws IOException { CreateIndexRequest authorsRequest = new CreateIndexRequest("authors") .mapping(XContentFactory.jsonBuilder().startObject() .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("user") .field("type", "keyword") .field("doc_values", "false") @@ -1263,6 +1266,9 @@ private void indexSearchTestData() throws IOException { CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors") .mapping(XContentFactory.jsonBuilder().startObject() .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("user") .field("type", "keyword") .field("store", "true") @@ -1274,19 +1280,19 @@ private void indexSearchTestData() throws IOException { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest("posts").id("1") - .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user", + .source(XContentType.JSON, "id", 1, "title", "In which order are my Elasticsearch queries executed?", "user", Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value"))); bulkRequest.add(new IndexRequest("posts").id("2") - .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user", + .source(XContentType.JSON, "id", 2, "title", "Current status and upcoming changes in Elasticsearch", "user", Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value"))); bulkRequest.add(new IndexRequest("posts").id("3") - .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user", + .source(XContentType.JSON, "id", 3, "title", "The Future of Federated Search in Elasticsearch", "user", Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value"))); bulkRequest.add(new IndexRequest("authors").id("1") - .source(XContentType.JSON, "user", "kimchy")); + .source(XContentType.JSON, "id", 1, "user", "kimchy")); bulkRequest.add(new IndexRequest("contributors").id("1") - .source(XContentType.JSON, "user", "tanguy")); + .source(XContentType.JSON, "id", 1, "user", "tanguy")); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); diff --git a/docs/reference/mapping/types/parent-join.asciidoc b/docs/reference/mapping/types/parent-join.asciidoc index 35b674bd1c61a..63056c734e1d7 100644 --- a/docs/reference/mapping/types/parent-join.asciidoc +++ b/docs/reference/mapping/types/parent-join.asciidoc @@ -16,6 +16,9 @@ PUT my_index { "mappings": { "properties": { + "my_id": { + "type": "keyword" + }, "my_join_field": { <1> "type": "join", "relations": { @@ -38,6 +41,7 @@ For instance the following example creates two `parent` documents in the `questi -------------------------------------------------- PUT my_index/_doc/1?refresh { + "my_id": "1", "text": "This is a question", "my_join_field": { "name": "question" <1> @@ -46,6 +50,7 @@ PUT my_index/_doc/1?refresh PUT my_index/_doc/2?refresh { + "my_id": "2", "text": "This is another question", "my_join_field": { "name": "question" @@ -63,12 +68,14 @@ as a shortcut instead of encapsulating it in the normal object notation: -------------------------------------------------- PUT my_index/_doc/1?refresh { + "my_id": "1", "text": "This is a question", "my_join_field": "question" <1> } PUT my_index/_doc/2?refresh { + "my_id": "2", "text": "This is another question", "my_join_field": "question" } @@ -89,6 +96,7 @@ For instance the following example shows how to index two `child` documents: -------------------------------------------------- PUT my_index/_doc/3?routing=1&refresh <1> { + "my_id": "3", "text": "This is an answer", "my_join_field": { "name": "answer", <2> @@ -98,6 +106,7 @@ PUT my_index/_doc/3?routing=1&refresh <1> PUT my_index/_doc/4?routing=1&refresh { + "my_id": "4", "text": "This is another answer", "my_join_field": { "name": "answer", @@ -159,7 +168,7 @@ GET my_index/_search "query": { "match_all": {} }, - "sort": ["_id"] + "sort": ["my_id"] } -------------------------- // TEST[continued] @@ -182,6 +191,7 @@ Will return: "_id": "1", "_score": null, "_source": { + "my_id": "1", "text": "This is a question", "my_join_field": "question" <1> }, @@ -194,6 +204,7 @@ Will return: "_id": "2", "_score": null, "_source": { + "my_id": "2", "text": "This is another question", "my_join_field": "question" <2> }, @@ -207,6 +218,7 @@ Will return: "_score": null, "_routing": "1", "_source": { + "my_id": "3", "text": "This is an answer", "my_join_field": { "name": "answer", <3> @@ -223,6 +235,7 @@ Will return: "_score": null, "_routing": "1", "_source": { + "my_id": "4", "text": "This is another answer", "my_join_field": { "name": "answer", diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java index 8f8c30e3ec587..ca15aa01e4c59 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java @@ -79,8 +79,7 @@ private SearchRequestBuilder buildRequest(String script, Object... params) { SearchRequestBuilder req = client().prepareSearch().setIndices("test"); req.setQuery(QueryBuilders.matchAllQuery()) - .addSort(SortBuilders.fieldSort("_id") - .order(SortOrder.ASC)) + .addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC).unmappedType("long")) .addScriptField("foo", new Script(ScriptType.INLINE, "expression", script, paramsMap)); return req; } @@ -147,8 +146,10 @@ public void testDateMethods() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date")); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), - client().prepareIndex("test").setId("2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); + client().prepareIndex("test").setId("1") + .setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), + client().prepareIndex("test").setId("2") + .setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get(); assertEquals(2, rsp.getHits().getTotalHits().value); SearchHits hits = rsp.getHits(); @@ -175,8 +176,10 @@ public void testDateObjectMethods() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date")); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), - client().prepareIndex("test").setId("2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); + client().prepareIndex("test").setId("1") + .setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), + client().prepareIndex("test").setId("2") + .setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); SearchResponse rsp = buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour").get(); assertEquals(2, rsp.getHits().getTotalHits().value); SearchHits hits = rsp.getHits(); @@ -207,15 +210,18 @@ public void testMultiValueMethods() throws Exception { ensureGreen("test"); Map doc1 = new HashMap<>(); + doc1.put("id", 1); doc1.put("double0", new Double[]{5.0d, 1.0d, 1.5d}); doc1.put("double1", new Double[]{1.2d, 2.4d}); doc1.put("double2", 3.0d); Map doc2 = new HashMap<>(); + doc2.put("id", 2); doc2.put("double0", 5.0d); doc2.put("double1", 3.0d); Map doc3 = new HashMap<>(); + doc3.put("id", 3); doc3.put("double0", new Double[]{5.0d, 1.0d, 1.5d, -1.5d}); doc3.put("double1", 4.0d); @@ -319,8 +325,8 @@ public void testSparseField() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long")); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("x", 4), - client().prepareIndex("test").setId("2").setSource("y", 2)); + client().prepareIndex("test").setId("1").setSource("id", 1, "x", 4), + client().prepareIndex("test").setId("2").setSource("id", 2, "y", 2)); SearchResponse rsp = buildRequest("doc['x'] + 1").get(); ElasticsearchAssertions.assertSearchResponse(rsp); SearchHits hits = rsp.getHits(); @@ -348,9 +354,9 @@ public void testParams() throws Exception { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("x", 10), - client().prepareIndex("test").setId("2").setSource("x", 3), - client().prepareIndex("test").setId("3").setSource("x", 5)); + client().prepareIndex("test").setId("1").setSource("id", 1, "x", 10), + client().prepareIndex("test").setId("2").setSource("id", 2, "x", 3), + client().prepareIndex("test").setId("3").setSource("id", 3, "x", 5)); // a = int, b = double, c = long String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)"; SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get(); @@ -621,9 +627,9 @@ public void testBoolean() throws Exception { assertAcked(prepareCreate("test").addMapping("doc", xContentBuilder)); ensureGreen(); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("price", 1.0, "vip", true), - client().prepareIndex("test").setId("2").setSource("price", 2.0, "vip", false), - client().prepareIndex("test").setId("3").setSource("price", 2.0, "vip", false)); + client().prepareIndex("test").setId("1").setSource("id", 1, "price", 1.0, "vip", true), + client().prepareIndex("test").setId("2").setSource("id", 2, "price", 2.0, "vip", false), + client().prepareIndex("test").setId("3").setSource("id", 3, "price", 2.0, "vip", false)); // access .value SearchResponse rsp = buildRequest("doc['vip'].value").get(); assertSearchResponse(rsp); @@ -652,8 +658,8 @@ public void testFilterScript() throws Exception { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("foo", 1.0), - client().prepareIndex("test").setId("2").setSource("foo", 0.0)); + client().prepareIndex("test").setId("1").setSource("id", 1, "foo", 1.0), + client().prepareIndex("test").setId("2").setSource("id", 2, "foo", 0.0)); SearchRequestBuilder builder = buildRequest("doc['foo'].value"); Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap()); builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script))); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java index 33e5c86b10b3a..150c1dc76afa5 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java @@ -103,7 +103,7 @@ public void testParentWithMultipleBuckets() throws Exception { .setQuery(matchQuery("randomized", false)) .addAggregation( terms("category").field("category").size(10000).subAggregation( - children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_id", SortOrder.ASC)) + children("to_comment", "comment").subAggregation(topHits("top_comments").sort("id", SortOrder.ASC)) ) ).get(); assertSearchResponse(searchResponse); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java index a5c56ee11be29..f6163eeca8aca 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java @@ -189,6 +189,9 @@ public void testSimpleParentChild() throws Exception { public void testRandomParentChild() throws Exception { assertAcked(prepareCreate("idx") .addMapping("doc", jsonBuilder().startObject().startObject("doc").startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("join_field") .field("type", "join") .startObject("relations") @@ -225,13 +228,13 @@ public void testRandomParentChild() throws Exception { BoolQueryBuilder boolQuery = new BoolQueryBuilder(); boolQuery.should(constantScoreQuery(hasChildQuery("child1", matchAllQuery(), ScoreMode.None) .innerHit(new InnerHitBuilder().setName("a") - .addSort(new FieldSortBuilder("_id").order(SortOrder.ASC)).setSize(size)))); + .addSort(new FieldSortBuilder("id").order(SortOrder.ASC)).setSize(size)))); boolQuery.should(constantScoreQuery(hasChildQuery("child2", matchAllQuery(), ScoreMode.None) .innerHit(new InnerHitBuilder().setName("b") - .addSort(new FieldSortBuilder("_id").order(SortOrder.ASC)).setSize(size)))); + .addSort(new FieldSortBuilder("id").order(SortOrder.ASC)).setSize(size)))); SearchResponse searchResponse = client().prepareSearch("idx") .setSize(numDocs) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(boolQuery) .get(); @@ -283,7 +286,7 @@ public void testInnerHitsOnHasParent() throws Exception { indexRandom(true, requests); SearchResponse response = client().prepareSearch("stack") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery( boolQuery() .must(matchQuery("body", "fail2ban")) @@ -386,7 +389,7 @@ public void testRoyals() throws Exception { hasChildQuery("baron", matchAllQuery(), ScoreMode.None) .innerHit(new InnerHitBuilder().setName("barons")), ScoreMode.None).innerHit(new InnerHitBuilder() - .addSort(SortBuilders.fieldSort("_id").order(SortOrder.ASC)) + .addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC)) .setName("earls") .setSize(4)) ) @@ -440,7 +443,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { SearchResponse response = client().prepareSearch("index") .setQuery(hasChildQuery("child", matchQuery("field", "value1").queryName("_name1"), ScoreMode.None) .innerHit(new InnerHitBuilder())) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -457,7 +460,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { .innerHit(new InnerHitBuilder()); response = client().prepareSearch("index") .setQuery(query) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java index 8436e2b8a3207..99e02aaf3d7cd 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java @@ -65,6 +65,7 @@ protected IndexRequestBuilder createIndexRequest(String index, String type, Stri for (int i = 0; i < fields.length; i += 2) { source.put((String) fields[i], fields[i + 1]); } + source.put("id", id); return createIndexRequest(index, type, id, parentId, source); } @@ -93,6 +94,7 @@ public static Map buildParentJoinFieldMappingFromSimplifiedDef(S } joinField.put("relations", relationMap); fields.put(joinFieldName, joinField); + fields.put("id", Collections.singletonMap("type", "keyword")); return Collections.singletonMap("properties", fields); } diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml index 9d8488f5c1451..0b65c744ec0b2 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml @@ -6,46 +6,47 @@ setup: mappings: properties: join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } } + id: { "type": "keyword" } - do: index: index: test id: 1 - body: { "join_field": { "name": "parent" } } + body: { "id", "1", "join_field": { "name": "parent" } } - do: index: index: test id: 2 - body: { "join_field": { "name": "parent" } } + body: { "id", "2", "join_field": { "name": "parent" } } - do: index: index: test id: 3 routing: 1 - body: { "join_field": { "name": "child", "parent": "1" } } + body: { "id", "3", "join_field": { "name": "child", "parent": "1" } } - do: index: index: test id: 4 routing: 1 - body: { "join_field": { "name": "child", "parent": "1" } } + body: { "id", "4", "join_field": { "name": "child", "parent": "1" } } - do: index: index: test id: 5 routing: 1 - body: { "join_field": { "name": "child", "parent": "2" } } + body: { "id", "5", "join_field": { "name": "child", "parent": "2" } } - do: index: index: test id: 6 routing: 1 - body: { "join_field": { "name": "grand_child", "parent": "5" } } + body: { "id", "6", "join_field": { "name": "grand_child", "parent": "5" } } - do: indices.refresh: {} @@ -55,7 +56,7 @@ setup: - do: search: rest_total_hits_as_int: true - body: { sort: ["join_field", "_id"] } + body: { sort: ["join_field", "id"] } - match: { hits.total: 6 } - match: { hits.hits.0._index: "test" } @@ -92,7 +93,7 @@ setup: search: rest_total_hits_as_int: true body: - sort: [ "_id" ] + sort: [ "id" ] query: parent_id: type: child diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 92a1802b39623..041073ebcfc3d 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -68,17 +68,23 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase { public void testPercolatorQuery() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") + .addMapping("type", "id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") ); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("query", matchAllQuery()).endObject()) .get(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "2") + .field("query", matchQuery("field1", "value")).endObject()) .get(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", boolQuery() + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("query", boolQuery() .must(matchQuery("field1", "value")) .must(matchQuery("field2", "value")) ).endObject()).get(); @@ -96,7 +102,7 @@ public void testPercolatorQuery() throws Exception { logger.info("percolating doc with 1 field"); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -108,7 +114,7 @@ public void testPercolatorQuery() throws Exception { logger.info("percolating doc with 2 fields"); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -124,7 +130,7 @@ public void testPercolatorQuery() throws Exception { BytesReference.bytes(jsonBuilder().startObject().field("field1", "value").endObject()), BytesReference.bytes(jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject()) ), XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -247,22 +253,26 @@ public void testPercolatorRangeQueries() throws Exception { public void testPercolatorGeoQueries() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", "type=geo_point", "field2", "type=geo_shape", "query", "type=percolator") - ); + .addMapping("type", "id", "type=keyword", + "field1", "type=geo_point", "field2", "type=geo_shape", "query", "type=percolator")); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", - geoDistanceQuery("field1").point(52.18, 4.38).distance(50, DistanceUnit.KILOMETERS)) + .setSource(jsonBuilder().startObject() + .field("query", geoDistanceQuery("field1").point(52.18, 4.38).distance(50, DistanceUnit.KILOMETERS)) + .field("id", "1") .endObject()).get(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", - geoBoundingBoxQuery("field1").setCorners(52.3, 4.4, 52.1, 4.6)) + .setSource(jsonBuilder().startObject() + .field("query", geoBoundingBoxQuery("field1").setCorners(52.3, 4.4, 52.1, 4.6)) + .field("id", "2") .endObject()).get(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", - geoPolygonQuery("field1", Arrays.asList(new GeoPoint(52.1, 4.4), new GeoPoint(52.3, 4.5), new GeoPoint(52.1, 4.6)))) + .setSource(jsonBuilder().startObject() + .field("query", + geoPolygonQuery("field1", Arrays.asList(new GeoPoint(52.1, 4.4), new GeoPoint(52.3, 4.5), new GeoPoint(52.1, 4.6)))) + .field("id", "3") .endObject()).get(); refresh(); @@ -271,7 +281,7 @@ public void testPercolatorGeoQueries() throws Exception { .endObject()); SearchResponse response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -281,24 +291,29 @@ public void testPercolatorGeoQueries() throws Exception { public void testPercolatorQueryExistingDocument() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") + .addMapping("type", "id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") ); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("query", matchAllQuery()).endObject()) .get(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "2") + .field("query", matchQuery("field1", "value")).endObject()) .get(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", boolQuery() + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("query", boolQuery() .must(matchQuery("field1", "value")) - .must(matchQuery("field2", "value")) - ).endObject()).get(); + .must(matchQuery("field2", "value"))).endObject()).get(); - client().prepareIndex("test").setId("4").setSource("{}", XContentType.JSON).get(); - client().prepareIndex("test").setId("5").setSource("field1", "value").get(); - client().prepareIndex("test").setId("6").setSource("field1", "value", "field2", "value").get(); + client().prepareIndex("test").setId("4").setSource("{\"id\": \"4\"}", XContentType.JSON).get(); + client().prepareIndex("test").setId("5").setSource("id", "5", "field1", "value").get(); + client().prepareIndex("test").setId("6").setSource("id", "6", "field1", "value", "field2", "value").get(); client().admin().indices().prepareRefresh().get(); logger.info("percolating empty doc"); @@ -311,7 +326,7 @@ public void testPercolatorQueryExistingDocument() throws Exception { logger.info("percolating doc with 1 field"); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", "test", "5", null, null, null)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -320,7 +335,7 @@ public void testPercolatorQueryExistingDocument() throws Exception { logger.info("percolating doc with 2 fields"); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", "test", "6", null, null, null)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -351,15 +366,19 @@ public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception public void testPercolatorSpecificQueries() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", "type=text", "field2", "type=text", "query", "type=percolator") + .addMapping("type", "id", "type=keyword", "field1", "type=text", "field2", "type=text", "query", "type=percolator") ); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", multiMatchQuery("quick brown fox", "field1", "field2") + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("query", multiMatchQuery("quick brown fox", "field1", "field2") .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)).endObject()) .get(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", + .setSource(jsonBuilder().startObject() + .field("id", "2") + .field("query", spanNearQuery(spanTermQuery("field1", "quick"), 0) .addClause(spanTermQuery("field1", "brown")) .addClause(spanTermQuery("field1", "fox")) @@ -369,7 +388,9 @@ public void testPercolatorSpecificQueries() throws Exception { client().admin().indices().prepareRefresh().get(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("query", spanNotQuery( spanNearQuery(spanTermQuery("field1", "quick"), 0) .addClause(spanTermQuery("field1", "brown")) @@ -384,7 +405,9 @@ public void testPercolatorSpecificQueries() throws Exception { // doesn't match client().prepareIndex("test").setId("4") - .setSource(jsonBuilder().startObject().field("query", + .setSource(jsonBuilder().startObject() + .field("id", "4") + .field("query", spanNotQuery( spanNearQuery(spanTermQuery("field1", "quick"), 0) .addClause(spanTermQuery("field1", "brown")) @@ -404,7 +427,7 @@ public void testPercolatorSpecificQueries() throws Exception { .endObject()); SearchResponse response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -424,22 +447,32 @@ public void testPercolatorQueryWithHighlighting() throws Exception { fieldMapping.append(",index_options=offsets"); } assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", fieldMapping, "query", "type=percolator") + .addMapping("type", "id", "type=keyword", "field1", fieldMapping, "query", "type=percolator") ); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "brown fox")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("query", matchQuery("field1", "brown fox")).endObject()) .execute().actionGet(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "lazy dog")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "2") + .field("query", matchQuery("field1", "lazy dog")).endObject()) .execute().actionGet(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "jumps")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("query", termQuery("field1", "jumps")).endObject()) .execute().actionGet(); client().prepareIndex("test").setId("4") - .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "dog")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "4") + .field("query", termQuery("field1", "dog")).endObject()) .execute().actionGet(); client().prepareIndex("test").setId("5") - .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "fox")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "5") + .field("query", termQuery("field1", "fox")).endObject()) .execute().actionGet(); client().admin().indices().prepareRefresh().get(); @@ -449,7 +482,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { SearchResponse searchResponse = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", document, XContentType.JSON)) .highlighter(new HighlightBuilder().field("field1")) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(searchResponse, 5); @@ -476,7 +509,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { .should(new PercolateQueryBuilder("query", document2, XContentType.JSON).setName("query2")) ) .highlighter(new HighlightBuilder().field("field1")) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); logger.info("searchResponse={}", searchResponse); assertHitCount(searchResponse, 5); @@ -500,7 +533,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { BytesReference.bytes(jsonBuilder().startObject().field("field1", "brown fox").endObject()) ), XContentType.JSON)) .highlighter(new HighlightBuilder().field("field1")) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(searchResponse, 5); assertThat(searchResponse.getHits().getAt(0).getFields().get("_percolator_document_slot").getValues(), @@ -540,7 +573,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { ), XContentType.JSON).setName("query2")) ) .highlighter(new HighlightBuilder().field("field1")) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); logger.info("searchResponse={}", searchResponse); assertHitCount(searchResponse, 5); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 271cfa630b470..fd9ce3cf8a6f0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -61,20 +61,20 @@ public void setup() { ensureGreen(); client().prepareIndex(TEST_INDEX).setId("1") - .setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get(); - client().prepareIndex(TEST_INDEX).setId("2").setSource("text", "amsterdam", "population", 851573) + .setSource("id", 1, "text", "berlin", "title", "Berlin, Germany", "population", 3670622).get(); + client().prepareIndex(TEST_INDEX).setId("2").setSource("id", 2, "text", "amsterdam", "population", 851573) .get(); - client().prepareIndex(TEST_INDEX).setId("3").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex(TEST_INDEX).setId("3").setSource("id", 3, "text", "amsterdam", "population", 851573) .get(); - client().prepareIndex(TEST_INDEX).setId("4").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex(TEST_INDEX).setId("4").setSource("id", 4, "text", "amsterdam", "population", 851573) .get(); - client().prepareIndex(TEST_INDEX).setId("5").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex(TEST_INDEX).setId("5").setSource("id", 5, "text", "amsterdam", "population", 851573) .get(); - client().prepareIndex(TEST_INDEX).setId("6").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex(TEST_INDEX).setId("6").setSource("id", 6, "text", "amsterdam", "population", 851573) .get(); // add another index for testing closed indices etc... - client().prepareIndex("test2").setId("7").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex("test2").setId("7").setSource("id", 7, "text", "amsterdam", "population", 851573) .get(); refresh(); @@ -91,7 +91,7 @@ public void testPrecisionAtRequest() { List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - testQuery.sort("_id"); + testQuery.sort("id"); RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("2", "3", "4", "5"), testQuery); amsterdamRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" })); @@ -168,7 +168,7 @@ public void testPrecisionAtRequest() { public void testDCGRequest() { SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - testQuery.sort("_id"); + testQuery.sort("id"); List specifications = new ArrayList<>(); List ratedDocs = Arrays.asList( @@ -202,7 +202,7 @@ public void testDCGRequest() { public void testMRRRequest() { SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - testQuery.sort("_id"); + testQuery.sort("id"); List specifications = new ArrayList<>(); specifications.add(new RatedRequest("amsterdam_query", createRelevant("5"), testQuery)); diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java index abecbcb1582e8..9242d61f24774 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java @@ -65,6 +65,9 @@ public void testBasicUsage() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "tr") @@ -76,8 +79,10 @@ public void testBasicUsage() throws Exception { // both values should collate to same value indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1") + .setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2") + .setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); // searching for either of the terms should return both results since they collate to the same value @@ -87,7 +92,7 @@ public void testBasicUsage() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate") - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); @@ -104,6 +109,9 @@ public void testMultipleValues() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -114,9 +122,10 @@ public void testMultipleValues() throws Exception { // everything should be indexed fine, no exceptions indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":[\"" + equivalent[0] + "\", \"" - + equivalent[1] + "\"]}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[2] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1") + .setSource("{\"id\":\"1\", \"collate\":[\"" + equivalent[0] + "\", \"" + equivalent[1] + "\"]}", XContentType.JSON), + client().prepareIndex(index).setId("2") + .setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[2] + "\"}", XContentType.JSON) ); // using sort mode = max, values B and C will be used for the sort @@ -127,7 +136,7 @@ public void testMultipleValues() throws Exception { .query(QueryBuilders.termQuery("collate", "a")) // if mode max we use c and b as sort values, if max we use "a" for both .sort(SortBuilders.fieldSort("collate").sortMode(SortMode.MAX).order(SortOrder.DESC)) - .sort("_id", SortOrder.DESC) // will be ignored + .sort("id", SortOrder.DESC) // will be ignored ); SearchResponse response = client().search(request).actionGet(); @@ -143,7 +152,7 @@ public void testMultipleValues() throws Exception { .query(QueryBuilders.termQuery("collate", "a")) // if mode max we use c and b as sort values, if max we use "a" for both .sort(SortBuilders.fieldSort("collate").sortMode(SortMode.MIN).order(SortOrder.DESC)) - .sort("_id", SortOrder.DESC) // will NOT be ignored and will determine order + .sort("id", SortOrder.DESC) // will NOT be ignored and will determine order ); response = client().search(request).actionGet(); @@ -163,6 +172,9 @@ public void testNormalization() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "tr") @@ -174,8 +186,10 @@ public void testNormalization() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1") + .setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2") + .setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); // searching for either of the terms should return both results since they collate to the same value @@ -185,7 +199,7 @@ public void testNormalization() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate") - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); @@ -205,6 +219,9 @@ public void testSecondaryStrength() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -216,8 +233,10 @@ public void testSecondaryStrength() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1") + .setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2") + .setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -226,7 +245,7 @@ public void testSecondaryStrength() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate") - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); @@ -247,6 +266,9 @@ public void testIgnorePunctuation() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -258,8 +280,8 @@ public void testIgnorePunctuation() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -268,7 +290,7 @@ public void testIgnorePunctuation() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate") - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); @@ -287,6 +309,9 @@ public void testIgnoreWhitespace() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -300,9 +325,9 @@ public void testIgnoreWhitespace() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"foo bar\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"foobar\"}", XContentType.JSON), - client().prepareIndex(index).setId("3").setSource("{\"collate\":\"foo-bar\"}", XContentType.JSON) + client().prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"foo bar\"}", XContentType.JSON), + client().prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"foobar\"}", XContentType.JSON), + client().prepareIndex(index).setId("3").setSource("{\"id\":\"3\",\"collate\":\"foo-bar\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -310,7 +335,7 @@ public void testIgnoreWhitespace() throws Exception { .source(new SearchSourceBuilder() .fetchSource(false) .sort("collate", SortOrder.ASC) - .sort("_id", SortOrder.ASC) // secondary sort should kick in on docs 1 and 3 because same value collate value + .sort("id", SortOrder.ASC) // secondary sort should kick in on docs 1 and 3 because same value collate value ); SearchResponse response = client().search(request).actionGet(); @@ -367,6 +392,9 @@ public void testIgnoreAccentsButNotCase() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -379,10 +407,10 @@ public void testIgnoreAccentsButNotCase() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"résumé\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"Resume\"}", XContentType.JSON), - client().prepareIndex(index).setId("3").setSource("{\"collate\":\"resume\"}", XContentType.JSON), - client().prepareIndex(index).setId("4").setSource("{\"collate\":\"Résumé\"}", XContentType.JSON) + client().prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"résumé\"}", XContentType.JSON), + client().prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"Resume\"}", XContentType.JSON), + client().prepareIndex(index).setId("3").setSource("{\"id\":\"3\",\"collate\":\"resume\"}", XContentType.JSON), + client().prepareIndex(index).setId("4").setSource("{\"id\":\"4\",\"collate\":\"Résumé\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -390,7 +418,7 @@ public void testIgnoreAccentsButNotCase() throws Exception { .source(new SearchSourceBuilder() .fetchSource(false) .sort("collate", SortOrder.ASC) - .sort("_id", SortOrder.DESC) + .sort("id", SortOrder.DESC) ); SearchResponse response = client().search(request).actionGet(); @@ -462,6 +490,9 @@ public void testCustomRules() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("rules", tailoredRules) @@ -472,8 +503,8 @@ public void testCustomRules() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -482,7 +513,7 @@ public void testCustomRules() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate", SortOrder.ASC) - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java index 558e6071255b1..a4e76fc7ec436 100644 --- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java @@ -165,7 +165,7 @@ private static void indexDocuments(String idPrefix) throws IOException, Interrup //this index with a single document is used to test partial failures IndexRequest indexRequest = new IndexRequest(INDEX_NAME + "_err"); indexRequest.id("id"); - indexRequest.source("creationDate", "err"); + indexRequest.source("id", "id", "creationDate", "err"); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT); assertEquals(201, indexResponse.status().getStatus()); @@ -178,6 +178,7 @@ private static void indexDocuments(String idPrefix) throws IOException, Interrup CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME); createIndexRequest.settings(Settings.builder().put("index.number_of_shards", numShards).put("index.number_of_replicas", 0)); createIndexRequest.mapping("{\"properties\":{" + + "\"id\":{\"type\":\"keyword\"}," + "\"suggest\":{\"type\":\"completion\"}," + "\"join\":{\"type\":\"join\", \"relations\": {\"question\":\"answer\"}}}}", XContentType.JSON); CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT); @@ -237,6 +238,7 @@ private static IndexRequest buildIndexRequest(String id, String type, String que joinField.put("parent", questionId); } indexRequest.source(XContentType.JSON, + "id", id, "type", type, "votes", randomIntBetween(0, 30), "questionId", questionId, @@ -614,7 +616,7 @@ public void testTopHits() throws Exception { topHits.from(10); topHits.size(10); topHits.sort("creationDate", SortOrder.DESC); - topHits.sort("_id", SortOrder.ASC); + topHits.sort("id", SortOrder.ASC); TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING); tags.field("tags.keyword"); tags.size(10); diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index 375ba12a35621..fa86389f0db2a 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -42,7 +42,7 @@ rest_total_hits_as_int: true index: queries body: - sort: _id + sort: id query: percolate: field: query diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index 2672cee7cc78a..e1ffcea930a42 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -66,6 +66,8 @@ body: mappings: properties: + id: + type: keyword query: type: percolator field1: @@ -80,6 +82,7 @@ index: queries id: q1 body: + id: q1 query: term: field1: value @@ -89,6 +92,7 @@ index: queries id: q2 body: + id: q2 query: bool: must: @@ -102,6 +106,7 @@ index: queries id: q3 body: + id: q3 query: bool: minimum_should_match: 2 @@ -133,7 +138,7 @@ rest_total_hits_as_int: true index: queries body: - sort: _id + sort: id query: percolate: field: query @@ -149,7 +154,7 @@ rest_total_hits_as_int: true index: queries body: - sort: _id + sort: id query: percolate: field: query diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 78a4aac867d8f..4368ffd602f58 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -30,6 +30,7 @@ id: q4 refresh: true body: + id: q4 query: bool: minimum_should_match: 2 @@ -57,7 +58,7 @@ rest_total_hits_as_int: true index: queries body: - sort: _id + sort: id query: percolate: field: query diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index ff001056957b7..98c8bdff66fc6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -6,19 +6,19 @@ setup: index: index: test id: 1 - body: { foo: bar, age: 18 } + body: { id: 1, foo: bar, age: 18 } - do: index: index: test id: 42 - body: { foo: bar, age: 18 } + body: { id: 42, foo: bar, age: 18 } - do: index: index: test id: 172 - body: { foo: bar, age: 24 } + body: { id: 172, foo: bar, age: 24 } - do: indices.refresh: @@ -36,13 +36,13 @@ setup: query: match: foo: bar - sort: [{ age: desc }, { _id: desc }] + sort: [{ age: desc }, { id: desc }] - match: {hits.total: 3 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - match: {hits.hits.0._id: "172" } - - match: {hits.hits.0.sort: [24, "172"] } + - match: {hits.hits.0.sort: [24, 172] } - do: search: @@ -53,14 +53,14 @@ setup: query: match: foo: bar - sort: [{ age: desc }, { _id: desc }] - search_after: [24, "172"] + sort: [{ age: desc }, { id: desc }] + search_after: [24, 172] - match: {hits.total: 3 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - match: {hits.hits.0._id: "42" } - - match: {hits.hits.0.sort: [18, "42"] } + - match: {hits.hits.0.sort: [18, 42] } - do: search: @@ -71,14 +71,14 @@ setup: query: match: foo: bar - sort: [ { age: desc }, { _id: desc } ] - search_after: [18, "42"] + sort: [ { age: desc }, { id: desc } ] + search_after: [18, 42] - match: {hits.total: 3} - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - match: {hits.hits.0._id: "1" } - - match: {hits.hits.0.sort: [18, "1"] } + - match: {hits.hits.0.sort: [18, 1] } - do: search: @@ -89,8 +89,8 @@ setup: query: match: foo: bar - sort: [{ age: desc }, { _id: desc } ] - search_after: [18, "1"] + sort: [{ age: desc }, { id: desc } ] + search_after: [18, 1] - match: {hits.total: 3} - length: {hits.hits: 0 } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 3ad8f099c69f0..52ee0be49ef65 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -180,7 +180,7 @@ public Set> entrySet() { try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(analyzerMap, analyzerMap, analyzerMap)) { MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService, - mapperRegistry, () -> null); + mapperRegistry, () -> null, () -> false); mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY); } } catch (Exception ex) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index f51dfb62e0593..4548448719a68 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -196,6 +196,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndicesQueryCache.INDICES_CACHE_QUERY_SIZE_SETTING, IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING, IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING, + IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING, MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING, MetaData.SETTING_READ_ONLY_SETTING, MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index b42bb2c2cb384..f1ce91b212b43 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -74,6 +74,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; @@ -394,7 +395,8 @@ public IndexService newIndexService( IndicesQueryCache indicesQueryCache, MapperRegistry mapperRegistry, IndicesFieldDataCache indicesFieldDataCache, - NamedWriteableRegistry namedWriteableRegistry) + NamedWriteableRegistry namedWriteableRegistry, + BooleanSupplier idFieldDataEnabled) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = @@ -422,7 +424,7 @@ public IndexService newIndexService( new SimilarityService(indexSettings, scriptService, similarities), shardStoreDeleter, indexAnalyzers, engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, client, queryCache, directoryFactory, eventListener, readerWrapperFactory, mapperRegistry, indicesFieldDataCache, searchOperationListeners, - indexOperationListeners, namedWriteableRegistry); + indexOperationListeners, namedWriteableRegistry, idFieldDataEnabled); success = true; return indexService; } finally { @@ -469,7 +471,7 @@ public MapperService newIndexMapperService(NamedXContentRegistry xContentRegistr ScriptService scriptService) throws IOException { return new MapperService(indexSettings, analysisRegistry.build(indexSettings), xContentRegistry, new SimilarityService(indexSettings, scriptService, similarities), mapperRegistry, - () -> { throw new UnsupportedOperationException("no index query shard context available"); }); + () -> { throw new UnsupportedOperationException("no index query shard context available"); }, () -> false); } /** diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 7c29e342cb1de..47c3bfb35fee7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -92,6 +92,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.LongSupplier; @@ -161,7 +162,8 @@ public IndexService( IndicesFieldDataCache indicesFieldDataCache, List searchOperationListeners, List indexingOperationListeners, - NamedWriteableRegistry namedWriteableRegistry) { + NamedWriteableRegistry namedWriteableRegistry, + BooleanSupplier idFieldDataEnabled) { super(indexSettings); this.indexSettings = indexSettings; this.xContentRegistry = xContentRegistry; @@ -172,7 +174,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null)); + () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index e0a2cd7ee428b..521521915caf1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.logging.log4j.LogManager; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -28,6 +29,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.Index; @@ -41,6 +43,7 @@ import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; @@ -55,6 +58,11 @@ * queries. */ public class IdFieldMapper extends MetadataFieldMapper { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(IdFieldMapper.class)); + static final String ID_FIELD_DATA_DEPRECATION_MESSAGE = + "Loading the fielddata on the _id field is deprecated and will be removed in future versions. " + + "If you require sorting or aggregating on this field you should also include the id in the " + + "body of your documents, and map this field as a keyword field that has [doc_values] enabled"; public static final String NAME = "_id"; @@ -158,6 +166,12 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + if (mapperService.isIdFieldDataEnabled() == false) { + throw new IllegalArgumentException("Fielddata access on the _id field is disallowed, " + + "you can re-enable it by updating the dynamic cluster setting: " + + IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey()); + } + deprecationLogger.deprecatedAndMaybeLog("id_field_data", ID_FIELD_DATA_DEPRECATION_MESSAGE); final IndexFieldData fieldData = fieldDataBuilder.build(indexSettings, fieldType, cache, breakerService, mapperService); return new IndexFieldData() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 2a57cacf7ff45..4b29856ee94ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -67,6 +67,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; @@ -142,9 +143,11 @@ public enum MergeReason { final MapperRegistry mapperRegistry; + private final BooleanSupplier idFieldDataEnabled; + public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, NamedXContentRegistry xContentRegistry, SimilarityService similarityService, MapperRegistry mapperRegistry, - Supplier queryShardContextSupplier) { + Supplier queryShardContextSupplier, BooleanSupplier idFieldDataEnabled) { super(indexSettings); this.indexAnalyzers = indexAnalyzers; this.fieldTypes = new FieldTypeLookup(); @@ -154,12 +157,12 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer()); this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer()); this.mapperRegistry = mapperRegistry; + this.idFieldDataEnabled = idFieldDataEnabled; if (INDEX_MAPPER_DYNAMIC_SETTING.exists(indexSettings.getSettings()) && indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0)) { throw new IllegalArgumentException("Setting " + INDEX_MAPPER_DYNAMIC_SETTING.getKey() + " was removed after version 6.0.0"); } - } public boolean hasNested() { @@ -663,6 +666,13 @@ public Analyzer searchQuoteAnalyzer() { return this.searchQuoteAnalyzer; } + /** + * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. + */ + public boolean isIdFieldDataEnabled() { + return idFieldDataEnabled.getAsBoolean(); + } + @Override public void close() throws IOException { indexAnalyzers.close(); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index b72da768a70dd..4810f9f00e8e5 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -92,6 +92,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.query.QueryBuilder; @@ -168,6 +169,9 @@ public class IndicesService extends AbstractLifecycleComponent public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout"; public static final Setting INDICES_CACHE_CLEAN_INTERVAL_SETTING = Setting.positiveTimeSetting("indices.cache.cleanup_interval", TimeValue.timeValueMinutes(1), Property.NodeScope); + public static final Setting INDICES_ID_FIELD_DATA_ENABLED_SETTING = + Setting.boolSetting("indices.id_field_data.enabled", false, Property.Dynamic, Property.NodeScope); + /** * The node's settings. @@ -203,6 +207,7 @@ public class IndicesService extends AbstractLifecycleComponent private final Map directoryFactories; final AbstractRefCounted indicesRefCount; // pkg-private for testing private final CountDownLatch closeLatch = new CountDownLatch(1); + private volatile boolean idFieldDataEnabled; @Override protected void doStart() { @@ -238,6 +243,8 @@ public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvi this.scriptService = scriptService; this.clusterService = clusterService; this.client = client; + this.idFieldDataEnabled = INDICES_ID_FIELD_DATA_ENABLED_SETTING.get(clusterService.getSettings()); + clusterService.getClusterSettings().addSettingsUpdateConsumer(INDICES_ID_FIELD_DATA_ENABLED_SETTING, this::setIdFieldDataEnabled); this.indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() { @Override public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) { @@ -558,7 +565,8 @@ private synchronized IndexService createIndexService(IndexService.IndexCreationC indicesQueryCache, mapperRegistry, indicesFieldDataCache, - namedWriteableRegistry + namedWriteableRegistry, + this::isIdFieldDataEnabled ); } @@ -1450,6 +1458,17 @@ public boolean isMetaDataField(Version indexCreatedVersion, String field) { return mapperRegistry.isMetaDataField(indexCreatedVersion, field); } + /** + * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. + */ + public boolean isIdFieldDataEnabled() { + return idFieldDataEnabled; + } + + private void setIdFieldDataEnabled(boolean value) { + this.idFieldDataEnabled = value; + } + /** * Checks to see if an operation can be performed without taking the cluster over the cluster-wide shard limit. Adds a deprecation * warning or returns an error message as appropriate diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 6310bf0457be4..adeb49faa8941 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -160,7 +160,7 @@ public void tearDown() throws Exception { private IndexService newIndexService(IndexModule module) throws IOException { return module.newIndexService(CREATE_INDEX, nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, null, indicesQueryCache, mapperRegistry, - new IndicesFieldDataCache(settings, listener), writableRegistry()); + new IndicesFieldDataCache(settings, listener), writableRegistry(), () -> false); } public void testWrapperIsBound() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index c225b090816ec..fa775a84c72ad 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -94,7 +94,7 @@ private CodecService createCodecService() throws IOException { IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry, - () -> null); + () -> null, () -> false); return new CodecService(service, LogManager.getLogger("test")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java index 4f2198c577a3f..f495da0186ab6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java @@ -28,7 +28,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -37,6 +39,9 @@ import java.util.Collection; import java.util.Collections; +import static org.elasticsearch.index.mapper.IdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE; +import static org.hamcrest.Matchers.containsString; + public class IdFieldMapperTests extends ESSingleNodeTestCase { @Override @@ -71,4 +76,30 @@ public void testDefaults() throws IOException { assertEquals(Uid.encodeId("id"), fields[0].binaryValue()); } + public void testEnableFieldData() throws IOException { + IndexService service = createIndex("test", Settings.EMPTY); + MapperService mapperService = service.mapperService(); + mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE); + IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) service.mapperService().fullName("_id"); + + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, + () -> ft.fielddataBuilder("test").build(mapperService.getIndexSettings(), + ft, null, null, mapperService)); + assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); + + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true)) + .get(); + try { + ft.fielddataBuilder("test").build(mapperService.getIndexSettings(), + ft, null, null, mapperService); + assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE); + } finally { + // unset cluster setting + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())) + .get(); + } + } + } diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java index d272e0603240a..fc991ca43cbc9 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java +++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java @@ -119,6 +119,7 @@ public void testProfileMatchesRegular() throws Exception { IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; i++) { docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource( + "id", String.valueOf(i), "field1", English.intToEnglish(i), "field2", i ); @@ -136,14 +137,14 @@ public void testProfileMatchesRegular() throws Exception { SearchRequestBuilder vanilla = client().prepareSearch("test") .setQuery(q) .setProfile(false) - .addSort("_id", SortOrder.ASC) + .addSort("id.keyword", SortOrder.ASC) .setSearchType(SearchType.QUERY_THEN_FETCH) .setRequestCache(false); SearchRequestBuilder profile = client().prepareSearch("test") .setQuery(q) .setProfile(true) - .addSort("_id", SortOrder.ASC) + .addSort("id.keyword", SortOrder.ASC) .setSearchType(SearchType.QUERY_THEN_FETCH) .setRequestCache(false); diff --git a/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java index cb0b448ede879..a15c33832b8df 100644 --- a/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java @@ -97,6 +97,7 @@ public void init() throws Exception { int numDocs = scaledRandomIntBetween(50, 100); List builders = new ArrayList<>(); builders.add(client().prepareIndex("test").setId("theone").setSource( + "id", "theone", "full_name", "Captain America", "first_name", "Captain", "last_name", "America", @@ -104,6 +105,7 @@ public void init() throws Exception { "skill", 15, "int-field", 25)); builders.add(client().prepareIndex("test").setId("theother").setSource( + "id", "theother", "full_name", "marvel hero", "first_name", "marvel", "last_name", "hero", @@ -111,6 +113,7 @@ public void init() throws Exception { "skill", 5)); builders.add(client().prepareIndex("test").setId("ultimate1").setSource( + "id", "ultimate1", "full_name", "Alpha the Ultimate Mutant", "first_name", "Alpha the", "last_name", "Ultimate Mutant", @@ -124,6 +127,7 @@ public void init() throws Exception { "skill", 3)); builders.add(client().prepareIndex("test").setId("anotherhero").setSource( + "id", "anotherhero", "full_name", "ultimate", "first_name", "wolferine", "last_name", "", @@ -131,6 +135,7 @@ public void init() throws Exception { "skill", 1)); builders.add(client().prepareIndex("test").setId("nowHero").setSource( + "id", "nowHero", "full_name", "now sort of", "first_name", "now", "last_name", "", @@ -147,6 +152,7 @@ public void init() throws Exception { String first = RandomPicks.randomFrom(random(), firstNames); String last = randomPickExcept(lastNames, first); builders.add(client().prepareIndex("test").setId("" + i).setSource( + "id", i, "full_name", first + " " + last, "first_name", first, "last_name", last, @@ -159,6 +165,9 @@ public void init() throws Exception { private XContentBuilder createMapping() throws IOException { return XContentFactory.jsonBuilder().startObject().startObject("test") .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("full_name") .field("type", "text") .field("copy_to", "full_name_phrase") @@ -274,17 +283,17 @@ public void testSingleField() throws NoSuchFieldException, IllegalAccessExceptio } MultiMatchQueryBuilder multiMatchQueryBuilder = randomizeType(multiMatchQuery(builder.toString(), field)); SearchResponse multiMatchResp = client().prepareSearch("test") - // _id sort field is a tie, in case hits have the same score, + // id sort field is a tie, in case hits have the same score, // the hits will be sorted the same consistently .addSort("_score", SortOrder.DESC) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(multiMatchQueryBuilder).get(); MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery(field, builder.toString()); SearchResponse matchResp = client().prepareSearch("test") - // _id tie sort + // id tie sort .addSort("_score", SortOrder.DESC) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(matchQueryBuilder).get(); assertThat("field: " + field + " query: " + builder.toString(), multiMatchResp.getHits().getTotalHits().value, equalTo(matchResp.getHits().getTotalHits().value)); @@ -312,12 +321,12 @@ public void testEquivalence() { multiMatchQuery("marvel hero captain america", "full_name", "first_name", "last_name", "category") : multiMatchQuery("marvel hero captain america", "*_name", randomBoolean() ? "category" : "categ*"); SearchResponse left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQueryBuilder .operator(Operator.OR).type(type))).get(); SearchResponse right = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(disMaxQuery(). add(matchQuery("full_name", "marvel hero captain america")) .add(matchQuery("first_name", "marvel hero captain america")) @@ -335,12 +344,12 @@ public void testEquivalence() { multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category") : multiMatchQuery("captain america", "*_name", randomBoolean() ? "category" : "categ*"); SearchResponse left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQueryBuilder .operator(op).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch).type(type))).get(); SearchResponse right = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(boolQuery().minimumShouldMatch(minShouldMatch) .should(randomBoolean() ? termQuery("full_name", "captain america") : matchQuery("full_name", "captain america").operator(op)) @@ -354,12 +363,12 @@ public void testEquivalence() { { String minShouldMatch = randomBoolean() ? null : "" + between(0, 1); SearchResponse left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQuery("capta", "full_name", "first_name", "last_name", "category") .type(MatchQuery.Type.PHRASE_PREFIX).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch))).get(); SearchResponse right = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(boolQuery().minimumShouldMatch(minShouldMatch) .should(matchPhrasePrefixQuery("full_name", "capta")) .should(matchPhrasePrefixQuery("first_name", "capta")) @@ -373,17 +382,17 @@ public void testEquivalence() { SearchResponse left; if (randomBoolean()) { left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category") .type(MatchQuery.Type.PHRASE).minimumShouldMatch(minShouldMatch))).get(); } else { left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category") .type(MatchQuery.Type.PHRASE).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch))).get(); } SearchResponse right = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(boolQuery().minimumShouldMatch(minShouldMatch) .should(matchPhraseQuery("full_name", "captain america")) .should(matchPhraseQuery("first_name", "captain america")) diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 53948fdca7e42..fe24db5161d1c 100644 --- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.query.WrapperQueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.index.search.MatchQuery; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.TermsLookup; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; @@ -1723,18 +1724,28 @@ public void testFieldAliasesForMetaFields() throws Exception { .setRouting("custom") .setSource("field", "value"); indexRandom(true, false, indexRequest); + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true)) + .get(); + try { + SearchResponse searchResponse = client().prepareSearch() + .setQuery(termQuery("routing-alias", "custom")) + .addDocValueField("id-alias") + .get(); + assertHitCount(searchResponse, 1L); + + SearchHit hit = searchResponse.getHits().getAt(0); + assertEquals(2, hit.getFields().size()); + assertTrue(hit.getFields().containsKey("id-alias")); + + DocumentField field = hit.getFields().get("id-alias"); + assertThat(field.getValue().toString(), equalTo("1")); + } finally { + // unset cluster setting + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())) + .get(); + } - SearchResponse searchResponse = client().prepareSearch() - .setQuery(termQuery("routing-alias", "custom")) - .addDocValueField("id-alias") - .get(); - assertHitCount(searchResponse, 1L); - - SearchHit hit = searchResponse.getHits().getAt(0); - assertEquals(2, hit.getFields().size()); - assertTrue(hit.getFields().containsKey("id-alias")); - - DocumentField field = hit.getFields().get("id-alias"); - assertThat(field.getValue().toString(), equalTo("1")); } } diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java index d7a4a02ed2bb2..fb4e3c8e828e6 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.MockScriptPlugin; @@ -1374,31 +1375,42 @@ public void testSortOnRareField() throws IOException { } public void testSortMetaField() throws Exception { - createIndex("test"); - ensureGreen(); - final int numDocs = randomIntBetween(10, 20); - IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; - for (int i = 0; i < numDocs; ++i) { - indexReqs[i] = client().prepareIndex("test").setId(Integer.toString(i)) + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true)) + .get(); + try { + createIndex("test"); + ensureGreen(); + final int numDocs = randomIntBetween(10, 20); + IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; + for (int i = 0; i < numDocs; ++i) { + indexReqs[i] = client().prepareIndex("test").setId(Integer.toString(i)) .setSource(); - } - indexRandom(true, indexReqs); + } + indexRandom(true, indexReqs); - SortOrder order = randomFrom(SortOrder.values()); - SearchResponse searchResponse = client().prepareSearch() + SortOrder order = randomFrom(SortOrder.values()); + SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .setSize(randomIntBetween(1, numDocs + 5)) .addSort("_id", order) .get(); - assertNoFailures(searchResponse); - SearchHit[] hits = searchResponse.getHits().getHits(); - BytesRef previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; - for (int i = 0; i < hits.length; ++i) { - String idString = hits[i].getId(); - final BytesRef id = new BytesRef(idString); - assertEquals(idString, hits[i].getSortValues()[0]); - assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id)); - previous = id; + assertNoFailures(searchResponse); + SearchHit[] hits = searchResponse.getHits().getHits(); + BytesRef previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; + for (int i = 0; i < hits.length; ++i) { + String idString = hits[i].getId(); + final BytesRef id = new BytesRef(idString); + assertEquals(idString, hits[i].getSortValues()[0]); + assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id)); + previous = id; + } + // assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE); + } finally { + // unset cluster setting + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())) + .get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 05421edc58f08..7711c4c5bd365 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -66,6 +66,6 @@ public static MapperService newMapperService(NamedXContentRegistry xContentRegis xContentRegistry, similarityService, mapperRegistry, - () -> null); + () -> null, () -> false); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 144efde14ba38..5adc5f86699f9 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -65,7 +65,7 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind SimilarityService similarityService = new SimilarityService(indexSettings, null, emptyMap()); MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry(); mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, - () -> null); + () -> null, () -> false); } private DocumentMapperForType docMapper(String type) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index e726b4dc15094..07420ce2d3f7b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -356,7 +356,7 @@ private static class ServiceHolder implements Closeable { similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, - () -> createShardContext(null)); + () -> createShardContext(null), () -> false); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 8214f74b40227..953aeed157cbf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -76,7 +76,7 @@ protected String configRoles() { " - names: '*'\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field1 ]\n" + + " grant: [ field1, id ]\n" + " query: '{\"term\" : {\"field1\" : \"value1\"}}'\n" + "role3:\n" + " cluster: [ all ]\n" + @@ -84,7 +84,7 @@ protected String configRoles() { " - names: '*'\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field2 ]\n" + + " grant: [ field2, id ]\n" + " query: '{\"term\" : {\"field2\" : \"value2\"}}'\n" + "role4:\n" + " cluster: [ all ]\n" + @@ -92,7 +92,7 @@ protected String configRoles() { " - names: '*'\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field1 ]\n" + + " grant: [ field1, id ]\n" + " query: '{\"term\" : {\"field2\" : \"value2\"}}'\n"; } @@ -106,12 +106,12 @@ public Settings nodeSettings(int nodeOrdinal) { public void testSimpleQuery() { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type1", "field1", "type=text", "field2", "type=text") + .addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text") ); - client().prepareIndex("test").setId("1").setSource("field1", "value1") + client().prepareIndex("test").setId("1").setSource("id", "1", "field1", "value1") .setRefreshPolicy(IMMEDIATE) .get(); - client().prepareIndex("test").setId("2").setSource("field2", "value2") + client().prepareIndex("test").setId("2").setSource("id", "2", "field2", "value2") .setRefreshPolicy(IMMEDIATE) .get(); @@ -121,20 +121,22 @@ public void testSimpleQuery() { .get(); assertHitCount(response, 1); assertSearchHits(response, "1"); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1").toString(), equalTo("value1")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id").toString(), equalTo("1")); response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))) .prepareSearch("test") .get(); assertHitCount(response, 1); assertSearchHits(response, "2"); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field2").toString(), equalTo("value2")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id").toString(), equalTo("2")); response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertSearchHits(response, "1", "2"); @@ -172,12 +174,12 @@ public void testDLSIsAppliedBeforeFLS() { public void testQueryCache() { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)) - .addMapping("type1", "field1", "type=text", "field2", "type=text") + .addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text") ); - client().prepareIndex("test").setId("1").setSource("field1", "value1") + client().prepareIndex("test").setId("1").setSource("id", "1", "field1", "value1") .setRefreshPolicy(IMMEDIATE) .get(); - client().prepareIndex("test").setId("2").setSource("field2", "value2") + client().prepareIndex("test").setId("2").setSource("id", "2", "field2", "value2") .setRefreshPolicy(IMMEDIATE) .get(); @@ -190,15 +192,17 @@ public void testQueryCache() { .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), equalTo("value1")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("1")); response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))) .prepareSearch("test") .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("2")); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field2"), equalTo("value2")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("2")); // this is a bit weird the document level permission (all docs with field2:value2) don't match with the field level // permissions (field1), @@ -208,21 +212,24 @@ public void testQueryCache() { .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("2")); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(0)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("2")); // user4 has all roles response = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), equalTo("value1")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("1")); assertThat(response.getHits().getAt(1).getId(), equalTo("2")); - assertThat(response.getHits().getAt(1).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(1).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(1).getSourceAsMap().get("field2"), equalTo("value2")); + assertThat(response.getHits().getAt(1).getSourceAsMap().get("id"), equalTo("2")); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index f23d84b9ad8a1..f4b2556fb4816 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -602,6 +602,9 @@ public void testGlobalAggregation() throws Exception { public void testParentChild() throws Exception { XContentBuilder mapping = jsonBuilder().startObject() .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("join_field") .field("type", "join") .startObject("relations") @@ -628,15 +631,18 @@ public void testParentChild() throws Exception { Map source = new HashMap<>(); source.put("field2", "value2"); + source.put("id", "c1"); Map joinField = new HashMap<>(); joinField.put("name", "child"); joinField.put("parent", "p1"); source.put("join_field", joinField); client().prepareIndex("test").setId("c1").setSource(source).setRouting("p1").get(); + source.put("id", "c2"); client().prepareIndex("test").setId("c2").setSource(source).setRouting("p1").get(); source = new HashMap<>(); source.put("field3", "value3"); source.put("join_field", joinField); + source.put("id", "c3"); client().prepareIndex("test").setId("c3").setSource(source).setRouting("p1").get(); refresh(); verifyParentChild(); @@ -651,7 +657,7 @@ private void verifyParentChild() { searchResponse = client().prepareSearch("test") .setQuery(hasParentQuery("parent", matchAllQuery(), false)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(searchResponse, 3L); assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("c1")); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java index 82ebe7c7d7489..a802c97a4647f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java @@ -64,6 +64,7 @@ protected String configRoles() { allowedFields = new HashSet<>(); disAllowedFields = new HashSet<>(); int numFields = scaledRandomIntBetween(5, 50); + allowedFields.add("id"); for (int i = 0; i < numFields; i++) { String field = "field" + i; if (i % 2 == 0) { @@ -100,21 +101,21 @@ protected String configRoles() { " privileges:\n" + " - all\n" + " field_security:\n" + - " grant: [ field1 ]\n" + + " grant: [ id, field1 ]\n" + "role4:\n" + " cluster: [ all ]\n" + " indices:\n" + " - names: test\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field2 ]\n" + + " grant: [ id, field2 ]\n" + "role5:\n" + " cluster: [ all ]\n" + " indices:\n" + " - names: test\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field3 ]\n"; + " grant: [ id, field3 ]\n"; } @Override @@ -166,7 +167,7 @@ public void testRandom() throws Exception { public void testDuel() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type1", "field1", "type=text", "field2", "type=text", "field3", "type=text") + .addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text", "field3", "type=text") ); int numDocs = scaledRandomIntBetween(32, 128); @@ -174,14 +175,14 @@ public void testDuel() throws Exception { for (int i = 1; i <= numDocs; i++) { String field = randomFrom("field1", "field2", "field3"); String value = "value"; - requests.add(client().prepareIndex("test").setId(value).setSource(field, value)); + requests.add(client().prepareIndex("test").setId(value).setSource("id", Integer.toString(i), field, value)); } indexRandom(true, requests); SearchResponse actual = client() .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field1", "value")) .should(QueryBuilders.termQuery("field2", "value")) @@ -189,7 +190,7 @@ public void testDuel() throws Exception { ) .get(); SearchResponse expected = client().prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field1", "value")) ) @@ -202,7 +203,7 @@ public void testDuel() throws Exception { actual = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field1", "value")) .should(QueryBuilders.termQuery("field2", "value")) @@ -210,7 +211,7 @@ public void testDuel() throws Exception { ) .get(); expected = client().prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field2", "value")) ) @@ -223,7 +224,7 @@ public void testDuel() throws Exception { actual = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field1", "value")) .should(QueryBuilders.termQuery("field2", "value")) @@ -231,7 +232,7 @@ public void testDuel() throws Exception { ) .get(); expected = client().prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field3", "value")) )