diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties
new file mode 100644
index 0000000000000..a0713ce128e6f
--- /dev/null
+++ b/.ci/java-versions.properties
@@ -0,0 +1,8 @@
+# This file is used with all of the non-matrix tests in Jenkins.
+
+# This .properties file defines the versions of Java with which to
+# build and test Elasticsearch for this branch. Valid Java versions
+# are 'java' or 'openjdk' followed by the major release number.
+
+ES_BUILD_JAVA=java10
+ES_RUNTIME_JAVA=java8
diff --git a/.ci/matrix-build-javas.yml b/.ci/matrix-build-javas.yml
new file mode 100644
index 0000000000000..17aa4b0bf222a
--- /dev/null
+++ b/.ci/matrix-build-javas.yml
@@ -0,0 +1,9 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are included as an axis of the matrix.
+
+# This axis of the build matrix represents the versions of Java with
+# which Elasticsearch will be built. Valid Java versions are 'java'
+# or 'openjdk' followed by the major release number.
+
+ES_BUILD_JAVA:
+ - java10
diff --git a/.ci/matrix-java-exclusions.yml b/.ci/matrix-java-exclusions.yml
new file mode 100644
index 0000000000000..e2adf9f0955db
--- /dev/null
+++ b/.ci/matrix-java-exclusions.yml
@@ -0,0 +1,14 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are excluded from the test matrix.
+
+# The yaml mapping below represents a single intersection on the build
+# matrix where a test *should not* be run. The value of the exclude
+# key is a list of maps.
+
+# In this example all of the combinations defined in the matrix will
+# run except for the test that builds with java10 and runs with java8.
+# exclude:
+# - ES_BUILD_JAVA: java10
+# ES_RUNTIME_JAVA: java8
+
+exclude:
diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml
new file mode 100644
index 0000000000000..72282ca805afd
--- /dev/null
+++ b/.ci/matrix-runtime-javas.yml
@@ -0,0 +1,10 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are included as an axis of the matrix.
+
+# This axis of the build matrix represents the versions of Java on
+# which Elasticsearch will be tested. Valid Java versions are 'java'
+# or 'openjdk' followed by the major release number.
+
+ES_RUNTIME_JAVA:
+ - java8
+ - java10
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 2617baadba013..69e90473a7f61 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -92,11 +92,11 @@ Contributing to the Elasticsearch codebase
**Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
-JDK 9 is required to build Elasticsearch. You must have a JDK 9 installation
+JDK 10 is required to build Elasticsearch. You must have a JDK 10 installation
with the environment variable `JAVA_HOME` referencing the path to Java home for
-your JDK 9 installation. By default, tests use the same runtime as `JAVA_HOME`.
+your JDK 10 installation. By default, tests use the same runtime as `JAVA_HOME`.
However, since Elasticsearch, supports JDK 8 the build supports compiling with
-JDK 9 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
+JDK 10 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
pointing to the Java home of a JDK 8 installation. Note that this mechanism can
be used to test against other JDKs as well, this is not only limited to JDK 8.
diff --git a/README.textile b/README.textile
index c964e31655dc8..ce7b3b7d34476 100644
--- a/README.textile
+++ b/README.textile
@@ -27,7 +27,6 @@ Elasticsearch is a distributed RESTful search engine built for the cloud. Featur
** All the power of Lucene easily exposed through simple configuration / plugins.
* Per operation consistency
** Single document level operations are atomic, consistent, isolated and durable.
-* Open Source under the Apache License, version 2 ("ALv2")
h2. Getting Started
@@ -217,23 +216,3 @@ Elasticsearch (1.x), it is required to perform a full cluster restart. Please
see the "setup reference":
https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html
for more details on the upgrade process.
-
-h1. License
-
-
-This software is licensed under the Apache License, version 2 ("ALv2"), quoted below.
-
-Copyright 2009-2016 Elasticsearch
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not
-use this file except in compliance with the License. You may obtain a copy of
-the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations under
-the License.
-
diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index f7725bd17d78d..97902d56ec8c7 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -296,7 +296,6 @@ e.g. -Dtests.rest.suite=index,get,create/10_with_id
* `tests.rest.blacklist`: comma separated globs that identify tests that are
blacklisted and need to be skipped
e.g. -Dtests.rest.blacklist=index/*/Index document,get/10_basic/*
-* `tests.rest.spec`: REST spec path (default /rest-api-spec/api)
Note that the REST tests, like all the integration tests, can be run against an external
cluster by specifying the `tests.cluster` property, which if present needs to contain a
@@ -414,16 +413,16 @@ and in another window:
----------------------------------------------------
vagrant up centos-7 --provider virtualbox && vagrant ssh centos-7
-cd $BATS_ARCHIVES
+cd $PACKAGING_ARCHIVES
sudo -E bats $BATS_TESTS/*rpm*.bats
----------------------------------------------------
If you wanted to retest all the release artifacts on a single VM you could:
-------------------------------------------------
-./gradlew setupBats
+./gradlew setupPackagingTest
cd qa/vagrant; vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
-cd $BATS_ARCHIVES
+cd $PACKAGING_ARCHIVES
sudo -E bats $BATS_TESTS/*.bats
-------------------------------------------------
@@ -477,12 +476,12 @@ branch. Finally, on a release branch, it will test against the most recent relea
=== BWC Testing against a specific remote/branch
Sometimes a backward compatibility change spans two versions. A common case is a new functionality
-that needs a BWC bridge in and an unreleased versioned of a release branch (for example, 5.x).
+that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
To test the changes, you can instruct Gradle to build the BWC version from a another remote/branch combination instead of
-pulling the release branch from GitHub. You do so using the `tests.bwc.remote` and `tests.bwc.refspec` system properties:
+pulling the release branch from GitHub. You do so using the `tests.bwc.remote` and `tests.bwc.refspec.BRANCH` system properties:
-------------------------------------------------
-./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec=index_req_bwc_5.x
+./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x
-------------------------------------------------
The branch needs to be available on the remote that the BWC makes of the
@@ -497,34 +496,23 @@ will need to:
will contain your change.
. Create a branch called `index_req_bwc_5.x` off `5.x`. This will contain your bwc layer.
. Push both branches to your remote repository.
-. Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec=index_req_bwc_5.x`.
+. Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x`.
-== Coverage analysis
+== Test coverage analysis
-Tests can be run instrumented with jacoco to produce a coverage report in
-`target/site/jacoco/`.
+Generating test coverage reports for Elasticsearch is currently not possible through Gradle.
+However, it _is_ possible to gain insight in code coverage using IntelliJ's built-in coverage
+analysis tool that can measure coverage upon executing specific tests. Eclipse may also be able
+to do the same using the EclEmma plugin.
-Unit test coverage:
-
----------------------------------------------------------------------------
-mvn -Dtests.coverage test jacoco:report
----------------------------------------------------------------------------
-
-Integration test coverage:
-
----------------------------------------------------------------------------
-mvn -Dtests.coverage -Dskip.unit.tests verify jacoco:report
----------------------------------------------------------------------------
-
-Combined (Unit+Integration) coverage:
-
----------------------------------------------------------------------------
-mvn -Dtests.coverage verify jacoco:report
----------------------------------------------------------------------------
+Test coverage reporting used to be possible with JaCoCo when Elasticsearch was using Maven
+as its build system. Since the switch to Gradle though, this is no longer possible, seeing as
+the code currently used to build Elasticsearch does not allow JaCoCo to recognize its tests.
+For more information on this, see the discussion in https://github.com/elastic/elasticsearch/issues/28867[issue #28867].
== Launching and debugging from an IDE
-If you want to run elasticsearch from your IDE, the `./gradlew run` task
+If you want to run Elasticsearch from your IDE, the `./gradlew run` task
supports a remote debugging option:
---------------------------------------------------------------------------
diff --git a/Vagrantfile b/Vagrantfile
index 683a5d83732bd..6f81ba0273c9f 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -334,9 +334,9 @@ export TAR=/elasticsearch/distribution/tar/build/distributions
export RPM=/elasticsearch/distribution/rpm/build/distributions
export DEB=/elasticsearch/distribution/deb/build/distributions
export BATS=/project/build/bats
-export BATS_UTILS=/project/build/bats/utils
-export BATS_TESTS=/project/build/bats/tests
-export BATS_ARCHIVES=/project/build/bats/archives
+export BATS_UTILS=/project/build/packaging/bats/utils
+export BATS_TESTS=/project/build/packaging/bats/tests
+export PACKAGING_ARCHIVES=/project/build/packaging/archives
VARS
cat \<\ /etc/sudoers.d/elasticsearch_vars
Defaults env_keep += "ZIP"
@@ -346,7 +346,7 @@ Defaults env_keep += "DEB"
Defaults env_keep += "BATS"
Defaults env_keep += "BATS_UTILS"
Defaults env_keep += "BATS_TESTS"
-Defaults env_keep += "BATS_ARCHIVES"
+Defaults env_keep += "PACKAGING_ARCHIVES"
SUDOERS_VARS
chmod 0440 /etc/sudoers.d/elasticsearch_vars
SHELL
diff --git a/build.gradle b/build.gradle
index 94823e0ce5b1a..dce2adf5ee0bd 100644
--- a/build.gradle
+++ b/build.gradle
@@ -196,6 +196,7 @@ subprojects {
"org.elasticsearch:elasticsearch-cli:${version}": ':server:cli',
"org.elasticsearch:elasticsearch-core:${version}": ':libs:elasticsearch-core',
"org.elasticsearch:elasticsearch-nio:${version}": ':libs:elasticsearch-nio',
+ "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
"org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
"org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
"org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 6bc461e1b598c..5256968b6ca3e 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -94,7 +94,7 @@ dependencies {
compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
- compile 'de.thetaphi:forbiddenapis:2.4.1'
+ compile 'de.thetaphi:forbiddenapis:2.5'
compile 'org.apache.rat:apache-rat:0.11'
compile "org.elasticsearch:jna:4.5.1"
}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 5eb82c12616fc..50e1cd68523d5 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -58,7 +58,7 @@ import java.time.ZonedDateTime
class BuildPlugin implements Plugin {
static final JavaVersion minimumRuntimeVersion = JavaVersion.VERSION_1_8
- static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_9
+ static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_10
@Override
void apply(Project project) {
@@ -311,8 +311,8 @@ class BuildPlugin implements Plugin {
/** Adds repositories used by ES dependencies */
static void configureRepositories(Project project) {
RepositoryHandler repos = project.repositories
- if (System.getProperty("repos.mavenlocal") != null) {
- // with -Drepos.mavenlocal=true we can force checking the local .m2 repo which is
+ if (System.getProperty("repos.mavenLocal") != null) {
+ // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
// useful for development ie. bwc tests where we install stuff in the local repository
// such that we don't have to pass hardcoded files to gradle
repos.mavenLocal()
@@ -551,7 +551,7 @@ class BuildPlugin implements Plugin {
if (project.licenseFile == null || project.noticeFile == null) {
throw new GradleException("Must specify license and notice file for project ${project.path}")
}
- jarTask.into('META-INF') {
+ jarTask.metaInf {
from(project.licenseFile.parent) {
include project.licenseFile.name
}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index a18472521522e..80cb376077ed1 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -95,7 +95,7 @@ public class PluginBuildPlugin extends BuildPlugin {
// we "upgrade" these optional deps to provided for plugins, since they will run
// with a full elasticsearch server that includes optional deps
compileOnly "org.locationtech.spatial4j:spatial4j:${project.versions.spatial4j}"
- compileOnly "com.vividsolutions:jts:${project.versions.jts}"
+ compileOnly "org.locationtech.jts:jts-core:${project.versions.jts}"
compileOnly "org.apache.logging.log4j:log4j-api:${project.versions.log4j}"
compileOnly "org.apache.logging.log4j:log4j-core:${project.versions.log4j}"
compileOnly "org.elasticsearch:jna:${project.versions.jna}"
@@ -168,12 +168,10 @@ public class PluginBuildPlugin extends BuildPlugin {
Files.copy(jarFile.resolveSibling(sourcesFileName), jarFile.resolveSibling(clientSourcesFileName),
StandardCopyOption.REPLACE_EXISTING)
- if (project.compilerJavaVersion < JavaVersion.VERSION_1_10) {
- String javadocFileName = jarFile.fileName.toString().replace('.jar', '-javadoc.jar')
- String clientJavadocFileName = clientFileName.replace('.jar', '-javadoc.jar')
- Files.copy(jarFile.resolveSibling(javadocFileName), jarFile.resolveSibling(clientJavadocFileName),
- StandardCopyOption.REPLACE_EXISTING)
- }
+ String javadocFileName = jarFile.fileName.toString().replace('.jar', '-javadoc.jar')
+ String clientJavadocFileName = clientFileName.replace('.jar', '-javadoc.jar')
+ Files.copy(jarFile.resolveSibling(javadocFileName), jarFile.resolveSibling(clientJavadocFileName),
+ StandardCopyOption.REPLACE_EXISTING)
}
project.assemble.dependsOn(clientJar)
}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/FilePermissionsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/FilePermissionsTask.groovy
new file mode 100644
index 0000000000000..d8da9a4207bf7
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/FilePermissionsTask.groovy
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
+import org.gradle.api.file.FileCollection
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.SourceSet
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.util.PatternSet
+import org.gradle.api.tasks.util.PatternFilterable
+import org.apache.tools.ant.taskdefs.condition.Os
+
+import java.nio.file.Files
+import java.nio.file.attribute.PosixFilePermission
+import java.nio.file.attribute.PosixFileAttributeView
+
+import static java.nio.file.attribute.PosixFilePermission.OTHERS_EXECUTE
+import static java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE
+
+/**
+ * Checks source files for correct file permissions.
+ */
+public class FilePermissionsTask extends DefaultTask {
+
+ /** A pattern set of which files should be checked. */
+ private PatternFilterable filesFilter = new PatternSet()
+
+ @OutputFile
+ File outputMarker = new File(project.buildDir, 'markers/filePermissions')
+
+ FilePermissionsTask() {
+ onlyIf { !Os.isFamily(Os.FAMILY_WINDOWS) }
+ description = "Checks java source files for correct file permissions"
+ // we always include all source files, and exclude what should not be checked
+ filesFilter.include('**')
+ // exclude sh files that might have the executable bit set
+ filesFilter.exclude('**/*.sh')
+ }
+
+ /** Returns the files this task will check */
+ @InputFiles
+ FileCollection files() {
+ List collections = new ArrayList<>()
+ for (SourceSet sourceSet : project.sourceSets) {
+ collections.add(sourceSet.allSource.matching(filesFilter))
+ }
+ return project.files(collections.toArray())
+ }
+
+ @TaskAction
+ void checkInvalidPermissions() {
+ List failures = new ArrayList<>()
+ for (File f : files()) {
+ PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(f.toPath(), PosixFileAttributeView.class)
+ Set permissions = fileAttributeView.readAttributes().permissions()
+ if (permissions.contains(OTHERS_EXECUTE) || permissions.contains(OWNER_EXECUTE) ||
+ permissions.contains(GROUP_EXECUTE)) {
+ failures.add("Source file is executable: " + f)
+ }
+ }
+ if (failures.isEmpty() == false) {
+ throw new GradleException('Found invalid file permissions:\n' + failures.join('\n'))
+ }
+ outputMarker.setText('done', 'UTF-8')
+ }
+
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index 9e1cdad04fd6c..09f0ad01578c9 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -18,6 +18,7 @@
*/
package org.elasticsearch.gradle.precommit
+import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import org.gradle.api.Project
import org.gradle.api.Task
@@ -37,6 +38,7 @@ class PrecommitTasks {
configureNamingConventions(project),
project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
+ project.tasks.create('filepermissions', FilePermissionsTask.class),
project.tasks.create('jarHell', JarHellTask.class),
project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
@@ -82,17 +84,14 @@ class PrecommitTasks {
getClass().getResource('/forbidden/es-all-signatures.txt')]
suppressAnnotations = ['**.SuppressForbidden']
}
- Task mainForbidden = project.tasks.findByName('forbiddenApisMain')
- if (mainForbidden != null) {
- mainForbidden.configure {
- signaturesURLs += getClass().getResource('/forbidden/es-server-signatures.txt')
- }
- }
- Task testForbidden = project.tasks.findByName('forbiddenApisTest')
- if (testForbidden != null) {
- testForbidden.configure {
- signaturesURLs += getClass().getResource('/forbidden/es-test-signatures.txt')
- signaturesURLs += getClass().getResource('/forbidden/http-signatures.txt')
+ project.tasks.withType(CheckForbiddenApis) {
+ // we do not use the += operator to add signatures, as conventionMappings of Gradle do not work when it's configured using withType:
+ if (name.endsWith('Test')) {
+ signaturesURLs = project.forbiddenApis.signaturesURLs +
+ [ getClass().getResource('/forbidden/es-test-signatures.txt'), getClass().getResource('/forbidden/http-signatures.txt') ]
+ } else {
+ signaturesURLs = project.forbiddenApis.signaturesURLs +
+ [ getClass().getResource('/forbidden/es-server-signatures.txt') ]
}
}
Task forbiddenApis = project.tasks.findByName('forbiddenApis')
@@ -143,21 +142,15 @@ class PrecommitTasks {
]
toolVersion = 7.5
}
- for (String taskName : ['checkstyleMain', 'checkstyleJava9', 'checkstyleTest']) {
- Task task = project.tasks.findByName(taskName)
- if (task != null) {
- project.tasks['check'].dependsOn.remove(task)
- checkstyleTask.dependsOn(task)
- task.dependsOn(copyCheckstyleConf)
- task.inputs.file(checkstyleSuppressions)
- task.reports {
- html.enabled false
- }
- }
- }
- project.tasks.withType(Checkstyle) {
- dependsOn(copyCheckstyleConf)
+ project.tasks.withType(Checkstyle) { task ->
+ project.tasks[JavaBasePlugin.CHECK_TASK_NAME].dependsOn.remove(task)
+ checkstyleTask.dependsOn(task)
+ task.dependsOn(copyCheckstyleConf)
+ task.inputs.file(checkstyleSuppressions)
+ task.reports {
+ html.enabled false
+ }
}
return checkstyleTask
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 4d6b54fa3bbee..8e97ee352ead2 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -494,7 +494,7 @@ class ClusterFormationTasks {
* the short name requiring the path to already exist.
*/
final Object esPluginUtil = "${-> node.binPath().resolve('elasticsearch-plugin').toString()}"
- final Object[] args = [esPluginUtil, 'install', file]
+ final Object[] args = [esPluginUtil, 'install', '--batch', file]
return configureExecTask(name, project, setup, node, args)
}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
index e6e7fca62f97e..c6d0f1d0425d0 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
@@ -37,9 +37,6 @@ class VagrantPropertiesExtension {
@Input
Boolean inheritTests
- @Input
- Boolean inheritTestArchives
-
@Input
Boolean inheritTestUtils
@@ -60,10 +57,6 @@ class VagrantPropertiesExtension {
this.inheritTests = inheritTests
}
- void setInheritTestArchives(Boolean inheritTestArchives) {
- this.inheritTestArchives = inheritTestArchives
- }
-
void setInheritTestUtils(Boolean inheritTestUtils) {
this.inheritTestUtils = inheritTestUtils
}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
index fc15258215e65..d7d1c01e7dd00 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -1,6 +1,5 @@
package org.elasticsearch.gradle.vagrant
-import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.FileContentsTask
import org.elasticsearch.gradle.LoggedExec
@@ -43,8 +42,9 @@ class VagrantTestPlugin implements Plugin {
/** Packages onboarded for upgrade tests **/
static List UPGRADE_FROM_ARCHIVES = ['rpm', 'deb']
+ private static final PACKAGING_CONFIGURATION = 'packaging'
private static final BATS = 'bats'
- private static final String BATS_TEST_COMMAND ="cd \$BATS_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
+ private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && ./gradlew test integTest"
@Override
@@ -53,11 +53,11 @@ class VagrantTestPlugin implements Plugin {
// Creates the Vagrant extension for the project
project.extensions.create('esvagrant', VagrantPropertiesExtension, listVagrantBoxes(project))
- // Add required repositories for Bats tests
- configureBatsRepositories(project)
+ // Add required repositories for packaging tests
+ configurePackagingArchiveRepositories(project)
// Creates custom configurations for Bats testing files (and associated scripts and archives)
- createBatsConfiguration(project)
+ createPackagingConfiguration(project)
// Creates all the main Vagrant tasks
createVagrantTasks(project)
@@ -87,7 +87,7 @@ class VagrantTestPlugin implements Plugin {
}
}
- private static void configureBatsRepositories(Project project) {
+ private static void configurePackagingArchiveRepositories(Project project) {
RepositoryHandler repos = project.repositories
// Try maven central first, it'll have releases before 5.0.0
@@ -102,10 +102,10 @@ class VagrantTestPlugin implements Plugin {
}
}
- private static void createBatsConfiguration(Project project) {
- project.configurations.create(BATS)
+ private static void createPackagingConfiguration(Project project) {
+ project.configurations.create(PACKAGING_CONFIGURATION)
- String upgradeFromVersion = System.getProperty("tests.packaging.upgradeVersion");
+ String upgradeFromVersion = System.getProperty("tests.packaging.upgradeVersion")
if (upgradeFromVersion == null) {
String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0)
final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16)
@@ -120,12 +120,14 @@ class VagrantTestPlugin implements Plugin {
} else {
it = "packages:${it}"
}
- project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
+ project.dependencies.add(PACKAGING_CONFIGURATION,
+ project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
}
UPGRADE_FROM_ARCHIVES.each {
// The version of elasticsearch that we upgrade *from*
- project.dependencies.add(BATS, "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+ project.dependencies.add(PACKAGING_CONFIGURATION,
+ "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
}
project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
@@ -154,22 +156,28 @@ class VagrantTestPlugin implements Plugin {
}
private static void createPrepareVagrantTestEnvTask(Project project) {
- File batsDir = new File("${project.buildDir}/${BATS}")
+ File packagingDir = new File(project.buildDir, PACKAGING_CONFIGURATION)
- Task createBatsDirsTask = project.tasks.create('createBatsDirs')
- createBatsDirsTask.outputs.dir batsDir
- createBatsDirsTask.doLast {
- batsDir.mkdirs()
+ File archivesDir = new File(packagingDir, 'archives')
+ Copy copyPackagingArchives = project.tasks.create('copyPackagingArchives', Copy) {
+ into archivesDir
+ from project.configurations[PACKAGING_CONFIGURATION]
}
- Copy copyBatsArchives = project.tasks.create('copyBatsArchives', Copy) {
- dependsOn createBatsDirsTask
- into "${batsDir}/archives"
- from project.configurations[BATS]
+ Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
+ dependsOn copyPackagingArchives
+ file "${archivesDir}/version"
+ contents project.version
+ }
+
+ Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) {
+ dependsOn copyPackagingArchives
+ file "${archivesDir}/upgrade_from_version"
+ contents project.extensions.esvagrant.upgradeFromVersion
}
+ File batsDir = new File(packagingDir, BATS)
Copy copyBatsTests = project.tasks.create('copyBatsTests', Copy) {
- dependsOn createBatsDirsTask
into "${batsDir}/tests"
from {
"${project.extensions.esvagrant.batsDir}/tests"
@@ -177,7 +185,6 @@ class VagrantTestPlugin implements Plugin {
}
Copy copyBatsUtils = project.tasks.create('copyBatsUtils', Copy) {
- dependsOn createBatsDirsTask
into "${batsDir}/utils"
from {
"${project.extensions.esvagrant.batsDir}/utils"
@@ -185,42 +192,30 @@ class VagrantTestPlugin implements Plugin {
}
// Now we iterate over dependencies of the bats configuration. When a project dependency is found,
- // we bring back its own archives, test files or test utils.
+ // we bring back its test files or test utils.
project.afterEvaluate {
- project.configurations.bats.dependencies.findAll {it.targetConfiguration == BATS }.each { d ->
- if (d instanceof DefaultProjectDependency) {
- DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d
- Project externalBatsProject = externalBatsDependency.dependencyProject
- String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir
-
- if (project.extensions.esvagrant.inheritTests) {
- copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests"))
- }
- if (project.extensions.esvagrant.inheritTestArchives) {
- copyBatsArchives.from(externalBatsDependency.projectConfiguration.files)
- }
- if (project.extensions.esvagrant.inheritTestUtils) {
- copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils"))
+ project.configurations[PACKAGING_CONFIGURATION].dependencies
+ .findAll {it.targetConfiguration == PACKAGING_CONFIGURATION }
+ .each { d ->
+ if (d instanceof DefaultProjectDependency) {
+ DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d
+ Project externalBatsProject = externalBatsDependency.dependencyProject
+ String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir
+
+ if (project.extensions.esvagrant.inheritTests) {
+ copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests"))
+ }
+ if (project.extensions.esvagrant.inheritTestUtils) {
+ copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils"))
+ }
}
- }
}
}
- Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
- dependsOn createBatsDirsTask
- file "${batsDir}/archives/version"
- contents project.version
- }
-
- Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) {
- dependsOn createBatsDirsTask
- file "${batsDir}/archives/upgrade_from_version"
- contents project.extensions.esvagrant.upgradeFromVersion
- }
-
- Task vagrantSetUpTask = project.tasks.create('setupBats')
+ Task vagrantSetUpTask = project.tasks.create('setupPackagingTest')
vagrantSetUpTask.dependsOn 'vagrantCheckVersion'
- vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils, copyBatsArchives, createVersionFile, createUpgradeFromFile
+ vagrantSetUpTask.dependsOn copyPackagingArchives, createVersionFile, createUpgradeFromFile
+ vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils
}
private static void createPackagingTestTask(Project project) {
@@ -270,8 +265,8 @@ class VagrantTestPlugin implements Plugin {
assert project.tasks.virtualboxCheckVersion != null
Task virtualboxCheckVersion = project.tasks.virtualboxCheckVersion
- assert project.tasks.setupBats != null
- Task setupBats = project.tasks.setupBats
+ assert project.tasks.setupPackagingTest != null
+ Task setupPackagingTest = project.tasks.setupPackagingTest
assert project.tasks.packagingTest != null
Task packagingTest = project.tasks.packagingTest
@@ -308,7 +303,7 @@ class VagrantTestPlugin implements Plugin {
environmentVars vagrantEnvVars
dependsOn vagrantCheckVersion, virtualboxCheckVersion
}
- update.mustRunAfter(setupBats)
+ update.mustRunAfter(setupPackagingTest)
/*
* Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag
@@ -359,32 +354,39 @@ class VagrantTestPlugin implements Plugin {
}
vagrantSmokeTest.dependsOn(smoke)
- Task packaging = project.tasks.create("vagrant${boxTask}#packagingTest", BatsOverVagrantTask) {
+ Task batsPackagingTest = project.tasks.create("vagrant${boxTask}#batsPackagingTest", BatsOverVagrantTask) {
remoteCommand BATS_TEST_COMMAND
boxName box
environmentVars vagrantEnvVars
- dependsOn up, setupBats
+ dependsOn up, setupPackagingTest
finalizedBy halt
}
- TaskExecutionAdapter packagingReproListener = new TaskExecutionAdapter() {
- @Override
- void afterExecute(Task task, TaskState state) {
- final String gradlew = Os.isFamily(Os.FAMILY_WINDOWS) ? "gradlew" : "./gradlew"
- if (state.failure != null) {
- println "REPRODUCE WITH: ${gradlew} ${packaging.path} " +
- "-Dtests.seed=${project.testSeed} "
- }
- }
+ TaskExecutionAdapter batsPackagingReproListener = createReproListener(project, batsPackagingTest.path)
+ batsPackagingTest.doFirst {
+ project.gradle.addListener(batsPackagingReproListener)
+ }
+ batsPackagingTest.doLast {
+ project.gradle.removeListener(batsPackagingReproListener)
}
- packaging.doFirst {
- project.gradle.addListener(packagingReproListener)
+ if (project.extensions.esvagrant.boxes.contains(box)) {
+ packagingTest.dependsOn(batsPackagingTest)
+ }
+
+ // This task doesn't do anything yet. In the future it will execute a jar containing tests on the vm
+ Task groovyPackagingTest = project.tasks.create("vagrant${boxTask}#groovyPackagingTest")
+ groovyPackagingTest.dependsOn(up)
+ groovyPackagingTest.finalizedBy(halt)
+
+ TaskExecutionAdapter groovyPackagingReproListener = createReproListener(project, groovyPackagingTest.path)
+ groovyPackagingTest.doFirst {
+ project.gradle.addListener(groovyPackagingReproListener)
}
- packaging.doLast {
- project.gradle.removeListener(packagingReproListener)
+ groovyPackagingTest.doLast {
+ project.gradle.removeListener(groovyPackagingReproListener)
}
if (project.extensions.esvagrant.boxes.contains(box)) {
- packagingTest.dependsOn(packaging)
+ packagingTest.dependsOn(groovyPackagingTest)
}
Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {
@@ -395,15 +397,7 @@ class VagrantTestPlugin implements Plugin {
finalizedBy halt
args '--command', PLATFORM_TEST_COMMAND + " -Dtests.seed=${-> project.testSeed}"
}
- TaskExecutionAdapter platformReproListener = new TaskExecutionAdapter() {
- @Override
- void afterExecute(Task task, TaskState state) {
- if (state.failure != null) {
- println "REPRODUCE WITH: gradle ${platform.path} " +
- "-Dtests.seed=${project.testSeed} "
- }
- }
- }
+ TaskExecutionAdapter platformReproListener = createReproListener(project, platform.path)
platform.doFirst {
project.gradle.addListener(platformReproListener)
}
@@ -415,4 +409,16 @@ class VagrantTestPlugin implements Plugin {
}
}
}
+
+ private static TaskExecutionAdapter createReproListener(Project project, String reproTaskPath) {
+ return new TaskExecutionAdapter() {
+ @Override
+ void afterExecute(Task task, TaskState state) {
+ final String gradlew = Os.isFamily(Os.FAMILY_WINDOWS) ? "gradlew" : "./gradlew"
+ if (state.failure != null) {
+ println "REPRODUCE WITH: ${gradlew} ${reproTaskPath} -Dtests.seed=${project.testSeed} "
+ }
+ }
+ }
+ }
}
diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index b1ef76c9d6a0e..11f19f683e557 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -248,9 +248,7 @@
-
-
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index fabcadabd9f96..e064b2f223cb6 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,9 +1,9 @@
elasticsearch = 7.0.0-alpha1
-lucene = 7.2.1
+lucene = 7.3.0-snapshot-98a6b3d
# optional dependencies
-spatial4j = 0.6
-jts = 1.13
+spatial4j = 0.7
+jts = 1.15.0
jackson = 2.8.10
snakeyaml = 1.17
# when updating log4j, please update also docs/java-api/index.asciidoc
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
index 0b366aa99e188..f5b46a6a53192 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
@@ -34,15 +34,17 @@
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
-import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
-import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
@@ -261,6 +263,28 @@ public void flushAsync(FlushRequest flushRequest, ActionListener
listener, emptySet(), headers);
}
+ /**
+ * Force merge one or more indices using the Force Merge API
+ *
+ * See
+ * Force Merge API on elastic.co
+ */
+ public ForceMergeResponse forceMerge(ForceMergeRequest forceMergeRequest, Header... headers) throws IOException {
+ return restHighLevelClient.performRequestAndParseEntity(forceMergeRequest, Request::forceMerge, ForceMergeResponse::fromXContent,
+ emptySet(), headers);
+ }
+
+ /**
+ * Asynchronously force merge one or more indices using the Force Merge API
+ *
+ * See
+ * Force Merge API on elastic.co
+ */
+ public void forceMergeAsync(ForceMergeRequest forceMergeRequest, ActionListener listener, Header... headers) {
+ restHighLevelClient.performRequestAsyncAndParseEntity(forceMergeRequest, Request::forceMerge, ForceMergeResponse::fromXContent,
+ listener, emptySet(), headers);
+ }
+
/**
* Clears the cache of one or more indices using the Clear Cache API
*
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
old mode 100755
new mode 100644
index 66b34da777b6a..802b1492be092
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
@@ -37,6 +37,7 @@
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
@@ -233,6 +234,17 @@ static Request flush(FlushRequest flushRequest) {
return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null);
}
+ static Request forceMerge(ForceMergeRequest forceMergeRequest) {
+ String[] indices = forceMergeRequest.indices() == null ? Strings.EMPTY_ARRAY : forceMergeRequest.indices();
+ String endpoint = endpoint(indices, "_forcemerge");
+ Params parameters = Params.builder();
+ parameters.withIndicesOptions(forceMergeRequest.indicesOptions());
+ parameters.putParam("max_num_segments", Integer.toString(forceMergeRequest.maxNumSegments()));
+ parameters.putParam("only_expunge_deletes", Boolean.toString(forceMergeRequest.onlyExpungeDeletes()));
+ parameters.putParam("flush", Boolean.toString(forceMergeRequest.flush()));
+ return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null);
+ }
+
static Request clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest) {
String[] indices = clearIndicesCacheRequest.indices() == null ? Strings.EMPTY_ARRAY :clearIndicesCacheRequest.indices();
String endpoint = endpoint(indices, "_cache/clear");
@@ -531,7 +543,7 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) {
}
static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException {
- String endpoint = endpoint(rankEvalRequest.getIndices(), Strings.EMPTY_ARRAY, "_rank_eval");
+ String endpoint = endpoint(rankEvalRequest.indices(), Strings.EMPTY_ARRAY, "_rank_eval");
HttpEntity entity = createEntity(rankEvalRequest.getRankEvalSpec(), REQUEST_BODY_CONTENT_TYPE);
return new Request(HttpGet.METHOD_NAME, endpoint, Collections.emptyMap(), entity);
}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
old mode 100755
new mode 100644
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
new file mode 100644
index 0000000000000..7f59fcc831213
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkProcessor;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.get.MultiGetRequest;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.Matchers.both;
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class BulkProcessorIT extends ESRestHighLevelClientTestCase {
+
+ private static BulkProcessor.Builder initBulkProcessorBuilder(BulkProcessor.Listener listener) {
+ return BulkProcessor.builder(highLevelClient()::bulkAsync, listener);
+ }
+
+ public void testThatBulkProcessorCountIsCorrect() throws Exception {
+ final CountDownLatch latch = new CountDownLatch(1);
+ BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
+
+ int numDocs = randomIntBetween(10, 100);
+ try (BulkProcessor processor = initBulkProcessorBuilder(listener)
+ //let's make sure that the bulk action limit trips, one single execution will index all the documents
+ .setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs)
+ .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))
+ .build()) {
+
+ MultiGetRequest multiGetRequest = indexDocs(processor, numDocs);
+
+ latch.await();
+
+ assertThat(listener.beforeCounts.get(), equalTo(1));
+ assertThat(listener.afterCounts.get(), equalTo(1));
+ assertThat(listener.bulkFailures.size(), equalTo(0));
+ assertResponseItems(listener.bulkItems, numDocs);
+ assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), numDocs);
+ }
+ }
+
+ public void testBulkProcessorFlush() throws Exception {
+ final CountDownLatch latch = new CountDownLatch(1);
+ BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
+
+ int numDocs = randomIntBetween(10, 100);
+
+ try (BulkProcessor processor = initBulkProcessorBuilder(listener)
+ //let's make sure that this bulk won't be automatically flushed
+ .setConcurrentRequests(randomIntBetween(0, 10)).setBulkActions(numDocs + randomIntBetween(1, 100))
+ .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
+
+ MultiGetRequest multiGetRequest = indexDocs(processor, numDocs);
+
+ assertThat(latch.await(randomInt(500), TimeUnit.MILLISECONDS), equalTo(false));
+ //we really need an explicit flush as none of the bulk thresholds was reached
+ processor.flush();
+ latch.await();
+
+ assertThat(listener.beforeCounts.get(), equalTo(1));
+ assertThat(listener.afterCounts.get(), equalTo(1));
+ assertThat(listener.bulkFailures.size(), equalTo(0));
+ assertResponseItems(listener.bulkItems, numDocs);
+ assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), numDocs);
+ }
+ }
+
+ public void testBulkProcessorConcurrentRequests() throws Exception {
+ int bulkActions = randomIntBetween(10, 100);
+ int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
+ int concurrentRequests = randomIntBetween(0, 7);
+
+ int expectedBulkActions = numDocs / bulkActions;
+
+ final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
+ int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
+ final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
+
+ BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
+
+ MultiGetRequest multiGetRequest;
+
+ try (BulkProcessor processor = initBulkProcessorBuilder(listener)
+ .setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions)
+ //set interval and size to high values
+ .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
+
+ multiGetRequest = indexDocs(processor, numDocs);
+
+ latch.await();
+
+ assertThat(listener.beforeCounts.get(), equalTo(expectedBulkActions));
+ assertThat(listener.afterCounts.get(), equalTo(expectedBulkActions));
+ assertThat(listener.bulkFailures.size(), equalTo(0));
+ assertThat(listener.bulkItems.size(), equalTo(numDocs - numDocs % bulkActions));
+ }
+
+ closeLatch.await();
+
+ assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
+ assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
+ assertThat(listener.bulkFailures.size(), equalTo(0));
+ assertThat(listener.bulkItems.size(), equalTo(numDocs));
+
+ Set ids = new HashSet<>();
+ for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
+ assertThat(bulkItemResponse.getFailureMessage(), bulkItemResponse.isFailed(), equalTo(false));
+ assertThat(bulkItemResponse.getIndex(), equalTo("test"));
+ assertThat(bulkItemResponse.getType(), equalTo("test"));
+ //with concurrent requests > 1 we can't rely on the order of the bulk requests
+ assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(numDocs)));
+ //we do want to check that we don't get duplicate ids back
+ assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
+ }
+
+ assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), numDocs);
+ }
+
+ public void testBulkProcessorWaitOnClose() throws Exception {
+ BulkProcessorTestListener listener = new BulkProcessorTestListener();
+
+ int numDocs = randomIntBetween(10, 100);
+ BulkProcessor processor = initBulkProcessorBuilder(listener)
+ //let's make sure that the bulk action limit trips, one single execution will index all the documents
+ .setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs)
+ .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(randomIntBetween(1, 10),
+ RandomPicks.randomFrom(random(), ByteSizeUnit.values())))
+ .build();
+
+ MultiGetRequest multiGetRequest = indexDocs(processor, numDocs);
+ assertThat(processor.awaitClose(1, TimeUnit.MINUTES), is(true));
+ if (randomBoolean()) { // check if we can call it multiple times
+ if (randomBoolean()) {
+ assertThat(processor.awaitClose(1, TimeUnit.MINUTES), is(true));
+ } else {
+ processor.close();
+ }
+ }
+
+ assertThat(listener.beforeCounts.get(), greaterThanOrEqualTo(1));
+ assertThat(listener.afterCounts.get(), greaterThanOrEqualTo(1));
+ for (Throwable bulkFailure : listener.bulkFailures) {
+ logger.error("bulk failure", bulkFailure);
+ }
+ assertThat(listener.bulkFailures.size(), equalTo(0));
+ assertResponseItems(listener.bulkItems, numDocs);
+ assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), numDocs);
+ }
+
+ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {
+
+ String createIndexBody = "{\n" +
+ " \"settings\" : {\n" +
+ " \"index\" : {\n" +
+ " \"blocks.write\" : true\n" +
+ " }\n" +
+ " }\n" +
+ " \n" +
+ "}";
+
+ NStringEntity entity = new NStringEntity(createIndexBody, ContentType.APPLICATION_JSON);
+ Response response = client().performRequest("PUT", "/test-ro", Collections.emptyMap(), entity);
+ assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+
+ int bulkActions = randomIntBetween(10, 100);
+ int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
+ int concurrentRequests = randomIntBetween(0, 10);
+
+ int expectedBulkActions = numDocs / bulkActions;
+
+ final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
+ int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
+ final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
+
+ int testDocs = 0;
+ int testReadOnlyDocs = 0;
+ MultiGetRequest multiGetRequest = new MultiGetRequest();
+ BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
+
+ try (BulkProcessor processor = initBulkProcessorBuilder(listener)
+ .setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions)
+ //set interval and size to high values
+ .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
+
+ for (int i = 1; i <= numDocs; i++) {
+ if (randomBoolean()) {
+ testDocs++;
+ processor.add(new IndexRequest("test", "test", Integer.toString(testDocs))
+ .source(XContentType.JSON, "field", "value"));
+ multiGetRequest.add("test", "test", Integer.toString(testDocs));
+ } else {
+ testReadOnlyDocs++;
+ processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs))
+ .source(XContentType.JSON, "field", "value"));
+ }
+ }
+ }
+
+ closeLatch.await();
+
+ assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
+ assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
+ assertThat(listener.bulkFailures.size(), equalTo(0));
+ assertThat(listener.bulkItems.size(), equalTo(testDocs + testReadOnlyDocs));
+
+ Set ids = new HashSet<>();
+ Set readOnlyIds = new HashSet<>();
+ for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
+ assertThat(bulkItemResponse.getIndex(), either(equalTo("test")).or(equalTo("test-ro")));
+ assertThat(bulkItemResponse.getType(), equalTo("test"));
+ if (bulkItemResponse.getIndex().equals("test")) {
+ assertThat(bulkItemResponse.isFailed(), equalTo(false));
+ //with concurrent requests > 1 we can't rely on the order of the bulk requests
+ assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testDocs)));
+ //we do want to check that we don't get duplicate ids back
+ assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
+ } else {
+ assertThat(bulkItemResponse.isFailed(), equalTo(true));
+ //with concurrent requests > 1 we can't rely on the order of the bulk requests
+ assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testReadOnlyDocs)));
+ //we do want to check that we don't get duplicate ids back
+ assertThat(readOnlyIds.add(bulkItemResponse.getId()), equalTo(true));
+ }
+ }
+
+ assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), testDocs);
+ }
+
+ private static MultiGetRequest indexDocs(BulkProcessor processor, int numDocs) throws Exception {
+ MultiGetRequest multiGetRequest = new MultiGetRequest();
+ for (int i = 1; i <= numDocs; i++) {
+ if (randomBoolean()) {
+ processor.add(new IndexRequest("test", "test", Integer.toString(i))
+ .source(XContentType.JSON, "field", randomRealisticUnicodeOfLengthBetween(1, 30)));
+ } else {
+ final String source = "{ \"index\":{\"_index\":\"test\",\"_type\":\"test\",\"_id\":\"" + Integer.toString(i) + "\"} }\n"
+ + Strings.toString(JsonXContent.contentBuilder()
+ .startObject().field("field", randomRealisticUnicodeOfLengthBetween(1, 30)).endObject()) + "\n";
+ processor.add(new BytesArray(source), null, null, XContentType.JSON);
+ }
+ multiGetRequest.add("test", "test", Integer.toString(i));
+ }
+ return multiGetRequest;
+ }
+
+ private static void assertResponseItems(List bulkItemResponses, int numDocs) {
+ assertThat(bulkItemResponses.size(), is(numDocs));
+ int i = 1;
+ for (BulkItemResponse bulkItemResponse : bulkItemResponses) {
+ assertThat(bulkItemResponse.getIndex(), equalTo("test"));
+ assertThat(bulkItemResponse.getType(), equalTo("test"));
+ assertThat(bulkItemResponse.getId(), equalTo(Integer.toString(i++)));
+ assertThat("item " + i + " failed with cause: " + bulkItemResponse.getFailureMessage(),
+ bulkItemResponse.isFailed(), equalTo(false));
+ }
+ }
+
+ private static void assertMultiGetResponse(MultiGetResponse multiGetResponse, int numDocs) {
+ assertThat(multiGetResponse.getResponses().length, equalTo(numDocs));
+ int i = 1;
+ for (MultiGetItemResponse multiGetItemResponse : multiGetResponse) {
+ assertThat(multiGetItemResponse.getIndex(), equalTo("test"));
+ assertThat(multiGetItemResponse.getType(), equalTo("test"));
+ assertThat(multiGetItemResponse.getId(), equalTo(Integer.toString(i++)));
+ }
+ }
+
+ private static class BulkProcessorTestListener implements BulkProcessor.Listener {
+
+ private final CountDownLatch[] latches;
+ private final AtomicInteger beforeCounts = new AtomicInteger();
+ private final AtomicInteger afterCounts = new AtomicInteger();
+ private final List bulkItems = new CopyOnWriteArrayList<>();
+ private final List bulkFailures = new CopyOnWriteArrayList<>();
+
+ private BulkProcessorTestListener(CountDownLatch... latches) {
+ this.latches = latches;
+ }
+
+ @Override
+ public void beforeBulk(long executionId, BulkRequest request) {
+ beforeCounts.incrementAndGet();
+ }
+
+ @Override
+ public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+ bulkItems.addAll(Arrays.asList(response.getItems()));
+ afterCounts.incrementAndGet();
+ for (CountDownLatch latch : latches) {
+ latch.countDown();
+ }
+ }
+
+ @Override
+ public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
+ bulkFailures.add(failure);
+ afterCounts.incrementAndGet();
+ for (CountDownLatch latch : latches) {
+ latch.countDown();
+ }
+ }
+ }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
old mode 100755
new mode 100644
index 8a2ba44791149..7a29a35d20ab1
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
@@ -38,6 +38,8 @@
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -467,6 +469,32 @@ public void testClearCache() throws IOException {
}
}
+ public void testForceMerge() throws IOException {
+ {
+ String index = "index";
+ Settings settings = Settings.builder()
+ .put("number_of_shards", 1)
+ .put("number_of_replicas", 0)
+ .build();
+ createIndex(index, settings);
+ ForceMergeRequest forceMergeRequest = new ForceMergeRequest(index);
+ ForceMergeResponse forceMergeResponse =
+ execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync);
+ assertThat(forceMergeResponse.getTotalShards(), equalTo(1));
+ assertThat(forceMergeResponse.getSuccessfulShards(), equalTo(1));
+ assertThat(forceMergeResponse.getFailedShards(), equalTo(0));
+ assertThat(forceMergeResponse.getShardFailures(), equalTo(BroadcastResponse.EMPTY));
+ }
+ {
+ String nonExistentIndex = "non_existent_index";
+ assertFalse(indexExists(nonExistentIndex));
+ ForceMergeRequest forceMergeRequest = new ForceMergeRequest(nonExistentIndex);
+ ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+ () -> execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync));
+ assertEquals(RestStatus.NOT_FOUND, exception.status());
+ }
+ }
+
public void testExistsAlias() throws IOException {
GetAliasesRequest getAliasesRequest = new GetAliasesRequest("alias");
assertFalse(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
old mode 100755
new mode 100644
index f79135c44f5ec..75ac543fbb4ce
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
@@ -40,6 +40,7 @@
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
@@ -621,6 +622,43 @@ public void testFlush() {
assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
}
+ public void testForceMerge() {
+ String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5);
+ ForceMergeRequest forceMergeRequest;
+ if (randomBoolean()) {
+ forceMergeRequest = new ForceMergeRequest(indices);
+ } else {
+ forceMergeRequest = new ForceMergeRequest();
+ forceMergeRequest.indices(indices);
+ }
+
+ Map expectedParams = new HashMap<>();
+ setRandomIndicesOptions(forceMergeRequest::indicesOptions, forceMergeRequest::indicesOptions, expectedParams);
+ if (randomBoolean()) {
+ forceMergeRequest.maxNumSegments(randomInt());
+ }
+ expectedParams.put("max_num_segments", Integer.toString(forceMergeRequest.maxNumSegments()));
+ if (randomBoolean()) {
+ forceMergeRequest.onlyExpungeDeletes(randomBoolean());
+ }
+ expectedParams.put("only_expunge_deletes", Boolean.toString(forceMergeRequest.onlyExpungeDeletes()));
+ if (randomBoolean()) {
+ forceMergeRequest.flush(randomBoolean());
+ }
+ expectedParams.put("flush", Boolean.toString(forceMergeRequest.flush()));
+
+ Request request = Request.forceMerge(forceMergeRequest);
+ StringJoiner endpoint = new StringJoiner("/", "/", "");
+ if (indices != null && indices.length > 0) {
+ endpoint.add(String.join(",", indices));
+ }
+ endpoint.add("_forcemerge");
+ assertThat(request.getEndpoint(), equalTo(endpoint.toString()));
+ assertThat(request.getParameters(), equalTo(expectedParams));
+ assertThat(request.getEntity(), nullValue());
+ assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
+ }
+
public void testClearCache() {
String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5);
ClearIndicesCacheRequest clearIndicesCacheRequest;
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
index 5adc7bee273a0..09a3fbd4d16a8 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
@@ -121,7 +121,7 @@ public void testIndex() throws Exception {
builder.startObject();
{
builder.field("user", "kimchy");
- builder.field("postDate", new Date());
+ builder.timeField("postDate", new Date());
builder.field("message", "trying out Elasticsearch");
}
builder.endObject();
@@ -331,7 +331,7 @@ public void testUpdate() throws Exception {
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
{
- builder.field("updated", new Date());
+ builder.timeField("updated", new Date());
builder.field("reason", "daily update");
}
builder.endObject();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
index fb9e56d222022..bc6946eb2dc7f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
@@ -37,6 +37,8 @@
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -771,6 +773,79 @@ public void onFailure(Exception e) {
}
}
+ public void testForceMergeIndex() throws Exception {
+ RestHighLevelClient client = highLevelClient();
+
+ {
+ createIndex("index", Settings.EMPTY);
+ }
+
+ {
+ // tag::force-merge-request
+ ForceMergeRequest request = new ForceMergeRequest("index1"); // <1>
+ ForceMergeRequest requestMultiple = new ForceMergeRequest("index1", "index2"); // <2>
+ ForceMergeRequest requestAll = new ForceMergeRequest(); // <3>
+ // end::force-merge-request
+
+ // tag::force-merge-request-indicesOptions
+ request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1>
+ // end::force-merge-request-indicesOptions
+
+ // tag::force-merge-request-segments-num
+ request.maxNumSegments(1); // <1>
+ // end::force-merge-request-segments-num
+
+ // tag::force-merge-request-only-expunge-deletes
+ request.onlyExpungeDeletes(true); // <1>
+ // end::force-merge-request-only-expunge-deletes
+
+ // tag::force-merge-request-flush
+ request.flush(true); // <1>
+ // end::force-merge-request-flush
+
+ // tag::force-merge-execute
+ ForceMergeResponse forceMergeResponse = client.indices().forceMerge(request);
+ // end::force-merge-execute
+
+ // tag::force-merge-response
+ int totalShards = forceMergeResponse.getTotalShards(); // <1>
+ int successfulShards = forceMergeResponse.getSuccessfulShards(); // <2>
+ int failedShards = forceMergeResponse.getFailedShards(); // <3>
+ DefaultShardOperationFailedException[] failures = forceMergeResponse.getShardFailures(); // <4>
+ // end::force-merge-response
+
+ // tag::force-merge-execute-listener
+ ActionListener listener = new ActionListener() {
+ @Override
+ public void onResponse(ForceMergeResponse forceMergeResponse) {
+ // <1>
+ }
+
+ @Override
+ public void onFailure(Exception e) {
+ // <2>
+ }
+ };
+ // end::force-merge-execute-listener
+
+ // tag::force-merge-execute-async
+ client.indices().forceMergeAsync(request, listener); // <1>
+ // end::force-merge-execute-async
+ }
+ {
+ // tag::force-merge-notfound
+ try {
+ ForceMergeRequest request = new ForceMergeRequest("does_not_exist");
+ client.indices().forceMerge(request);
+ } catch (ElasticsearchException exception) {
+ if (exception.status() == RestStatus.NOT_FOUND) {
+ // <1>
+ }
+ }
+ // end::force-merge-notfound
+ }
+ }
+
public void testClearCache() throws Exception {
RestHighLevelClient client = highLevelClient();
@@ -855,7 +930,6 @@ public void onFailure(Exception e) {
}
}
-
public void testCloseIndex() throws Exception {
RestHighLevelClient client = highLevelClient();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
index 22421dec6d9b9..96d962c3ac553 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
@@ -27,6 +27,8 @@
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.MultiSearchRequest;
+import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequest;
@@ -85,45 +87,15 @@
import static org.hamcrest.Matchers.greaterThan;
/**
- * This class is used to generate the Java High Level REST Client Search API documentation.
- *
- * You need to wrap your code between two tags like:
- * // tag::example
- * // end::example
- *
- * Where example is your tag name.
- *
- * Then in the documentation, you can extract what is between tag and end tags with
- * ["source","java",subs="attributes,callouts,macros"]
- * --------------------------------------------------
- * include-tagged::{doc-tests}/SearchDocumentationIT.java[example]
- * --------------------------------------------------
- *
- * The column width of the code block is 84. If the code contains a line longer
- * than 84, the line will be cut and a horizontal scroll bar will be displayed.
- * (the code indentation of the tag is not included in the width)
+ * Documentation for search APIs in the high level java client.
+ * Code wrapped in {@code tag} and {@code end} tags is included in the docs.
*/
public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
@SuppressWarnings({"unused", "unchecked"})
public void testSearch() throws Exception {
+ indexSearchTestData();
RestHighLevelClient client = highLevelClient();
- {
- BulkRequest request = new BulkRequest();
- request.add(new IndexRequest("posts", "doc", "1")
- .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
- Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
- request.add(new IndexRequest("posts", "doc", "2")
- .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
- Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
- request.add(new IndexRequest("posts", "doc", "3")
- .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
- Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));
- request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
- BulkResponse bulkResponse = client.bulk(request);
- assertSame(RestStatus.OK, bulkResponse.status());
- assertFalse(bulkResponse.hasFailures());
- }
{
// tag::search-request-basic
SearchRequest searchRequest = new SearchRequest(); // <1>
@@ -715,4 +687,90 @@ public void onFailure(Exception e) {
assertTrue(succeeded);
}
}
+
+ public void testMultiSearch() throws Exception {
+ indexSearchTestData();
+ RestHighLevelClient client = highLevelClient();
+ {
+ // tag::multi-search-request-basic
+ MultiSearchRequest request = new MultiSearchRequest(); // <1>
+ SearchRequest firstSearchRequest = new SearchRequest(); // <2>
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ searchSourceBuilder.query(QueryBuilders.matchQuery("user", "kimchy"));
+ firstSearchRequest.source(searchSourceBuilder);
+ request.add(firstSearchRequest); // <3>
+ SearchRequest secondSearchRequest = new SearchRequest(); // <4>
+ searchSourceBuilder = new SearchSourceBuilder();
+ searchSourceBuilder.query(QueryBuilders.matchQuery("user", "luca"));
+ secondSearchRequest.source(searchSourceBuilder);
+ request.add(secondSearchRequest);
+ // end::multi-search-request-basic
+ // tag::multi-search-execute
+ MultiSearchResponse response = client.multiSearch(request);
+ // end::multi-search-execute
+ // tag::multi-search-response
+ MultiSearchResponse.Item firstResponse = response.getResponses()[0]; // <1>
+ assertNull(firstResponse.getFailure()); // <2>
+ SearchResponse searchResponse = firstResponse.getResponse(); // <3>
+ assertEquals(3, searchResponse.getHits().getTotalHits());
+ MultiSearchResponse.Item secondResponse = response.getResponses()[1]; // <4>
+ assertNull(secondResponse.getFailure());
+ searchResponse = secondResponse.getResponse();
+ assertEquals(1, searchResponse.getHits().getTotalHits());
+ // end::multi-search-response
+
+ // tag::multi-search-execute-listener
+ ActionListener listener = new ActionListener() {
+ @Override
+ public void onResponse(MultiSearchResponse response) {
+ // <1>
+ }
+
+ @Override
+ public void onFailure(Exception e) {
+ // <2>
+ }
+ };
+ // end::multi-search-execute-listener
+
+ // Replace the empty listener by a blocking listener in test
+ final CountDownLatch latch = new CountDownLatch(1);
+ listener = new LatchedActionListener<>(listener, latch);
+
+ // tag::multi-search-execute-async
+ client.multiSearchAsync(request, listener); // <1>
+ // end::multi-search-execute-async
+
+ assertTrue(latch.await(30L, TimeUnit.SECONDS));
+ }
+ {
+ // tag::multi-search-request-index
+ MultiSearchRequest request = new MultiSearchRequest();
+ request.add(new SearchRequest("posts") // <1>
+ .types("doc")); // <2>
+ // end::multi-search-request-index
+ MultiSearchResponse response = client.multiSearch(request);
+ MultiSearchResponse.Item firstResponse = response.getResponses()[0];
+ assertNull(firstResponse.getFailure());
+ SearchResponse searchResponse = firstResponse.getResponse();
+ assertEquals(3, searchResponse.getHits().getTotalHits());
+ }
+ }
+
+ private void indexSearchTestData() throws IOException {
+ BulkRequest request = new BulkRequest();
+ request.add(new IndexRequest("posts", "doc", "1")
+ .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
+ Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
+ request.add(new IndexRequest("posts", "doc", "2")
+ .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
+ Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
+ request.add(new IndexRequest("posts", "doc", "3")
+ .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
+ Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));
+ request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+ BulkResponse bulkResponse = highLevelClient().bulk(request);
+ assertSame(RestStatus.OK, bulkResponse.status());
+ assertFalse(bulkResponse.hasFailures());
+ }
}
diff --git a/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java b/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java
index a7b222da70e1d..452e71b14d93a 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java
@@ -26,31 +26,50 @@
* when the host should be retried (based on number of previous failed attempts).
* Class is immutable, a new copy of it should be created each time the state has to be changed.
*/
-final class DeadHostState {
+final class DeadHostState implements Comparable {
private static final long MIN_CONNECTION_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(1);
private static final long MAX_CONNECTION_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(30);
- static final DeadHostState INITIAL_DEAD_STATE = new DeadHostState();
-
private final int failedAttempts;
private final long deadUntilNanos;
+ private final TimeSupplier timeSupplier;
- private DeadHostState() {
+ /**
+ * Build the initial dead state of a host. Useful when a working host stops functioning
+ * and needs to be marked dead after its first failure. In such case the host will be retried after a minute or so.
+ *
+ * @param timeSupplier a way to supply the current time and allow for unit testing
+ */
+ DeadHostState(TimeSupplier timeSupplier) {
this.failedAttempts = 1;
- this.deadUntilNanos = System.nanoTime() + MIN_CONNECTION_TIMEOUT_NANOS;
+ this.deadUntilNanos = timeSupplier.nanoTime() + MIN_CONNECTION_TIMEOUT_NANOS;
+ this.timeSupplier = timeSupplier;
}
/**
- * We keep track of how many times a certain node fails consecutively. The higher that number is the longer we will wait
- * to retry that same node again. Minimum is 1 minute (for a node the only failed once), maximum is 30 minutes (for a node
- * that failed many consecutive times).
+ * Build the dead state of a host given its previous dead state. Useful when a host has been failing before, hence
+ * it already failed for one or more consecutive times. The more failed attempts we register the longer we wait
+ * to retry that same host again. Minimum is 1 minute (for a node the only failed once created
+ * through {@link #DeadHostState(TimeSupplier)}), maximum is 30 minutes (for a node that failed more than 10 consecutive times)
+ *
+ * @param previousDeadHostState the previous state of the host which allows us to increase the wait till the next retry attempt
*/
- DeadHostState(DeadHostState previousDeadHostState) {
+ DeadHostState(DeadHostState previousDeadHostState, TimeSupplier timeSupplier) {
long timeoutNanos = (long)Math.min(MIN_CONNECTION_TIMEOUT_NANOS * 2 * Math.pow(2, previousDeadHostState.failedAttempts * 0.5 - 1),
MAX_CONNECTION_TIMEOUT_NANOS);
- this.deadUntilNanos = System.nanoTime() + timeoutNanos;
+ this.deadUntilNanos = timeSupplier.nanoTime() + timeoutNanos;
this.failedAttempts = previousDeadHostState.failedAttempts + 1;
+ this.timeSupplier = timeSupplier;
+ }
+
+ /**
+ * Indicates whether it's time to retry to failed host or not.
+ *
+ * @return true if the host should be retried, false otherwise
+ */
+ boolean shallBeRetried() {
+ return timeSupplier.nanoTime() - deadUntilNanos > 0;
}
/**
@@ -61,6 +80,15 @@ long getDeadUntilNanos() {
return deadUntilNanos;
}
+ int getFailedAttempts() {
+ return failedAttempts;
+ }
+
+ @Override
+ public int compareTo(DeadHostState other) {
+ return Long.compare(deadUntilNanos, other.deadUntilNanos);
+ }
+
@Override
public String toString() {
return "DeadHostState{" +
@@ -68,4 +96,19 @@ public String toString() {
", deadUntilNanos=" + deadUntilNanos +
'}';
}
+
+ /**
+ * Time supplier that makes timing aspects pluggable to ease testing
+ */
+ interface TimeSupplier {
+
+ TimeSupplier DEFAULT = new TimeSupplier() {
+ @Override
+ public long nanoTime() {
+ return System.nanoTime();
+ }
+ };
+
+ long nanoTime();
+ }
}
diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
index 4aa1a9d815cf4..48349c3858938 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
@@ -47,6 +47,7 @@
import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
+import javax.net.ssl.SSLHandshakeException;
import java.io.Closeable;
import java.io.IOException;
import java.net.SocketTimeoutException;
@@ -72,7 +73,6 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-import javax.net.ssl.SSLHandshakeException;
/**
* Client that connects to an Elasticsearch cluster through HTTP.
@@ -457,18 +457,18 @@ private HostTuple> nextHost() {
do {
Set filteredHosts = new HashSet<>(hostTuple.hosts);
for (Map.Entry entry : blacklist.entrySet()) {
- if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) {
+ if (entry.getValue().shallBeRetried() == false) {
filteredHosts.remove(entry.getKey());
}
}
if (filteredHosts.isEmpty()) {
- //last resort: if there are no good host to use, return a single dead one, the one that's closest to being retried
+ //last resort: if there are no good hosts to use, return a single dead one, the one that's closest to being retried
List> sortedHosts = new ArrayList<>(blacklist.entrySet());
if (sortedHosts.size() > 0) {
Collections.sort(sortedHosts, new Comparator>() {
@Override
public int compare(Map.Entry o1, Map.Entry o2) {
- return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos());
+ return o1.getValue().compareTo(o2.getValue());
}
});
HttpHost deadHost = sortedHosts.get(0).getKey();
@@ -499,14 +499,15 @@ private void onResponse(HttpHost host) {
* Called after each failed attempt.
* Receives as an argument the host that was used for the failed attempt.
*/
- private void onFailure(HttpHost host) throws IOException {
+ private void onFailure(HttpHost host) {
while(true) {
- DeadHostState previousDeadHostState = blacklist.putIfAbsent(host, DeadHostState.INITIAL_DEAD_STATE);
+ DeadHostState previousDeadHostState = blacklist.putIfAbsent(host, new DeadHostState(DeadHostState.TimeSupplier.DEFAULT));
if (previousDeadHostState == null) {
logger.debug("added host [" + host + "] to blacklist");
break;
}
- if (blacklist.replace(host, previousDeadHostState, new DeadHostState(previousDeadHostState))) {
+ if (blacklist.replace(host, previousDeadHostState,
+ new DeadHostState(previousDeadHostState, DeadHostState.TimeSupplier.DEFAULT))) {
logger.debug("updated host [" + host + "] already in blacklist");
break;
}
diff --git a/client/rest/src/test/java/org/elasticsearch/client/DeadHostStateTests.java b/client/rest/src/test/java/org/elasticsearch/client/DeadHostStateTests.java
new file mode 100644
index 0000000000000..75fbafd88f83c
--- /dev/null
+++ b/client/rest/src/test/java/org/elasticsearch/client/DeadHostStateTests.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
+
+public class DeadHostStateTests extends RestClientTestCase {
+
+ private static long[] EXPECTED_TIMEOUTS_SECONDS = new long[]{60, 84, 120, 169, 240, 339, 480, 678, 960, 1357, 1800};
+
+ public void testInitialDeadHostStateDefaultTimeSupplier() {
+ DeadHostState deadHostState = new DeadHostState(DeadHostState.TimeSupplier.DEFAULT);
+ long currentTime = System.nanoTime();
+ assertThat(deadHostState.getDeadUntilNanos(), greaterThan(currentTime));
+ assertThat(deadHostState.getFailedAttempts(), equalTo(1));
+ }
+
+ public void testDeadHostStateFromPreviousDefaultTimeSupplier() {
+ DeadHostState previous = new DeadHostState(DeadHostState.TimeSupplier.DEFAULT);
+ int iters = randomIntBetween(5, 30);
+ for (int i = 0; i < iters; i++) {
+ DeadHostState deadHostState = new DeadHostState(previous, DeadHostState.TimeSupplier.DEFAULT);
+ assertThat(deadHostState.getDeadUntilNanos(), greaterThan(previous.getDeadUntilNanos()));
+ assertThat(deadHostState.getFailedAttempts(), equalTo(previous.getFailedAttempts() + 1));
+ previous = deadHostState;
+ }
+ }
+
+ public void testCompareToDefaultTimeSupplier() {
+ int numObjects = randomIntBetween(EXPECTED_TIMEOUTS_SECONDS.length, 30);
+ DeadHostState[] deadHostStates = new DeadHostState[numObjects];
+ for (int i = 0; i < numObjects; i++) {
+ if (i == 0) {
+ deadHostStates[i] = new DeadHostState(DeadHostState.TimeSupplier.DEFAULT);
+ } else {
+ deadHostStates[i] = new DeadHostState(deadHostStates[i - 1], DeadHostState.TimeSupplier.DEFAULT);
+ }
+ }
+ for (int k = 1; k < deadHostStates.length; k++) {
+ assertThat(deadHostStates[k - 1].getDeadUntilNanos(), lessThan(deadHostStates[k].getDeadUntilNanos()));
+ assertThat(deadHostStates[k - 1], lessThan(deadHostStates[k]));
+ }
+ }
+
+ public void testShallBeRetried() {
+ ConfigurableTimeSupplier timeSupplier = new ConfigurableTimeSupplier();
+ DeadHostState deadHostState = null;
+ for (int i = 0; i < EXPECTED_TIMEOUTS_SECONDS.length; i++) {
+ long expectedTimeoutSecond = EXPECTED_TIMEOUTS_SECONDS[i];
+ timeSupplier.nanoTime = 0;
+ if (i == 0) {
+ deadHostState = new DeadHostState(timeSupplier);
+ } else {
+ deadHostState = new DeadHostState(deadHostState, timeSupplier);
+ }
+ for (int j = 0; j < expectedTimeoutSecond; j++) {
+ timeSupplier.nanoTime += TimeUnit.SECONDS.toNanos(1);
+ assertThat(deadHostState.shallBeRetried(), is(false));
+ }
+ int iters = randomIntBetween(5, 30);
+ for (int j = 0; j < iters; j++) {
+ timeSupplier.nanoTime += TimeUnit.SECONDS.toNanos(1);
+ assertThat(deadHostState.shallBeRetried(), is(true));
+ }
+ }
+ }
+
+ public void testDeadHostStateTimeouts() {
+ ConfigurableTimeSupplier zeroTimeSupplier = new ConfigurableTimeSupplier();
+ zeroTimeSupplier.nanoTime = 0L;
+ DeadHostState previous = new DeadHostState(zeroTimeSupplier);
+ for (long expectedTimeoutsSecond : EXPECTED_TIMEOUTS_SECONDS) {
+ assertThat(TimeUnit.NANOSECONDS.toSeconds(previous.getDeadUntilNanos()), equalTo(expectedTimeoutsSecond));
+ previous = new DeadHostState(previous, zeroTimeSupplier);
+ }
+ //check that from here on the timeout does not increase
+ int iters = randomIntBetween(5, 30);
+ for (int i = 0; i < iters; i++) {
+ DeadHostState deadHostState = new DeadHostState(previous, zeroTimeSupplier);
+ assertThat(TimeUnit.NANOSECONDS.toSeconds(deadHostState.getDeadUntilNanos()),
+ equalTo(EXPECTED_TIMEOUTS_SECONDS[EXPECTED_TIMEOUTS_SECONDS.length - 1]));
+ previous = deadHostState;
+ }
+ }
+
+ private static class ConfigurableTimeSupplier implements DeadHostState.TimeSupplier {
+
+ long nanoTime;
+
+ @Override
+ public long nanoTime() {
+ return nanoTime;
+ }
+ }
+}
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index caf9ce6be2e07..7786eefb97f01 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -101,7 +101,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
@Before
@SuppressWarnings("unchecked")
- public void createRestClient() throws IOException {
+ public void createRestClient() {
httpClient = mock(CloseableHttpAsyncClient.class);
when(httpClient.execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer>() {
@@ -160,17 +160,6 @@ public void shutdownExec() {
exec.shutdown();
}
- public void testNullPath() throws IOException {
- for (String method : getHttpMethods()) {
- try {
- restClient.performRequest(method, null);
- fail("path set to null should fail!");
- } catch (NullPointerException e) {
- assertEquals("path must not be null", e.getMessage());
- }
- }
- }
-
/**
* Verifies the content of the {@link HttpRequest} that's internally created and passed through to the http client
*/
@@ -196,33 +185,6 @@ public void testInternalHttpRequest() throws Exception {
}
}
- public void testSetHosts() throws IOException {
- try {
- restClient.setHosts((HttpHost[]) null);
- fail("setHosts should have failed");
- } catch (IllegalArgumentException e) {
- assertEquals("hosts must not be null nor empty", e.getMessage());
- }
- try {
- restClient.setHosts();
- fail("setHosts should have failed");
- } catch (IllegalArgumentException e) {
- assertEquals("hosts must not be null nor empty", e.getMessage());
- }
- try {
- restClient.setHosts((HttpHost) null);
- fail("setHosts should have failed");
- } catch (NullPointerException e) {
- assertEquals("host cannot be null", e.getMessage());
- }
- try {
- restClient.setHosts(new HttpHost("localhost", 9200), null, new HttpHost("localhost", 9201));
- fail("setHosts should have failed");
- } catch (NullPointerException e) {
- assertEquals("host cannot be null", e.getMessage());
- }
- }
-
/**
* End to end test for ok status codes
*/
@@ -289,7 +251,7 @@ public void testErrorStatusCodes() throws IOException {
}
}
- public void testIOExceptions() throws IOException {
+ public void testIOExceptions() {
for (String method : getHttpMethods()) {
//IOExceptions should be let bubble up
try {
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java
index 33323d39663e2..ee6dbf449bd56 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -28,6 +28,7 @@
import java.util.Collections;
import java.util.concurrent.CountDownLatch;
+import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
@@ -147,8 +148,48 @@ public void testBuildUriLeavesPathUntouched() {
}
}
+ public void testSetHostsWrongArguments() throws IOException {
+ try (RestClient restClient = createRestClient()) {
+ restClient.setHosts((HttpHost[]) null);
+ fail("setHosts should have failed");
+ } catch (IllegalArgumentException e) {
+ assertEquals("hosts must not be null nor empty", e.getMessage());
+ }
+ try (RestClient restClient = createRestClient()) {
+ restClient.setHosts();
+ fail("setHosts should have failed");
+ } catch (IllegalArgumentException e) {
+ assertEquals("hosts must not be null nor empty", e.getMessage());
+ }
+ try (RestClient restClient = createRestClient()) {
+ restClient.setHosts((HttpHost) null);
+ fail("setHosts should have failed");
+ } catch (NullPointerException e) {
+ assertEquals("host cannot be null", e.getMessage());
+ }
+ try (RestClient restClient = createRestClient()) {
+ restClient.setHosts(new HttpHost("localhost", 9200), null, new HttpHost("localhost", 9201));
+ fail("setHosts should have failed");
+ } catch (NullPointerException e) {
+ assertEquals("host cannot be null", e.getMessage());
+ }
+ }
+
+ public void testNullPath() throws IOException {
+ try (RestClient restClient = createRestClient()) {
+ for (String method : getHttpMethods()) {
+ try {
+ restClient.performRequest(method, null);
+ fail("path set to null should fail!");
+ } catch (NullPointerException e) {
+ assertEquals("path must not be null", e.getMessage());
+ }
+ }
+ }
+ }
+
private static RestClient createRestClient() {
HttpHost[] hosts = new HttpHost[]{new HttpHost("localhost", 9200)};
- return new RestClient(mock(CloseableHttpAsyncClient.class), randomLongBetween(1_000, 30_000), new Header[]{}, hosts, null, null);
+ return new RestClient(mock(CloseableHttpAsyncClient.class), randomIntBetween(1_000, 30_000), new Header[]{}, hosts, null, null);
}
}
diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle
index 840c69742a0c7..8d5aa204c487d 100644
--- a/distribution/bwc/build.gradle
+++ b/distribution/bwc/build.gradle
@@ -91,7 +91,7 @@ subprojects {
String buildMetadataKey = "bwc_refspec_${project.path.substring(1)}"
task checkoutBwcBranch(type: LoggedExec) {
- String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(buildMetadataKey, "${remote}/${bwcBranch}"))
+ String refspec = System.getProperty("tests.bwc.refspec.${bwcBranch}", buildMetadata.get(buildMetadataKey, "${remote}/${bwcBranch}"))
dependsOn fetchLatest
workingDir = checkoutDir
commandLine = ['git', 'checkout', refspec]
diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 84f3764880243..5a14d041c763b 100644
--- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -208,7 +208,7 @@ protected void printAdditionalHelp(Terminal terminal) {
@Override
protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
String pluginId = arguments.value(options);
- boolean isBatch = options.has(batchOption) || System.console() == null;
+ final boolean isBatch = options.has(batchOption);
execute(terminal, pluginId, isBatch, env);
}
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index 1c55e3b8a4e55..9f7fdc9ea2f17 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -1,7 +1,7 @@
:version: 7.0.0-alpha1
:major-version: 7.x
-:lucene_version: 7.2.1
-:lucene_version_path: 7_2_1
+:lucene_version: 7.3.0
+:lucene_version_path: 7_3_0
:branch: master
:jdk: 1.8.0_131
:jdk_major: 8
diff --git a/docs/community-clients/index.asciidoc b/docs/community-clients/index.asciidoc
index afa8b0f5879b7..0fd5c3a483f50 100644
--- a/docs/community-clients/index.asciidoc
+++ b/docs/community-clients/index.asciidoc
@@ -8,6 +8,7 @@ Besides the link:/guide[officially supported Elasticsearch clients], there are
a number of clients that have been contributed by the community for various languages:
* <>
+* <>
* <>
* <>
* <>
@@ -35,6 +36,10 @@ a number of clients that have been contributed by the community for various lang
* https://www.b4x.com/android/forum/threads/server-jelasticsearch-search-and-text-analytics.73335/
B4J client based on the official Java REST client.
+[[cpp]]
+== C++
+* https://github.com/seznam/elasticlient[elasticlient]: simple library for simplified work with Elasticsearch in C++
+
[[clojure]]
== Clojure
diff --git a/docs/java-api/query-dsl/geo-shape-query.asciidoc b/docs/java-api/query-dsl/geo-shape-query.asciidoc
index c8084c5ea9fd6..803f1849b5cdf 100644
--- a/docs/java-api/query-dsl/geo-shape-query.asciidoc
+++ b/docs/java-api/query-dsl/geo-shape-query.asciidoc
@@ -12,13 +12,13 @@ to your classpath in order to use this type:
org.locationtech.spatial4jspatial4j
- 0.6 <1>
+ 0.7 <1>
- com.vividsolutions
- jts
- 1.13 <2>
+ org.locationtech.jts
+ jts-core
+ 1.15.0 <2>
xerces
@@ -28,7 +28,7 @@ to your classpath in order to use this type:
-----------------------------------------------
<1> check for updates in http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.locationtech.spatial4j%22%20AND%20a%3A%22spatial4j%22[Maven Central]
-<2> check for updates in http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.vividsolutions%22%20AND%20a%3A%22jts%22[Maven Central]
+<2> check for updates in http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.locationtech.jts%22%20AND%20a%3A%22jts-core%22[Maven Central]
[source,java]
--------------------------------------------------
diff --git a/docs/java-rest/high-level/indices/force_merge.asciidoc b/docs/java-rest/high-level/indices/force_merge.asciidoc
new file mode 100644
index 0000000000000..6fe1fcd82b749
--- /dev/null
+++ b/docs/java-rest/high-level/indices/force_merge.asciidoc
@@ -0,0 +1,102 @@
+[[java-rest-high-force-merge]]
+=== Force Merge API
+
+[[java-rest-high-force-merge-request]]
+==== Force merge Request
+
+A `ForceMergeRequest` can be applied to one or more indices, or even on `_all` the indices:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-request]
+--------------------------------------------------
+<1> Force merge one index
+<2> Force merge multiple indices
+<3> Force merge all the indices
+
+==== Optional arguments
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-request-indicesOptions]
+--------------------------------------------------
+<1> Setting `IndicesOptions` controls how unavailable indices are resolved and
+how wildcard expressions are expanded
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-request-segments-num]
+--------------------------------------------------
+<1> Set `max_num_segments` to control the number of segments to merge down to.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-request-only-expunge-deletes]
+--------------------------------------------------
+<1> Set the `only_expunge_deletes` flag to `true`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-request-flush]
+--------------------------------------------------
+<1> Set the `flush` flag to `true`
+
+[[java-rest-high-force-merge-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-execute]
+--------------------------------------------------
+
+[[java-rest-high-force-merge-async]]
+==== Asynchronous Execution
+
+The asynchronous execution of a force merge request requires both the `ForceMergeRequest`
+instance and an `ActionListener` instance to be passed to the asynchronous
+method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-execute-async]
+--------------------------------------------------
+<1> The `ForceMergeRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `ForceMergeResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
+
+[[java-rest-high-force-merge-response]]
+==== Force Merge Response
+
+The returned `ForceMergeResponse` allows to retrieve information about the
+executed operation as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-response]
+--------------------------------------------------
+<1> Total number of shards hit by the force merge request
+<2> Number of shards where the force merge has succeeded
+<3> Number of shards where the force merge has failed
+<4> A list of failures if the operation failed on one or more shards
+
+By default, if the indices were not found, an `ElasticsearchException` will be thrown:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[force-merge-notfound]
+--------------------------------------------------
+<1> Do something if the indices to be force merged were not found
\ No newline at end of file
diff --git a/docs/java-rest/high-level/search/multi-search.asciidoc b/docs/java-rest/high-level/search/multi-search.asciidoc
new file mode 100644
index 0000000000000..1b76f8976666a
--- /dev/null
+++ b/docs/java-rest/high-level/search/multi-search.asciidoc
@@ -0,0 +1,90 @@
+[[java-rest-high-multi-search]]
+=== Multi-Search API
+
+The `multiSearch` API executes multiple <>
+requests in a single http request in parallel.
+
+[[java-rest-high-multi-search-request]]
+==== Multi-Search Request
+
+The `MultiSearchRequest` is built empty and you add all of the searches that
+you wish to execute to it:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[multi-search-request-basic]
+--------------------------------------------------
+<1> Create an empty `MultiSearchRequest`.
+<2> Create an empty `SearchRequest` and populate it just like you
+would for a regular <>.
+<3> Add the `SearchRequest` to the `MultiSearchRequest`.
+<4> Build a second `SearchRequest` and add it to the `MultiSearchRequest`.
+
+===== Optional arguments
+
+The `SearchRequest`s inside of `MultiSearchRequest` support all of
+<>'s optional arguments.
+For example:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-indices-types]
+--------------------------------------------------
+<1> Restricts the request to an index
+<2> Limits the request to a type
+
+[[java-rest-high-multi-search-sync]]
+==== Synchronous Execution
+
+The `multiSearch` method executes `MultiSearchRequest`s synchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[multi-search-execute]
+--------------------------------------------------
+
+[[java-rest-high-multi-search-async]]
+==== Asynchronous Execution
+
+The `multiSearchAsync` method executes `MultiSearchRequest`s asynchronously,
+calling the provided `ActionListener` when the response is ready.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[search-execute-async]
+--------------------------------------------------
+<1> The `MultiSearchRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `MultiSearchResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[multi-search-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed.
+<2> Called when the whole `SearchRequest` fails.
+
+==== MultiSearchResponse
+
+The `MultiSearchResponse` that is returned by executing the `multiSearch`
+a `MultiSearchResponse.Item` for each `SearchRequest` in the
+`MultiSearchRequest`. Each `MultiSearchResponse.Item` contains an
+exception in `getFailure` if the request failed or a
+<> in `getResponse` if
+the request succeeded:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SearchDocumentationIT.java[multi-search-response]
+--------------------------------------------------
+<1> The item for the first search.
+<2> It succeeded so `getFailure` returns null.
+<3> And there is a <> in
+`getResponse`.
+<4> The item for the second search.
diff --git a/docs/java-rest/high-level/search/search.asciidoc b/docs/java-rest/high-level/search/search.asciidoc
index 2e8dda64286f4..af81775a90072 100644
--- a/docs/java-rest/high-level/search/search.asciidoc
+++ b/docs/java-rest/high-level/search/search.asciidoc
@@ -20,6 +20,7 @@ include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-basic]
<3> Add a `match_all` query to the `SearchSourceBuilder`.
<4> Add the `SearchSourceBuilder` to the `SeachRequest`.
+[[java-rest-high-search-request-optional]]
===== Optional arguments
Let's first look at some of the optional arguments of a `SearchRequest`:
@@ -140,7 +141,7 @@ The `SearchSourceBuilder` allows to add one or more `SortBuilder` instances. The
include-tagged::{doc-tests}/SearchDocumentationIT.java[search-source-sorting]
--------------------------------------------------
<1> Sort descending by `_score` (the default)
-<2> Also sort ascending by `_id` field
+<2> Also sort ascending by `_id` field
===== Source filtering
@@ -268,6 +269,7 @@ include-tagged::{doc-tests}/SearchDocumentationIT.java[search-execute-listener]
<1> Called when the execution is successfully completed.
<2> Called when the whole `SearchRequest` fails.
+[[java-rest-high-search-response]]
==== SearchResponse
The `SearchResponse` that is returned by executing the search provides details
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index bea30690fe183..0330b1903c5bf 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -31,9 +31,11 @@ The Java High Level REST Client supports the following Search APIs:
* <>
* <>
* <>
+* <>
include::search/search.asciidoc[]
include::search/scroll.asciidoc[]
+include::search/multi-search.asciidoc[]
== Miscellaneous APIs
@@ -60,6 +62,7 @@ Index Management::
* <>
* <>
* <>
+* <>
* <>
Mapping Management::
@@ -79,6 +82,7 @@ include::indices/split_index.asciidoc[]
include::indices/refresh.asciidoc[]
include::indices/flush.asciidoc[]
include::indices/clear_cache.asciidoc[]
+include::indices/force_merge.asciidoc[]
include::indices/rollover.asciidoc[]
include::indices/put_mapping.asciidoc[]
include::indices/update_aliases.asciidoc[]
diff --git a/docs/java-rest/low-level/configuration.asciidoc b/docs/java-rest/low-level/configuration.asciidoc
index 54f7cd2817354..b0753496558bb 100644
--- a/docs/java-rest/low-level/configuration.asciidoc
+++ b/docs/java-rest/low-level/configuration.asciidoc
@@ -86,3 +86,16 @@ will be used.
For any other required configuration needed, the Apache HttpAsyncClient docs
should be consulted: https://hc.apache.org/httpcomponents-asyncclient-4.1.x/ .
+
+NOTE: If your application runs under the security manager you might be subject
+to the JVM default policies of caching positive hostname resolutions
+indefinitely and negative hostname resolutions for ten seconds. If the resolved
+addresses of the hosts to which you are connecting the client to vary with time
+then you might want to modify the default JVM behavior. These can be modified by
+adding
+http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.ttl=`]
+and
+http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.negative.ttl=`]
+to your
+http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java
+security policy].
diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc
index 162988fe3fc15..90f2c685fdaeb 100644
--- a/docs/plugins/integrations.asciidoc
+++ b/docs/plugins/integrations.asciidoc
@@ -82,6 +82,9 @@ releases 2.0 and later do not support rivers.
[float]
==== Supported by Elasticsearch:
+* https://github.com/elastic/ansible-elasticsearch[Ansible playbook for Elasticsearch]:
+ An officially supported ansible playbook for Elasticsearch. Tested with the latest version of 5.x and 6.x on Ubuntu 14.04/16.04, Debian 8, Centos 7.
+
* https://github.com/elastic/puppet-elasticsearch[Puppet]:
Elasticsearch puppet module.
diff --git a/docs/reference/aggregations.asciidoc b/docs/reference/aggregations.asciidoc
index f2fdd9a16de82..472b87b72fe67 100644
--- a/docs/reference/aggregations.asciidoc
+++ b/docs/reference/aggregations.asciidoc
@@ -40,6 +40,10 @@ NOTE: Bucketing aggregations can have sub-aggregations (bucketing or metric). Th
aggregations (one can nest an aggregation under a "parent" aggregation, which is itself a sub-aggregation of
another higher-level aggregation).
+NOTE: Aggregations operate on the `double` representation of
+ the data. As a consequence, the result may be approximate when running on longs
+ whose absolute value is greater than `2^53`.
+
[float]
== Structuring Aggregations
diff --git a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc
index 5d46a7f4c4a99..688cf20c5320f 100644
--- a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc
@@ -545,88 +545,3 @@ GET /_search
}
--------------------------------------------------
// TESTRESPONSE[s/\.\.\.//]
-
-==== Index sorting
-
-By default this aggregation runs on every document that match the query.
-Though if the index sort matches the composite sort this aggregation can optimize
-the execution and can skip documents that contain composite buckets that would not
-be part of the response.
-
-For instance the following aggregations:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
- "aggs" : {
- "my_buckets": {
- "composite" : {
- "size": 2,
- "sources" : [
- { "date": { "date_histogram": { "field": "timestamp", "interval": "1d", "order": "asc" } } },
- { "product": { "terms": { "field": "product", "order": "asc" } } }
- ]
- }
- }
- }
-}
---------------------------------------------------
-// CONSOLE
-
-\... is much faster on an index that uses the following sort:
-
-[source,js]
---------------------------------------------------
-PUT twitter
-{
- "settings" : {
- "index" : {
- "sort.field" : ["timestamp", "product"],
- "sort.order" : ["asc", "asc"]
- }
- },
- "mappings": {
- "sales": {
- "properties": {
- "timestamp": {
- "type": "date"
- },
- "product": {
- "type": "keyword"
- }
- }
- }
- }
-}
---------------------------------------------------
-// CONSOLE
-
-WARNING: The optimization takes effect only if the fields used for sorting are single-valued and follow
-the same order as the aggregation (`desc` or `asc`).
-
-If only the aggregation results are needed it is also better to set the size of the query to 0
-and `track_total_hits` to false in order to remove other slowing factors:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
- "size": 0,
- "track_total_hits": false,
- "aggs" : {
- "my_buckets": {
- "composite" : {
- "size": 2,
- "sources" : [
- { "date": { "date_histogram": { "field": "timestamp", "interval": "1d" } } },
- { "product": { "terms": { "field": "product" } } }
- ]
- }
- }
- }
-}
---------------------------------------------------
-// CONSOLE
-
-See <> for more details.
diff --git a/docs/reference/cat/thread_pool.asciidoc b/docs/reference/cat/thread_pool.asciidoc
index 163a729e51cc3..bfc5ca415c3ba 100644
--- a/docs/reference/cat/thread_pool.asciidoc
+++ b/docs/reference/cat/thread_pool.asciidoc
@@ -113,14 +113,15 @@ in the table below.
|Field Name |Alias |Description
|`type` |`t` |The current (*) type of thread pool (`fixed` or `scaling`)
|`active` |`a` |The number of active threads in the current thread pool
-|`size` |`s` |The number of threads in the current thread pool
+|`pool_size` |`psz` |The number of threads in the current thread pool
|`queue` |`q` |The number of tasks in the queue for the current thread pool
|`queue_size` |`qs` |The maximum number of tasks permitted in the queue for the current thread pool
|`rejected` |`r` |The number of tasks rejected by the thread pool executor
|`largest` |`l` |The highest number of active threads in the current thread pool
|`completed` |`c` |The number of tasks completed by the thread pool executor
-|`min` |`mi` |The configured minimum number of active threads allowed in the current thread pool
-|`max` |`ma` |The configured maximum number of active threads allowed in the current thread pool
+|`core` |`cr` |The configured core number of active threads allowed in the current thread pool
+|`max` |`mx` |The configured maximum number of active threads allowed in the current thread pool
+|`size` |`sz` |The configured fixed number of active threads allowed in the current thread pool
|`keep_alive` |`k` |The configured keep alive time for threads
|=======================================================================
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index ff42adf91336b..fe1ebf4739632 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -249,7 +249,7 @@ on a per-operation basis using the `routing` parameter. For example:
[source,js]
--------------------------------------------------
-POST twitter/tweet?routing=kimchy
+POST twitter/_doc?routing=kimchy
{
"user" : "kimchy",
"post_date" : "2009-11-15T14:12:12",
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index fe13c0c206eb7..5f34371ab8467 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -136,7 +136,7 @@ POST _reindex
// TEST[setup:twitter]
You can limit the documents by adding a type to the `source` or by adding a
-query. This will only copy ++tweet++'s made by `kimchy` into `new_twitter`:
+query. This will only copy tweets made by `kimchy` into `new_twitter`:
[source,js]
--------------------------------------------------
@@ -161,11 +161,13 @@ POST _reindex
`index` and `type` in `source` can both be lists, allowing you to copy from
lots of sources in one request. This will copy documents from the `_doc` and
-`post` types in the `twitter` and `blog` index. It'd include the `post` type in
-the `twitter` index and the `_doc` type in the `blog` index. If you want to be
-more specific you'll need to use the `query`. It also makes no effort to handle
-ID collisions. The target index will remain valid but it's not easy to predict
-which document will survive because the iteration order isn't well defined.
+`post` types in the `twitter` and `blog` index. The copied documents would include the
+`post` type in the `twitter` index and the `_doc` type in the `blog` index. For more
+specific parameters, you can use `query`.
+
+The Reindex API makes no effort to handle ID collisions. For such issues, the target index
+will remain valid, but it's not easy to predict which document will survive because
+the iteration order isn't well defined.
[source,js]
--------------------------------------------------
@@ -203,8 +205,8 @@ POST _reindex
// CONSOLE
// TEST[setup:twitter]
-If you want a particular set of documents from the twitter index you'll
-need to sort. Sorting makes the scroll less efficient but in some contexts
+If you want a particular set of documents from the `twitter` index you'll
+need to use `sort`. Sorting makes the scroll less efficient but in some contexts
it's worth it. If possible, prefer a more selective query to `size` and `sort`.
This will copy 10000 documents from `twitter` into `new_twitter`:
@@ -226,8 +228,8 @@ POST _reindex
// TEST[setup:twitter]
The `source` section supports all the elements that are supported in a
-<>. For instance only a subset of the
-fields from the original documents can be reindexed using source filtering
+<>. For instance, only a subset of the
+fields from the original documents can be reindexed using `source` filtering
as follows:
[source,js]
@@ -286,10 +288,10 @@ Set `ctx.op = "delete"` if your script decides that the document must be
deleted from the destination index. The deletion will be reported in the
`deleted` counter in the <>.
-Setting `ctx.op` to anything else is an error. Setting any
-other field in `ctx` is an error.
+Setting `ctx.op` to anything else will return an error, as will setting any
+other field in `ctx`.
-Think of the possibilities! Just be careful! With great power.... You can
+Think of the possibilities! Just be careful; you are able to
change:
* `_id`
@@ -299,7 +301,7 @@ change:
* `_routing`
Setting `_version` to `null` or clearing it from the `ctx` map is just like not
-sending the version in an indexing request. It will cause that document to be
+sending the version in an indexing request; it will cause the document to be
overwritten in the target index regardless of the version on the target or the
version type you use in the `_reindex` request.
@@ -310,11 +312,11 @@ preserved unless it's changed by the script. You can set `routing` on the
`keep`::
Sets the routing on the bulk request sent for each match to the routing on
-the match. The default.
+the match. This is the default value.
`discard`::
-Sets the routing on the bulk request sent for each match to null.
+Sets the routing on the bulk request sent for each match to `null`.
`=`::
@@ -422,7 +424,7 @@ POST _reindex
The `host` parameter must contain a scheme, host, and port (e.g.
`https://otherhost:9200`). The `username` and `password` parameters are
-optional and when they are present reindex will connect to the remote
+optional, and when they are present `_reindex` will connect to the remote
Elasticsearch node using basic auth. Be sure to use `https` when using
basic auth or the password will be sent in plain text.
@@ -446,7 +448,7 @@ NOTE: Reindexing from remote clusters does not support
Reindexing from a remote server uses an on-heap buffer that defaults to a
maximum size of 100mb. If the remote index includes very large documents you'll
-need to use a smaller batch size. The example below sets the batch size `10`
+need to use a smaller batch size. The example below sets the batch size to `10`
which is very, very small.
[source,js]
@@ -477,8 +479,8 @@ POST _reindex
It is also possible to set the socket read timeout on the remote connection
with the `socket_timeout` field and the connection timeout with the
-`connect_timeout` field. Both default to thirty seconds. This example
-sets the socket read timeout to one minute and the connection timeout to ten
+`connect_timeout` field. Both default to 30 seconds. This example
+sets the socket read timeout to one minute and the connection timeout to 10
seconds:
[source,js]
@@ -533,14 +535,14 @@ for details. `timeout` controls how long each write request waits for unavailabl
shards to become available. Both work exactly how they work in the
<>. As `_reindex` uses scroll search, you can also specify
the `scroll` parameter to control how long it keeps the "search context" alive,
-eg `?scroll=10m`, by default it's 5 minutes.
+(e.g. `?scroll=10m`). The default value is 5 minutes.
`requests_per_second` can be set to any positive decimal number (`1.4`, `6`,
-`1000`, etc) and throttles rate at which reindex issues batches of index
+`1000`, etc) and throttles the rate at which `_reindex` issues batches of index
operations by padding each batch with a wait time. The throttling can be
disabled by setting `requests_per_second` to `-1`.
-The throttling is done by waiting between batches so that scroll that reindex
+The throttling is done by waiting between batches so that the `scroll` which `_reindex`
uses internally can be given a timeout that takes into account the padding.
The padding time is the difference between the batch size divided by the
`requests_per_second` and the time spent writing. By default the batch size is
@@ -552,9 +554,9 @@ target_time = 1000 / 500 per second = 2 seconds
wait_time = target_time - write_time = 2 seconds - .5 seconds = 1.5 seconds
--------------------------------------------------
-Since the batch is issued as a single `_bulk` request large batch sizes will
+Since the batch is issued as a single `_bulk` request, large batch sizes will
cause Elasticsearch to create many requests and then wait for a while before
-starting the next set. This is "bursty" instead of "smooth". The default is `-1`.
+starting the next set. This is "bursty" instead of "smooth". The default value is `-1`.
[float]
[[docs-reindex-response-body]]
@@ -606,12 +608,12 @@ The JSON response looks like this:
`took`::
-The number of milliseconds from start to end of the whole operation.
+The total milliseconds the entire operation took.
`timed_out`::
This flag is set to `true` if any of the requests executed during the
-reindex has timed out.
+reindex timed out.
`total`::
@@ -657,7 +659,7 @@ The number of requests per second effectively executed during the reindex.
`throttled_until_millis`::
-This field should always be equal to zero in a delete by query response. It only
+This field should always be equal to zero in a `_delete_by_query` response. It only
has meaning when using the <>, where it
indicates the next time (in milliseconds since epoch) a throttled request will be
executed again in order to conform to `requests_per_second`.
@@ -681,7 +683,7 @@ GET _tasks?detailed=true&actions=*reindex
--------------------------------------------------
// CONSOLE
-The responses looks like:
+The response looks like:
[source,js]
--------------------------------------------------
@@ -726,9 +728,9 @@ The responses looks like:
// NOTCONSOLE
// We can't test tasks output
-<1> this object contains the actual status. It is just like the response json
-with the important addition of the `total` field. `total` is the total number
-of operations that the reindex expects to perform. You can estimate the
+<1> this object contains the actual status. It is identical to the response JSON
+except for the important addition of the `total` field. `total` is the total number
+of operations that the `_reindex` expects to perform. You can estimate the
progress by adding the `updated`, `created`, and `deleted` fields. The request
will finish when their sum is equal to the `total` field.
@@ -743,7 +745,7 @@ GET /_tasks/taskId:1
The advantage of this API is that it integrates with `wait_for_completion=false`
to transparently return the status of completed tasks. If the task is completed
-and `wait_for_completion=false` was set on it them it'll come back with a
+and `wait_for_completion=false` was set, it will return a
`results` or an `error` field. The cost of this feature is the document that
`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to
you to delete that document.
@@ -761,10 +763,10 @@ POST _tasks/task_id:1/_cancel
--------------------------------------------------
// CONSOLE
-The `task_id` can be found using the tasks API above.
+The `task_id` can be found using the Tasks API.
-Cancelation should happen quickly but might take a few seconds. The task status
-API above will continue to list the task until it is wakes to cancel itself.
+Cancelation should happen quickly but might take a few seconds. The Tasks
+API will continue to list the task until it wakes to cancel itself.
[float]
@@ -780,9 +782,9 @@ POST _reindex/task_id:1/_rethrottle?requests_per_second=-1
--------------------------------------------------
// CONSOLE
-The `task_id` can be found using the tasks API above.
+The `task_id` can be found using the Tasks API above.
-Just like when setting it on the `_reindex` API `requests_per_second`
+Just like when setting it on the Reindex API, `requests_per_second`
can be either `-1` to disable throttling or any decimal number
like `1.7` or `12` to throttle to that level. Rethrottling that speeds up the
query takes effect immediately but rethrotting that slows down the query will
@@ -806,7 +808,7 @@ POST test/_doc/1?refresh
--------------------------------------------------
// CONSOLE
-But you don't like the name `flag` and want to replace it with `tag`.
+but you don't like the name `flag` and want to replace it with `tag`.
`_reindex` can create the other index for you:
[source,js]
@@ -836,7 +838,7 @@ GET test2/_doc/1
// CONSOLE
// TEST[continued]
-and it'll look like:
+which will return:
[source,js]
--------------------------------------------------
@@ -854,8 +856,6 @@ and it'll look like:
--------------------------------------------------
// TESTRESPONSE
-Or you can search by `tag` or whatever you want.
-
[float]
[[docs-reindex-slice]]
=== Slicing
@@ -902,7 +902,7 @@ POST _reindex
// CONSOLE
// TEST[setup:big_twitter]
-Which you can verify works with:
+You can verify this works by:
[source,js]
----------------------------------------------------------------
@@ -912,7 +912,7 @@ POST new_twitter/_search?size=0&filter_path=hits.total
// CONSOLE
// TEST[continued]
-Which results in a sensible `total` like this one:
+which results in a sensible `total` like this one:
[source,js]
----------------------------------------------------------------
@@ -928,7 +928,7 @@ Which results in a sensible `total` like this one:
[[docs-reindex-automatic-slice]]
==== Automatic slicing
-You can also let reindex automatically parallelize using <> to
+You can also let `_reindex` automatically parallelize using <> to
slice on `_uid`. Use `slices` to specify the number of slices to use:
[source,js]
@@ -946,7 +946,7 @@ POST _reindex?slices=5&refresh
// CONSOLE
// TEST[setup:big_twitter]
-Which you also can verify works with:
+You can also this verify works by:
[source,js]
----------------------------------------------------------------
@@ -955,7 +955,7 @@ POST new_twitter/_search?size=0&filter_path=hits.total
// CONSOLE
// TEST[continued]
-Which results in a sensible `total` like this one:
+which results in a sensible `total` like this one:
[source,js]
----------------------------------------------------------------
@@ -979,7 +979,7 @@ section above, creating sub-requests which means it has some quirks:
sub-requests are "child" tasks of the task for the request with `slices`.
* Fetching the status of the task for the request with `slices` only contains
the status of completed slices.
-* These sub-requests are individually addressable for things like cancellation
+* These sub-requests are individually addressable for things like cancelation
and rethrottling.
* Rethrottling the request with `slices` will rethrottle the unfinished
sub-request proportionally.
@@ -992,7 +992,7 @@ are distributed proportionally to each sub-request. Combine that with the point
above about distribution being uneven and you should conclude that the using
`size` with `slices` might not result in exactly `size` documents being
`_reindex`ed.
-* Each sub-requests gets a slightly different snapshot of the source index
+* Each sub-request gets a slightly different snapshot of the source index,
though these are all taken at approximately the same time.
[float]
@@ -1000,12 +1000,12 @@ though these are all taken at approximately the same time.
===== Picking the number of slices
If slicing automatically, setting `slices` to `auto` will choose a reasonable
-number for most indices. If you're slicing manually or otherwise tuning
+number for most indices. If slicing manually or otherwise tuning
automatic slicing, use these guidelines.
Query performance is most efficient when the number of `slices` is equal to the
-number of shards in the index. If that number is large, (for example,
-500) choose a lower number as too many `slices` will hurt performance. Setting
+number of shards in the index. If that number is large (e.g. 500),
+choose a lower number as too many `slices` will hurt performance. Setting
`slices` higher than the number of shards generally does not improve efficiency
and adds overhead.
@@ -1018,10 +1018,10 @@ documents being reindexed and cluster resources.
[float]
=== Reindex daily indices
-You can use `_reindex` in combination with <>
- to reindex daily indices to apply a new template to the existing documents.
+You can use `_reindex` in combination with <>
+to reindex daily indices to apply a new template to the existing documents.
-Assuming you have indices consisting of documents as following:
+Assuming you have indices consisting of documents as follows:
[source,js]
----------------------------------------------------------------
@@ -1032,12 +1032,12 @@ PUT metricbeat-2016.05.31/_doc/1?refresh
----------------------------------------------------------------
// CONSOLE
-The new template for the `metricbeat-*` indices is already loaded into Elasticsearch
+The new template for the `metricbeat-*` indices is already loaded into Elasticsearch,
but it applies only to the newly created indices. Painless can be used to reindex
the existing documents and apply the new template.
The script below extracts the date from the index name and creates a new index
-with `-1` appended. All data from `metricbeat-2016.05.31` will be reindex
+with `-1` appended. All data from `metricbeat-2016.05.31` will be reindexed
into `metricbeat-2016.05.31-1`.
[source,js]
@@ -1059,7 +1059,7 @@ POST _reindex
// CONSOLE
// TEST[continued]
-All documents from the previous metricbeat indices now can be found in the `*-1` indices.
+All documents from the previous metricbeat indices can now be found in the `*-1` indices.
[source,js]
----------------------------------------------------------------
@@ -1069,13 +1069,13 @@ GET metricbeat-2016.05.31-1/_doc/1
// CONSOLE
// TEST[continued]
-The previous method can also be used in combination with <>
-to only load the existing data into the new index, but also rename fields if needed.
+The previous method can also be used in conjunction with <>
+to load only the existing data into the new index and rename any fields if needed.
[float]
=== Extracting a random subset of an index
-Reindex can be used to extract a random subset of an index for testing:
+`_reindex` can be used to extract a random subset of an index for testing:
[source,js]
----------------------------------------------------------------
@@ -1100,5 +1100,5 @@ POST _reindex
// CONSOLE
// TEST[setup:big_twitter]
-<1> Reindex defaults to sorting by `_doc` so `random_score` won't have any
+<1> `_reindex` defaults to sorting by `_doc` so `random_score` will not have any
effect unless you override the sort to `_score`.
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
index 18aee6094f80a..7ba7e2da63369 100644
--- a/docs/reference/docs/update.asciidoc
+++ b/docs/reference/docs/update.asciidoc
@@ -118,8 +118,11 @@ POST test/_doc/1/_update
The update API also support passing a partial document,
which will be merged into the existing document (simple recursive merge,
-inner merging of objects, replacing core "keys/values" and arrays). For
-example:
+inner merging of objects, replacing core "keys/values" and arrays).
+To fully replace the existing document, the <> should
+be used instead.
+The following partial update adds a new field to the
+existing document:
[source,js]
--------------------------------------------------
diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index af7fc8fa6d69b..937917823f5a6 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -777,7 +777,7 @@ GET /bank/_search
// CONSOLE
// TEST[continued]
-The difference here is that instead of passing `q=*` in the URI, we POST a JSON-style query request body to the `_search` API. We'll discuss this JSON query in the next section.
+The difference here is that instead of passing `q=*` in the URI, we provide a JSON-style query request body to the `_search` API. We'll discuss this JSON query in the next section.
////
Hidden response just so we can assert that it is indeed the same but don't have
diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
index d0fd5dd399867..40f7de90c0ab2 100644
--- a/docs/reference/index-modules/similarity.asciidoc
+++ b/docs/reference/index-modules/similarity.asciidoc
@@ -82,20 +82,6 @@ This similarity has the following options:
Type name: `BM25`
-[float]
-[[classic-similarity]]
-==== Classic similarity
-
-The classic similarity that is based on the TF/IDF model. This
-similarity has the following option:
-
-`discount_overlaps`::
- Determines whether overlap tokens (Tokens with
- 0 position increment) are ignored when computing norm. By default this
- is true, meaning overlap tokens do not count when computing norms.
-
-Type name: `classic`
-
[float]
[[dfr]]
==== DFR similarity
@@ -541,7 +527,7 @@ PUT /index
"index": {
"similarity": {
"default": {
- "type": "classic"
+ "type": "boolean"
}
}
}
@@ -563,7 +549,7 @@ PUT /index/_settings
"index": {
"similarity": {
"default": {
- "type": "classic"
+ "type": "boolean"
}
}
}
diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc
index 91fac0908ef7f..db1f7c2fe00a9 100644
--- a/docs/reference/indices/flush.asciidoc
+++ b/docs/reference/indices/flush.asciidoc
@@ -98,7 +98,7 @@ which returns something similar to:
"translog_uuid" : "hnOG3xFcTDeoI_kvvvOdNA",
"history_uuid" : "XP7KDJGiS1a2fHYiFL5TXQ",
"local_checkpoint" : "-1",
- "translog_generation" : "3",
+ "translog_generation" : "2",
"max_seq_no" : "-1",
"sync_id" : "AVvFY-071siAOuFGEO9P", <1>
"max_unsafe_auto_id_timestamp" : "-1"
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index b31fc1ef5ea1d..8a7c33086abe8 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -1341,7 +1341,7 @@ Here is an example of a pipeline specifying custom pattern definitions:
{
"grok": {
"field": "message",
- "patterns": ["my %{FAVORITE_DOG:dog} is colored %{RGB:color}"]
+ "patterns": ["my %{FAVORITE_DOG:dog} is colored %{RGB:color}"],
"pattern_definitions" : {
"FAVORITE_DOG" : "beagle",
"RGB" : "RED|GREEN|BLUE"
diff --git a/docs/reference/mapping/params/similarity.asciidoc b/docs/reference/mapping/params/similarity.asciidoc
index 3509cd0cf8eb5..a0be0fb3ccbeb 100644
--- a/docs/reference/mapping/params/similarity.asciidoc
+++ b/docs/reference/mapping/params/similarity.asciidoc
@@ -44,13 +44,9 @@ PUT my_index
"default_field": { <1>
"type": "text"
},
- "classic_field": {
- "type": "text",
- "similarity": "classic" <2>
- },
"boolean_sim_field": {
"type": "text",
- "similarity": "boolean" <3>
+ "similarity": "boolean" <2>
}
}
}
@@ -59,5 +55,4 @@ PUT my_index
--------------------------------------------------
// CONSOLE
<1> The `default_field` uses the `BM25` similarity.
-<2> The `classic_field` uses the `classic` similarity (ie TF/IDF).
-<3> The `boolean_sim_field` uses the `boolean` similarity.
+<2> The `boolean_sim_field` uses the `boolean` similarity.
diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
index 7d92bb3b2e7c7..ae81773e6a0a2 100644
--- a/docs/reference/mapping/types/geo-point.asciidoc
+++ b/docs/reference/mapping/types/geo-point.asciidoc
@@ -105,6 +105,13 @@ The following parameters are accepted by `geo_point` fields:
If `true`, malformed geo-points are ignored. If `false` (default),
malformed geo-points throw an exception and reject the whole document.
+`ignore_z_value`::
+
+ If `true` (default) three dimension points will be accepted (stored in source)
+ but only latitude and longitude values will be indexed; the third dimension is
+ ignored. If `false`, geo-points containing any more than latitude and longitude
+ (two dimensions) values throw an exception and reject the whole document.
+
==== Using geo-points in scripts
When accessing the value of a geo-point in a script, the value is returned as
diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc
index 23caaf6a8ec5c..43ad71e37073f 100644
--- a/docs/reference/mapping/types/geo-shape.asciidoc
+++ b/docs/reference/mapping/types/geo-shape.asciidoc
@@ -91,6 +91,12 @@ false (default), malformed GeoJSON and WKT shapes throw an exception and reject
entire document.
| `false`
+|`ignore_z_value` |If `true` (default) three dimension points will be accepted (stored in source)
+but only latitude and longitude values will be indexed; the third dimension is ignored. If `false`,
+geo-points containing any more than latitude and longitude (two dimensions) values throw an exception
+and reject the whole document.
+| `true`
+
|=======================================================================
@@ -148,12 +154,12 @@ are provided:
[float]
===== Accuracy
-Geo_shape does not provide 100% accuracy and depending on how it is
-configured it may return some false positives or false negatives for
-certain queries. To mitigate this, it is important to select an
-appropriate value for the tree_levels parameter and to adjust
-expectations accordingly. For example, a point may be near the border of
-a particular grid cell and may thus not match a query that only matches the
+Geo_shape does not provide 100% accuracy and depending on how it is configured
+it may return some false positives for `INTERSECTS`, `WITHIN` and `CONTAINS`
+queries, and some false negatives for `DISJOINT` queries. To mitigate this, it
+is important to select an appropriate value for the tree_levels parameter and
+to adjust expectations accordingly. For example, a point may be near the border
+of a particular grid cell and may thus not match a query that only matches the
cell right next to it -- even though the shape is very close to the point.
[float]
@@ -214,7 +220,7 @@ to Elasticsearch types:
|=======================================================================
|GeoJSON Type |WKT Type |Elasticsearch Type |Description
-|`Point` |`POINT` |`point` |A single geographic coordinate.
+|`Point` |`POINT` |`point` |A single geographic coordinate. Note: Elasticsearch uses WGS-84 coordinates only.
|`LineString` |`LINESTRING` |`linestring` |An arbitrary line given two or more points.
|`Polygon` |`POLYGON` |`polygon` |A _closed_ polygon whose first and last point
must match, thus requiring `n + 1` vertices to create an `n`-sided
@@ -372,22 +378,24 @@ POST /example/doc
// CONSOLE
// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
-*IMPORTANT NOTE:* GeoJSON and WKT do not enforce a specific order for vertices
-thus ambiguous polygons around the dateline and poles are possible. To alleviate
-ambiguity the Open Geospatial Consortium (OGC)
-http://www.opengeospatial.org/standards/sfa[Simple Feature Access] specification
-defines the following vertex ordering:
-
-* Outer Ring - Counterclockwise
-* Inner Ring(s) / Holes - Clockwise
-
-For polygons that do not cross the dateline, vertex order will not matter in
-Elasticsearch. For polygons that do cross the dateline, Elasticsearch requires
-vertex ordering to comply with the OGC specification. Otherwise, an unintended polygon
-may be created and unexpected query/filter results will be returned.
-
-The following provides an example of an ambiguous polygon. Elasticsearch will apply
-OGC standards to eliminate ambiguity resulting in a polygon that crosses the dateline.
+*IMPORTANT NOTE:* WKT does not enforce a specific order for vertices thus
+ambiguous polygons around the dateline and poles are possible.
+https://tools.ietf.org/html/rfc7946#section-3.1.6[GeoJSON] mandates that the
+outer polygon must be counterclockwise and interior shapes must be clockwise,
+which agrees with the Open Geospatial Consortium (OGC)
+http://www.opengeospatial.org/standards/sfa[Simple Feature Access]
+specification for vertex ordering.
+
+Elasticsearch accepts both clockwise and counterclockwise polygons if they
+appear not to cross the dateline (i.e. they cross less than 180° of longitude),
+but for polygons that do cross the dateline (or for other polygons wider than
+180°) Elasticsearch requires the vertex ordering to comply with the OGC and
+GeoJSON specifications. Otherwise, an unintended polygon may be created and
+unexpected query/filter results will be returned.
+
+The following provides an example of an ambiguous polygon. Elasticsearch will
+apply the GeoJSON standard to eliminate ambiguity resulting in a polygon that
+crosses the dateline.
[source,js]
--------------------------------------------------
diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc
index 8f1474aa57cbe..b0ab90546c3a8 100644
--- a/docs/reference/migration/migrate_7_0/mappings.asciidoc
+++ b/docs/reference/migration/migrate_7_0/mappings.asciidoc
@@ -24,3 +24,16 @@ the index setting `index.mapping.nested_objects.limit`.
==== The `update_all_types` option has been removed
This option is useless now that all indices have at most one type.
+
+=== The `classic` similarity has been removed
+
+The `classic` similarity relied on coordination factors for scoring to be good
+in presence of stopwords in the query. This feature has been removed from
+Lucene, which means that the `classic` similarity now produces scores of lower
+quality. It is advised to switch to `BM25` instead, which is widely accepted
+as a better alternative.
+
+=== Similarities fail when unsupported options are provided
+
+An error will now be thrown when unknown configuration options are provided
+to similarities. Such unknown parameters were ignored before.
diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc
index c55ad8c424057..0d3770993b2ff 100644
--- a/docs/reference/migration/migrate_7_0/search.asciidoc
+++ b/docs/reference/migration/migrate_7_0/search.asciidoc
@@ -5,6 +5,10 @@
* The default value for `transpositions` parameter of `fuzzy` query
has been changed to `true`.
+* The `query_string` options `use_dismax`, `split_on_whitespace`,
+ `all_fields`, `locale`, `auto_generate_phrase_query` and
+ `lowercase_expanded_terms` deprecated in 6.x have been removed.
+
==== Adaptive replica selection enabled by default
Adaptive replica selection has been enabled by default. If you wish to return to
diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc
index 3963312c0f4ea..837cfcc43ebf7 100644
--- a/docs/reference/modules/cluster/misc.asciidoc
+++ b/docs/reference/modules/cluster/misc.asciidoc
@@ -56,3 +56,30 @@ PUT /_cluster/settings
}
-------------------------------
// CONSOLE
+
+
+[[persistent-tasks-allocation]]
+==== Persistent Tasks Allocations
+
+Plugins can create a kind of tasks called persistent tasks. Those tasks are
+usually long-live tasks and are stored in the cluster state, allowing the
+tasks to be revived after a full cluster restart.
+
+Every time a persistent task is created, the master nodes takes care of
+assigning the task to a node of the cluster, and the assigned node will then
+pick up the task and execute it locally. The process of assigning persistent
+tasks to nodes is controlled by the following property, which can be updated
+dynamically:
+
+`cluster.persistent_tasks.allocation.enable`::
++
+--
+Enable or disable allocation for persistent tasks:
+
+* `all` - (default) Allows persistent tasks to be assigned to nodes
+* `none` - No allocations are allowed for any type of persistent task
+
+This setting does not affect the persistent tasks that are already being executed.
+Only newly created persistent tasks, or tasks that must be reassigned (after a node
+left the cluster, for example), are impacted by this setting.
+--
diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc
index a83270ec2aace..920f62043cfe2 100644
--- a/docs/reference/modules/http.asciidoc
+++ b/docs/reference/modules/http.asciidoc
@@ -39,7 +39,7 @@ from the outside. Defaults to the actual port assigned via `http.port`.
|`http.host` |Used to set the `http.bind_host` and the `http.publish_host` Defaults to `http.host` or `network.host`.
|`http.max_content_length` |The max content of an HTTP request. Defaults to
-`100mb`. If set to greater than `Integer.MAX_VALUE`, it will be reset to 100mb.
+`100mb`.
|`http.max_initial_line_length` |The max length of an HTTP URL. Defaults
to `4kb`
diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index b8883173b9890..ea3f99debb94e 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -401,7 +401,7 @@ created the snapshotting process will be aborted and all files created as part o
cleaned. Therefore, the delete snapshot operation can be used to cancel long running snapshot operations that were
started by mistake.
-A repository can be deleted using the following command:
+A repository can be unregistered using the following command:
[source,sh]
-----------------------------------
@@ -410,7 +410,7 @@ DELETE /_snapshot/my_fs_backup
// CONSOLE
// TEST[continued]
-When a repository is deleted, Elasticsearch only removes the reference to the location where the repository is storing
+When a repository is unregistered, Elasticsearch only removes the reference to the location where the repository is storing
the snapshots. The snapshots themselves are left untouched and in place.
[float]
diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc
index 40b0978cc1977..17d07e25259f5 100644
--- a/docs/reference/query-dsl/query-string-query.asciidoc
+++ b/docs/reference/query-dsl/query-string-query.asciidoc
@@ -99,8 +99,6 @@ phrase matches are required. Default value is `0`.
|`boost` |Sets the boost value of the query. Defaults to `1.0`.
-|`auto_generate_phrase_queries` |Defaults to `false`.
-
|`analyze_wildcard` |By default, wildcards terms in a query string are
not analyzed. By setting this value to `true`, a best effort will be
made to analyze those as well.
@@ -129,10 +127,6 @@ comprehensive example.
|`auto_generate_synonyms_phrase_query` |Whether phrase queries should be automatically generated for multi terms synonyms.
Defaults to `true`.
-|`all_fields` | deprecated[6.0.0, set `default_field` to `*` instead]
-Perform the query on all fields detected in the mapping that can
-be queried.
-
|=======================================================================
When a multi term query is being generated, one can control how it gets
@@ -318,7 +312,7 @@ GET /_search
The example above creates a boolean query:
-`(ny OR (new AND york)) city)`
+`(ny OR (new AND york)) city`
that matches documents with the term `ny` or the conjunction `new AND york`.
By default the parameter `auto_generate_synonyms_phrase_query` is set to `true`.
diff --git a/docs/reference/query-dsl/query-string-syntax.asciidoc b/docs/reference/query-dsl/query-string-syntax.asciidoc
index c73543c99a1d9..54bae2588e1f6 100644
--- a/docs/reference/query-dsl/query-string-syntax.asciidoc
+++ b/docs/reference/query-dsl/query-string-syntax.asciidoc
@@ -23,11 +23,9 @@ search terms, but it is possible to specify other fields in the query syntax:
status:active
-* where the `title` field contains `quick` or `brown`.
- If you omit the OR operator the default operator will be used
+* where the `title` field contains `quick` or `brown`
title:(quick OR brown)
- title:(quick brown)
* where the `author` field contains the exact phrase `"john smith"`
@@ -36,7 +34,7 @@ search terms, but it is possible to specify other fields in the query syntax:
* where any of the fields `book.title`, `book.content` or `book.date` contains
`quick` or `brown` (note how we need to escape the `*` with a backslash):
- book.\*:(quick brown)
+ book.\*:(quick OR brown)
* where the field `title` has any non-null value:
diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
index 53c6ac9cf6030..e2998086c8917 100644
--- a/docs/reference/search/rank-eval.asciidoc
+++ b/docs/reference/search/rank-eval.asciidoc
@@ -1,14 +1,14 @@
[[search-rank-eval]]
== Ranking Evaluation API
+experimental[The ranking evaluation API is experimental and may be changed or removed completely in a future release, as well as change in non-backwards compatible ways on minor versions updates. Elastic will take a best effort approach to fix any issues, but experimental features are not subject to the support SLA of official GA features.]
+
The ranking evaluation API allows to evaluate the quality of ranked search
results over a set of typical search queries. Given this set of queries and a
list or manually rated documents, the `_rank_eval` endpoint calculates and
returns typical information retrieval metrics like _mean reciprocal rank_,
_precision_ or _discounted cumulative gain_.
-experimental[The ranking evaluation API is new and may change in non-backwards compatible ways in the future, even on minor versions updates.]
-
[float]
=== Overview
@@ -17,7 +17,7 @@ Users have a specific _information need_, e.g. they are looking for gift in a we
They usually enters some search terms into a search box or some other web form.
All of this information, together with meta information about the user (e.g. the browser, location, earlier preferences etc...) then gets translated into a query to the underlying search system.
-The challenge for search engineers is to tweak this translation process from user entries to a concrete query in such a way, that the search results contain the most relevant information with respect to the users information_need.
+The challenge for search engineers is to tweak this translation process from user entries to a concrete query in such a way, that the search results contain the most relevant information with respect to the users information need.
This can only be done if the search result quality is evaluated constantly across a representative test suite of typical user queries, so that improvements in the rankings for one particular query doesn't negatively effect the ranking for other types of queries.
In order to get started with search quality evaluation, three basic things are needed:
@@ -26,7 +26,7 @@ In order to get started with search quality evaluation, three basic things are n
. a collection of typical search requests that users enter into your system
. a set of document ratings that judge the documents relevance with respect to a search request+
It is important to note that one set of document ratings is needed per test query, and that
- the relevance judgements are based on the _information_need_ of the user that entered the query.
+ the relevance judgements are based on the information need of the user that entered the query.
The ranking evaluation API provides a convenient way to use this information in a ranking evaluation request to calculate different search evaluation metrics. This gives a first estimation of your overall search quality and give you a measurement to optimize against when fine-tuning various aspect of the query generation in your application.
@@ -41,7 +41,7 @@ GET /my_index/_rank_eval
{
"requests": [ ... ], <1>
"metric": { <2>
- "reciprocal_rank": { ... } <3>
+ "mean_reciprocal_rank": { ... } <3>
}
}
------------------------------
@@ -85,7 +85,7 @@ The request section contains several search requests typical to your application
<3> a list of document ratings, each entry containing the documents `_index` and `_id` together with
the rating of the documents relevance with regards to this search request
-A document `rating` can be any integer value that expresses the relevance of the document on a user defined scale. For some of the metrics, just giving a binary rating (e.g. `0` for irrelevant and `1` for relevant) will be sufficient, other metrics can use a more fine grained scale.
+A document `rating` can be any integer value that expresses the relevance of the document on a user defined scale. For some of the metrics, just giving a binary rating (e.g. `0` for irrelevant and `1` for relevant) will be sufficient, other metrics can use a more fine grained scale.
[float]
=== Template based ranking evaluation
@@ -158,6 +158,7 @@ GET /twitter/_rank_eval
}],
"metric": {
"precision": {
+ "k" : 20,
"relevant_rating_threshold": 1,
"ignore_unlabeled": false
}
@@ -172,7 +173,9 @@ The `precision` metric takes the following optional parameters
[cols="<,<",options="header",]
|=======================================================================
|Parameter |Description
-|`relevant_rating_threshold` |Sets the rating threshold above which documents are considered to be
+|`k` |sets the maximum number of documents retrieved per query. This value will act in place of the usual `size` parameter
+in the query. Defaults to 10.
+|`relevant_rating_threshold` |sets the rating threshold above which documents are considered to be
"relevant". Defaults to `1`.
|`ignore_unlabeled` |controls how unlabeled documents in the search results are counted.
If set to 'true', unlabeled documents are ignored and neither count as relevant or irrelevant. Set to 'false' (the default), they are treated as irrelevant.
@@ -198,6 +201,7 @@ GET /twitter/_rank_eval
}],
"metric": {
"mean_reciprocal_rank": {
+ "k" : 20,
"relevant_rating_threshold" : 1
}
}
@@ -211,6 +215,8 @@ The `mean_reciprocal_rank` metric takes the following optional parameters
[cols="<,<",options="header",]
|=======================================================================
|Parameter |Description
+|`k` |sets the maximum number of documents retrieved per query. This value will act in place of the usual `size` parameter
+in the query. Defaults to 10.
|`relevant_rating_threshold` |Sets the rating threshold above which documents are considered to be
"relevant". Defaults to `1`.
|=======================================================================
@@ -234,6 +240,7 @@ GET /twitter/_rank_eval
}],
"metric": {
"dcg": {
+ "k" : 20,
"normalize": false
}
}
@@ -247,6 +254,8 @@ The `dcg` metric takes the following optional parameters:
[cols="<,<",options="header",]
|=======================================================================
|Parameter |Description
+|`k` |sets the maximum number of documents retrieved per query. This value will act in place of the usual `size` parameter
+in the query. Defaults to 10.
|`normalize` | If set to `true`, this metric will calculate the https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG[Normalized DCG].
|=======================================================================
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index 997f267a7e29f..b9b99b708031e 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -30,3 +30,5 @@ include::important-settings/heap-size.asciidoc[]
include::important-settings/heap-dump-path.asciidoc[]
include::important-settings/gc-logging.asciidoc[]
+
+include::important-settings/error-file.asciidoc[]
diff --git a/server/src/main/java/org/elasticsearch/common/Booleans.java b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Booleans.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/Booleans.java
rename to libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Booleans.java
index 025174c477d64..7447f0111f7e2 100644
--- a/server/src/main/java/org/elasticsearch/common/Booleans.java
+++ b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Booleans.java
@@ -73,6 +73,19 @@ public static boolean parseBoolean(String value) {
throw new IllegalArgumentException("Failed to parse value [" + value + "] as only [true] or [false] are allowed.");
}
+ private static boolean hasText(CharSequence str) {
+ if (str == null || str.length() == 0) {
+ return false;
+ }
+ int strLen = str.length();
+ for (int i = 0; i < strLen; i++) {
+ if (!Character.isWhitespace(str.charAt(i))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
*
* @param value text to parse.
@@ -80,14 +93,14 @@ public static boolean parseBoolean(String value) {
* @return see {@link #parseBoolean(String)}
*/
public static boolean parseBoolean(String value, boolean defaultValue) {
- if (Strings.hasText(value)) {
+ if (hasText(value)) {
return parseBoolean(value);
}
return defaultValue;
}
public static Boolean parseBoolean(String value, Boolean defaultValue) {
- if (Strings.hasText(value)) {
+ if (hasText(value)) {
return parseBoolean(value);
}
return defaultValue;
diff --git a/server/src/main/java/org/elasticsearch/common/CheckedFunction.java b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/CheckedFunction.java
similarity index 100%
rename from server/src/main/java/org/elasticsearch/common/CheckedFunction.java
rename to libs/elasticsearch-core/src/main/java/org/elasticsearch/common/CheckedFunction.java
diff --git a/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Glob.java b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Glob.java
new file mode 100644
index 0000000000000..f0baf75bd4db1
--- /dev/null
+++ b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Glob.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+/**
+ * Utility class for glob-like matching
+ */
+public class Glob {
+
+ /**
+ * Match a String against the given pattern, supporting the following simple
+ * pattern styles: "xxx*", "*xxx", "*xxx*" and "xxx*yyy" matches (with an
+ * arbitrary number of pattern parts), as well as direct equality.
+ *
+ * @param pattern the pattern to match against
+ * @param str the String to match
+ * @return whether the String matches the given pattern
+ */
+ public static boolean globMatch(String pattern, String str) {
+ if (pattern == null || str == null) {
+ return false;
+ }
+ int firstIndex = pattern.indexOf('*');
+ if (firstIndex == -1) {
+ return pattern.equals(str);
+ }
+ if (firstIndex == 0) {
+ if (pattern.length() == 1) {
+ return true;
+ }
+ int nextIndex = pattern.indexOf('*', firstIndex + 1);
+ if (nextIndex == -1) {
+ return str.endsWith(pattern.substring(1));
+ } else if (nextIndex == 1) {
+ // Double wildcard "**" - skipping the first "*"
+ return globMatch(pattern.substring(1), str);
+ }
+ String part = pattern.substring(1, nextIndex);
+ int partIndex = str.indexOf(part);
+ while (partIndex != -1) {
+ if (globMatch(pattern.substring(nextIndex), str.substring(partIndex + part.length()))) {
+ return true;
+ }
+ partIndex = str.indexOf(part, partIndex + 1);
+ }
+ return false;
+ }
+ return (str.length() >= firstIndex &&
+ pattern.substring(0, firstIndex).equals(str.substring(0, firstIndex)) &&
+ globMatch(pattern.substring(firstIndex), str.substring(firstIndex)));
+ }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/common/Nullable.java b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Nullable.java
similarity index 100%
rename from server/src/main/java/org/elasticsearch/common/Nullable.java
rename to libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Nullable.java
diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle
new file mode 100644
index 0000000000000..c8b37108ff93c
--- /dev/null
+++ b/libs/x-content/build.gradle
@@ -0,0 +1,85 @@
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.build'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-scm'
+
+archivesBaseName = 'elasticsearch-x-content'
+
+publishing {
+ publications {
+ nebula {
+ artifactId = archivesBaseName
+ }
+ }
+}
+
+dependencies {
+ compile "org.elasticsearch:elasticsearch-core:${version}"
+
+ compile "org.yaml:snakeyaml:${versions.snakeyaml}"
+ compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+ compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}"
+ compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
+ compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
+
+ testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+ testCompile "junit:junit:${versions.junit}"
+ testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+
+ if (isEclipse == false || project.path == ":libs:x-content-tests") {
+ testCompile("org.elasticsearch.test:framework:${version}") {
+ exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content'
+ }
+ }
+
+}
+
+forbiddenApisMain {
+ // x-content does not depend on server
+ // TODO: Need to decide how we want to handle for forbidden signatures with the changes to core
+ signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+if (isEclipse) {
+ // in eclipse the project is under a fake root, we need to change around the source sets
+ sourceSets {
+ if (project.path == ":libs:x-content") {
+ main.java.srcDirs = ['java']
+ main.resources.srcDirs = ['resources']
+ } else {
+ test.java.srcDirs = ['java']
+ test.resources.srcDirs = ['resources']
+ }
+ }
+}
+
+thirdPartyAudit.excludes = [
+ // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
+ 'com.fasterxml.jackson.databind.ObjectMapper',
+]
+
+dependencyLicenses {
+ mapping from: /jackson-.*/, to: 'jackson'
+}
+
+jarHell.enabled = false
diff --git a/server/licenses/jackson-LICENSE b/libs/x-content/licenses/jackson-LICENSE
similarity index 100%
rename from server/licenses/jackson-LICENSE
rename to libs/x-content/licenses/jackson-LICENSE
diff --git a/server/licenses/jackson-NOTICE b/libs/x-content/licenses/jackson-NOTICE
similarity index 100%
rename from server/licenses/jackson-NOTICE
rename to libs/x-content/licenses/jackson-NOTICE
diff --git a/server/licenses/jackson-core-2.8.10.jar.sha1 b/libs/x-content/licenses/jackson-core-2.8.10.jar.sha1
similarity index 100%
rename from server/licenses/jackson-core-2.8.10.jar.sha1
rename to libs/x-content/licenses/jackson-core-2.8.10.jar.sha1
diff --git a/server/licenses/jackson-dataformat-cbor-2.8.10.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.8.10.jar.sha1
similarity index 100%
rename from server/licenses/jackson-dataformat-cbor-2.8.10.jar.sha1
rename to libs/x-content/licenses/jackson-dataformat-cbor-2.8.10.jar.sha1
diff --git a/server/licenses/jackson-dataformat-smile-2.8.10.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.8.10.jar.sha1
similarity index 100%
rename from server/licenses/jackson-dataformat-smile-2.8.10.jar.sha1
rename to libs/x-content/licenses/jackson-dataformat-smile-2.8.10.jar.sha1
diff --git a/server/licenses/jackson-dataformat-yaml-2.8.10.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.8.10.jar.sha1
similarity index 100%
rename from server/licenses/jackson-dataformat-yaml-2.8.10.jar.sha1
rename to libs/x-content/licenses/jackson-dataformat-yaml-2.8.10.jar.sha1
diff --git a/server/licenses/snakeyaml-1.17.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.17.jar.sha1
similarity index 100%
rename from server/licenses/snakeyaml-1.17.jar.sha1
rename to libs/x-content/licenses/snakeyaml-1.17.jar.sha1
diff --git a/server/licenses/snakeyaml-LICENSE.txt b/libs/x-content/licenses/snakeyaml-LICENSE.txt
similarity index 100%
rename from server/licenses/snakeyaml-LICENSE.txt
rename to libs/x-content/licenses/snakeyaml-LICENSE.txt
diff --git a/server/licenses/snakeyaml-NOTICE.txt b/libs/x-content/licenses/snakeyaml-NOTICE.txt
similarity index 100%
rename from server/licenses/snakeyaml-NOTICE.txt
rename to libs/x-content/licenses/snakeyaml-NOTICE.txt
diff --git a/libs/x-content/src/main/eclipse-build.gradle b/libs/x-content/src/main/eclipse-build.gradle
new file mode 100644
index 0000000000000..a17f089781183
--- /dev/null
+++ b/libs/x-content/src/main/eclipse-build.gradle
@@ -0,0 +1,3 @@
+
+// this is just shell gradle file for eclipse to have separate projects for secure-sm src and tests
+apply from: '../../build.gradle'
diff --git a/server/src/main/java/org/elasticsearch/common/ParseField.java b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/ParseField.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
index 2c68ea7711bb2..084d82372c0ce 100644
--- a/server/src/main/java/org/elasticsearch/common/ParseField.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java
@@ -35,6 +35,8 @@ public class ParseField {
private String allReplacedWith = null;
private final String[] allNames;
+ private static final String[] EMPTY = new String[0];
+
/**
* @param name
* the primary name for this field. This will be returned by
@@ -46,7 +48,7 @@ public class ParseField {
public ParseField(String name, String... deprecatedNames) {
this.name = name;
if (deprecatedNames == null || deprecatedNames.length == 0) {
- this.deprecatedNames = Strings.EMPTY_ARRAY;
+ this.deprecatedNames = EMPTY;
} else {
final HashSet set = new HashSet<>();
Collections.addAll(set, deprecatedNames);
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ContextParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ContextParser.java
similarity index 100%
rename from server/src/main/java/org/elasticsearch/common/xcontent/ContextParser.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ContextParser.java
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
similarity index 100%
rename from server/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationPlugin.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedObjectNotFoundException.java
similarity index 57%
rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationPlugin.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedObjectNotFoundException.java
index bd49acbe4da0b..ecc322b60d8fc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationPlugin.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedObjectNotFoundException.java
@@ -17,20 +17,19 @@
* under the License.
*/
-package org.elasticsearch.search.aggregations.bucket.composite;
+package org.elasticsearch.common.xcontent;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SearchPlugin;
+/**
+ * Thrown when {@link NamedXContentRegistry} cannot locate a named object to
+ * parse for a particular name
+ */
+public class NamedObjectNotFoundException extends XContentParseException {
-import java.util.Arrays;
-import java.util.List;
+ public NamedObjectNotFoundException(String message) {
+ this(null, message);
+ }
-public class CompositeAggregationPlugin extends Plugin implements SearchPlugin {
- @Override
- public List getAggregations() {
- return Arrays.asList(
- new AggregationSpec(CompositeAggregationBuilder.NAME, CompositeAggregationBuilder::new, CompositeAggregationBuilder::parse)
- .addResultReader(InternalComposite::new)
- );
+ public NamedObjectNotFoundException(XContentLocation location, String message) {
+ super(location, message);
}
}
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
index c19a667776f2e..9135bf648a19e 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
@@ -19,10 +19,8 @@
package org.elasticsearch.common.xcontent;
-import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
import java.io.IOException;
import java.util.ArrayList;
@@ -114,28 +112,31 @@ public NamedXContentRegistry(List entries) {
}
/**
- * Parse a named object, throwing an exception if the parser isn't found. Throws an {@link ElasticsearchException} if the
- * {@code categoryClass} isn't registered because this is almost always a bug. Throws a {@link UnknownNamedObjectException} if the
+ * Parse a named object, throwing an exception if the parser isn't found. Throws an {@link NamedObjectNotFoundException} if the
+ * {@code categoryClass} isn't registered because this is almost always a bug. Throws an {@link NamedObjectNotFoundException} if the
* {@code categoryClass} is registered but the {@code name} isn't.
+ *
+ * @throws NamedObjectNotFoundException if the categoryClass or name is not registered
*/
public T parseNamedObject(Class categoryClass, String name, XContentParser parser, C context) throws IOException {
Map parsers = registry.get(categoryClass);
if (parsers == null) {
if (registry.isEmpty()) {
// The "empty" registry will never work so we throw a better exception as a hint.
- throw new ElasticsearchException("namedObject is not supported for this parser");
+ throw new NamedObjectNotFoundException("named objects are not supported for this parser");
}
- throw new ElasticsearchException("Unknown namedObject category [" + categoryClass.getName() + "]");
+ throw new NamedObjectNotFoundException("unknown named object category [" + categoryClass.getName() + "]");
}
Entry entry = parsers.get(name);
if (entry == null) {
- throw new UnknownNamedObjectException(parser.getTokenLocation(), categoryClass, name);
+ throw new NamedObjectNotFoundException(parser.getTokenLocation(), "unable to parse " + categoryClass.getSimpleName() +
+ " with name [" + name + "]: parser not found");
}
if (false == entry.name.match(name, parser.getDeprecationHandler())) {
/* Note that this shouldn't happen because we already looked up the entry using the names but we need to call `match` anyway
* because it is responsible for logging deprecation warnings. */
- throw new ParsingException(parser.getTokenLocation(),
- "Unknown " + categoryClass.getSimpleName() + " [" + name + "]: Parser didn't match");
+ throw new NamedObjectNotFoundException(parser.getTokenLocation(),
+ "unable to parse " + categoryClass.getSimpleName() + " with name [" + name + "]: parser didn't match");
}
return categoryClass.cast(entry.parser.parse(parser, context));
}
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
index f74bdec17a9f6..74542bb809f71 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
@@ -19,6 +19,8 @@
package org.elasticsearch.common.xcontent;
+import org.elasticsearch.common.Booleans;
+
import java.io.IOException;
import java.util.Map;
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ToXContentFragment.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ToXContentFragment.java
similarity index 100%
rename from server/src/main/java/org/elasticsearch/common/xcontent/ToXContentFragment.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ToXContentFragment.java
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java
similarity index 100%
rename from server/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java
similarity index 99%
rename from server/src/main/java/org/elasticsearch/common/xcontent/XContent.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java
index 6f6ee4ffdda54..1eaaac104f29d 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/XContent.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java
@@ -19,6 +19,8 @@
package org.elasticsearch.common.xcontent;
+import org.elasticsearch.common.Booleans;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
similarity index 85%
rename from server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index b51add28bf539..eae5e48a557f3 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -19,37 +19,32 @@
package org.elasticsearch.common.xcontent;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.CollectionUtils;
-import org.joda.time.DateTimeZone;
-import org.joda.time.ReadableInstant;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-
import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
+import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
+import java.util.GregorianCalendar;
import java.util.HashMap;
+import java.util.IdentityHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.ServiceLoader;
import java.util.Set;
+import java.util.function.Function;
/**
* A utility to build XContent (ie json).
*/
-public final class XContentBuilder implements Releasable, Flushable {
+public final class XContentBuilder implements Closeable, Flushable {
/**
* Create a new {@link XContentBuilder} using the given {@link XContent} content.
@@ -82,17 +77,15 @@ public static XContentBuilder builder(XContent xContent, Set includes, S
return new XContentBuilder(xContent, new ByteArrayOutputStream(), includes, excludes);
}
- public static final DateTimeFormatter DEFAULT_DATE_PRINTER = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC);
-
private static final Map, Writer> WRITERS;
private static final Map, HumanReadableTransformer> HUMAN_READABLE_TRANSFORMERS;
+ private static final Map, Function> DATE_TRANSFORMERS;
static {
Map, Writer> writers = new HashMap<>();
writers.put(Boolean.class, (b, v) -> b.value((Boolean) v));
writers.put(Byte.class, (b, v) -> b.value((Byte) v));
writers.put(byte[].class, (b, v) -> b.value((byte[]) v));
- writers.put(BytesRef.class, (b, v) -> b.binaryValue((BytesRef) v));
- writers.put(Date.class, (b, v) -> b.value((Date) v));
+ writers.put(Date.class, XContentBuilder::timeValue);
writers.put(Double.class, (b, v) -> b.value((Double) v));
writers.put(double[].class, (b, v) -> b.values((double[]) v));
writers.put(Float.class, (b, v) -> b.value((Float) v));
@@ -105,29 +98,40 @@ public static XContentBuilder builder(XContent xContent, Set includes, S
writers.put(short[].class, (b, v) -> b.values((short[]) v));
writers.put(String.class, (b, v) -> b.value((String) v));
writers.put(String[].class, (b, v) -> b.values((String[]) v));
+ writers.put(Locale.class, (b, v) -> b.value(v.toString()));
+ writers.put(Class.class, (b, v) -> b.value(v.toString()));
+ writers.put(ZonedDateTime.class, (b, v) -> b.value(v.toString()));
+ writers.put(Calendar.class, XContentBuilder::timeValue);
+ writers.put(GregorianCalendar.class, XContentBuilder::timeValue);
Map, HumanReadableTransformer> humanReadableTransformer = new HashMap<>();
- // These will be moved to a different class at a later time to decouple them from XContentBuilder
- humanReadableTransformer.put(TimeValue.class, v -> ((TimeValue) v).millis());
- humanReadableTransformer.put(ByteSizeValue.class, v -> ((ByteSizeValue) v).getBytes());
+ Map, Function> dateTransformers = new HashMap<>();
+
+ // treat strings as already converted
+ dateTransformers.put(String.class, Function.identity());
// Load pluggable extensions
for (XContentBuilderExtension service : ServiceLoader.load(XContentBuilderExtension.class)) {
Map, Writer> addlWriters = service.getXContentWriters();
Map, HumanReadableTransformer> addlTransformers = service.getXContentHumanReadableTransformers();
+ Map, Function> addlDateTransformers = service.getDateTransformers();
addlWriters.forEach((key, value) -> Objects.requireNonNull(value,
"invalid null xcontent writer for class " + key));
addlTransformers.forEach((key, value) -> Objects.requireNonNull(value,
"invalid null xcontent transformer for human readable class " + key));
+ dateTransformers.forEach((key, value) -> Objects.requireNonNull(value,
+ "invalid null xcontent date transformer for class " + key));
writers.putAll(addlWriters);
humanReadableTransformer.putAll(addlTransformers);
+ dateTransformers.putAll(addlDateTransformers);
}
WRITERS = Collections.unmodifiableMap(writers);
HUMAN_READABLE_TRANSFORMERS = Collections.unmodifiableMap(humanReadableTransformer);
+ DATE_TRANSFORMERS = Collections.unmodifiableMap(dateTransformers);
}
@FunctionalInterface
@@ -613,110 +617,63 @@ public XContentBuilder value(byte[] value, int offset, int length) throws IOExce
}
/**
- * Writes the binary content of the given {@link BytesRef}.
- *
- * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
- */
- public XContentBuilder field(String name, BytesRef value) throws IOException {
- return field(name).binaryValue(value);
- }
-
- /**
- * Writes the binary content of the given {@link BytesRef} as UTF-8 bytes.
+ * Writes the binary content of the given byte array as UTF-8 bytes.
*
* Use {@link XContentParser#charBuffer()} to read the value back
*/
- public XContentBuilder utf8Field(String name, BytesRef value) throws IOException {
- return field(name).utf8Value(value);
- }
-
- /**
- * Writes the binary content of the given {@link BytesRef}.
- *
- * Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
- */
- public XContentBuilder binaryValue(BytesRef value) throws IOException {
- if (value == null) {
- return nullValue();
- }
- value(value.bytes, value.offset, value.length);
+ public XContentBuilder utf8Value(byte[] bytes, int offset, int length) throws IOException {
+ generator.writeUTF8String(bytes, offset, length);
return this;
}
- /**
- * Writes the binary content of the given {@link BytesRef} as UTF-8 bytes.
- *
- * Use {@link XContentParser#charBuffer()} to read the value back
- */
- public XContentBuilder utf8Value(BytesRef value) throws IOException {
- if (value == null) {
- return nullValue();
- }
- generator.writeUTF8String(value.bytes, value.offset, value.length);
- return this;
- }
////////////////////////////////////////////////////////////////////////////
// Date
//////////////////////////////////
- public XContentBuilder field(String name, ReadableInstant value) throws IOException {
- return field(name).value(value);
- }
-
- public XContentBuilder field(String name, ReadableInstant value, DateTimeFormatter formatter) throws IOException {
- return field(name).value(value, formatter);
- }
-
- public XContentBuilder value(ReadableInstant value) throws IOException {
- return value(value, DEFAULT_DATE_PRINTER);
- }
-
- public XContentBuilder value(ReadableInstant value, DateTimeFormatter formatter) throws IOException {
- if (value == null) {
- return nullValue();
- }
- ensureFormatterNotNull(formatter);
- return value(formatter.print(value));
- }
-
- public XContentBuilder field(String name, Date value) throws IOException {
- return field(name).value(value);
- }
-
- public XContentBuilder field(String name, Date value, DateTimeFormatter formatter) throws IOException {
- return field(name).value(value, formatter);
- }
-
- public XContentBuilder value(Date value) throws IOException {
- return value(value, DEFAULT_DATE_PRINTER);
- }
+ /**
+ * Write a time-based field and value, if the passed timeValue is null a
+ * null value is written, otherwise a date transformers lookup is performed.
- public XContentBuilder value(Date value, DateTimeFormatter formatter) throws IOException {
- if (value == null) {
- return nullValue();
- }
- return value(formatter, value.getTime());
+ * @throws IllegalArgumentException if there is no transformers for the type of object
+ */
+ public XContentBuilder timeField(String name, Object timeValue) throws IOException {
+ return field(name).timeValue(timeValue);
}
- public XContentBuilder dateField(String name, String readableName, long value) throws IOException {
+ /**
+ * If the {@code humanReadable} flag is set, writes both a formatted and
+ * unformatted version of the time value using the date transformer for the
+ * {@link Long} class.
+ */
+ public XContentBuilder timeField(String name, String readableName, long value) throws IOException {
if (humanReadable) {
- field(readableName).value(DEFAULT_DATE_PRINTER, value);
+ Function longTransformer = DATE_TRANSFORMERS.get(Long.class);
+ if (longTransformer == null) {
+ throw new IllegalArgumentException("cannot write time value xcontent for unknown value of type Long");
+ }
+ field(readableName).value(longTransformer.apply(value));
}
field(name, value);
return this;
}
- XContentBuilder value(Calendar value) throws IOException {
- if (value == null) {
+ /**
+ * Write a time-based value, if the value is null a null value is written,
+ * otherwise a date transformers lookup is performed.
+
+ * @throws IllegalArgumentException if there is no transformers for the type of object
+ */
+ public XContentBuilder timeValue(Object timeValue) throws IOException {
+ if (timeValue == null) {
return nullValue();
+ } else {
+ Function transformer = DATE_TRANSFORMERS.get(timeValue.getClass());
+ if (transformer == null) {
+ throw new IllegalArgumentException("cannot write time value xcontent for unknown value of type " + timeValue.getClass());
+ }
+ return value(transformer.apply(timeValue));
}
- return value(DEFAULT_DATE_PRINTER, value.getTimeInMillis());
- }
-
- XContentBuilder value(DateTimeFormatter formatter, long value) throws IOException {
- ensureFormatterNotNull(formatter);
- return value(formatter.print(value));
}
////////////////////////////////////////////////////////////////////////////
@@ -782,21 +739,20 @@ private void unknownValue(Object value, boolean ensureNoSelfReferences) throws I
//Path implements Iterable and causes endless recursion and a StackOverFlow if treated as an Iterable here
value((Path) value);
} else if (value instanceof Map) {
- map((Map) value, ensureNoSelfReferences);
+ @SuppressWarnings("unchecked")
+ final Map valueMap = (Map) value;
+ map(valueMap, ensureNoSelfReferences);
} else if (value instanceof Iterable) {
value((Iterable) value, ensureNoSelfReferences);
} else if (value instanceof Object[]) {
values((Object[]) value, ensureNoSelfReferences);
- } else if (value instanceof Calendar) {
- value((Calendar) value);
- } else if (value instanceof ReadableInstant) {
- value((ReadableInstant) value);
} else if (value instanceof ToXContent) {
value((ToXContent) value);
- } else {
- // This is a "value" object (like enum, DistanceUnit, etc) just toString() it
- // (yes, it can be misleading when toString a Java class, but really, jackson should be used in that case)
+ } else if (value instanceof Enum) {
+ // Write out the Enum toString
value(Objects.toString(value));
+ } else {
+ throw new IllegalArgumentException("cannot write xcontent for unknown value of type " + value.getClass());
}
}
@@ -844,7 +800,7 @@ private XContentBuilder map(Map values, boolean ensureNoSelfReference
// checks that the map does not contain references to itself because
// iterating over map entries will cause a stackoverflow error
if (ensureNoSelfReferences) {
- CollectionUtils.ensureNoSelfReferences(values);
+ ensureNoSelfReferences(values);
}
startObject();
@@ -873,7 +829,7 @@ private XContentBuilder value(Iterable values, boolean ensureNoSelfReferences
// checks that the iterable does not contain references to itself because
// iterating over entries will cause a stackoverflow error
if (ensureNoSelfReferences) {
- CollectionUtils.ensureNoSelfReferences(values);
+ ensureNoSelfReferences(values);
}
startArray();
for (Object value : values) {
@@ -920,14 +876,6 @@ public XContentBuilder percentageField(String rawFieldName, String readableField
return this;
}
- public XContentBuilder byteSizeField(String rawFieldName, String readableFieldName, long rawSize) throws IOException {
- if (humanReadable) {
- field(readableFieldName, new ByteSizeValue(rawSize).toString());
- }
- field(rawFieldName, rawSize);
- return this;
- }
-
////////////////////////////////////////////////////////////////////////////
// Raw fields
//////////////////////////////////
@@ -985,13 +933,44 @@ static void ensureNameNotNull(String name) {
ensureNotNull(name, "Field name cannot be null");
}
- static void ensureFormatterNotNull(DateTimeFormatter formatter) {
- ensureNotNull(formatter, "DateTimeFormatter cannot be null");
- }
-
static void ensureNotNull(Object value, String message) {
if (value == null) {
throw new IllegalArgumentException(message);
}
}
+
+ private static void ensureNoSelfReferences(Object value) {
+ Iterable it = convert(value);
+ if (it != null) {
+ ensureNoSelfReferences(it, value, Collections.newSetFromMap(new IdentityHashMap<>()));
+ }
+ }
+
+ private static Iterable convert(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Map) {
+ return ((Map) value).values();
+ } else if ((value instanceof Iterable) && (value instanceof Path == false)) {
+ return (Iterable) value;
+ } else if (value instanceof Object[]) {
+ return Arrays.asList((Object[]) value);
+ } else {
+ return null;
+ }
+ }
+
+ private static void ensureNoSelfReferences(final Iterable value, Object originalReference, final Set