Merge remote-tracking branch 'elastic/master' into ccr-replicate-aliases

* elastic/master: (84 commits)
  [ML] adds geo_centroid aggregation support to data frames (elastic#42088)
  Add documentation for calendar/fixed intervals (elastic#41919)
  Remove global checkpoint assertion in peer recovery (elastic#41987)
  Don't create tempdir for cli scripts (elastic#41913)
  Fix debian-8 update (elastic#42056)
  Cleanup plugin bin directories (elastic#41907)
  Prevent order being lost for _nodes API filters (elastic#42045)
  Change IndexAnalyzers default analyzer access (elastic#42011)
  Remove reference to fs.data.spins in docs
  Mute failing AsyncTwoPhaseIndexerTests
  Remove close method in PageCacheRecycler/Recycler (elastic#41917)
  [ML] adding pivot.max_search_page_size option for setting paging size (elastic#41920)
  Docs: Tweak list formatting
  Simplify handling of keyword field normalizers (elastic#42002)
  [ML] properly nesting objects in document source (elastic#41901)
  Remove extra `ms` from log message (elastic#42068)
  Increase the sample space for random inner hits name generator (elastic#42057)
  Recognise direct buffers in heap size docs (elastic#42070)
  shouldRollGeneration should execute under read lock (elastic#41696)
  Wait for active shard after close in mixed cluster (elastic#42029)
  ...

.ci/matrix-runtime-javas.yml

@@ -11,3 +11,4 @@ ES_RUNTIME_JAVA:
   - openjdk12
   - zulu11
   - zulu12
+  - corretto11

Vagrantfile

@@ -67,7 +67,10 @@ Vagrant.configure(2) do |config|
   'debian-8'.tap do |box|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/debian-8-x86_64'
-      deb_common config, box
+      deb_common config, box, extra: <<-SHELL
+        # this sometimes gets a bad ip, and doesn't appear to be needed
+        rm /etc/apt/sources.list.d/http_debian_net_debian.list
+      SHELL
     end
   end
   'debian-9'.tap do |box|
@@ -158,13 +161,17 @@ def deb_common(config, name, extra: '')
       s.privileged = false
       s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile"
   end
+  extra_with_lintian = <<-SHELL
+    #{extra}
+    install lintian
+  SHELL
   linux_common(
     config,
     name,
     update_command: 'apt-get update',
     update_tracking_file: '/var/cache/apt/archives/last_update',
     install_command: 'apt-get install -y',
-    extra: extra
+    extra: extra_with_lintian
   )
 end
 

buildSrc/build.gradle

@@ -124,6 +124,7 @@ dependencies {
   compile 'com.avast.gradle:gradle-docker-compose-plugin:0.8.12'
   testCompile "junit:junit:${props.getProperty('junit')}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+  testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
 }
 
 /*****************************************************************************

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -34,6 +34,9 @@ import org.gradle.api.plugins.quality.Checkstyle
 class PrecommitTasks {
 
     /** Adds a precommit task, which depends on non-test verification tasks. */
+
+    public static final String CHECKSTYLE_VERSION = '8.20'
+
     public static Task create(Project project, boolean includeDependencyLicenses) {
         project.configurations.create("forbiddenApisCliJar")
         project.dependencies {
@@ -213,7 +216,7 @@ class PrecommitTasks {
             configProperties = [
                 suppressions: checkstyleSuppressions
             ]
-            toolVersion = '8.10.1'
+            toolVersion = CHECKSTYLE_VERSION
         }
 
         project.tasks.withType(Checkstyle) { task ->

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -70,7 +70,7 @@ class RestIntegTestTask extends DefaultTask {
             project.testClusters {
                 "$name" {
                     distribution = 'INTEG_TEST'
-                    version = project.version
+                    version = VersionProperties.elasticsearch
                     javaHome = project.file(project.ext.runtimeJavaHome)
                 }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -30,7 +30,7 @@ class VagrantTestPlugin implements Plugin<Project> {
             'oel-6',
             'oel-7',
             'opensuse-42',
-            'rhel-8',
+            /* TODO: need a real RHEL license now that it is out of beta 'rhel-8',*/
             'sles-12',
             'ubuntu-1604',
             'ubuntu-1804'

buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Buildable;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.TaskDependency;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class Jdk implements Buildable, Iterable<File> {
+
+    static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+)(@([a-f0-9]{32}))?");
+    private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("linux", "windows", "darwin"));
+
+    private final String name;
+    private final Configuration configuration;
+
+    private final Property<String> version;
+    private final Property<String> platform;
+
+
+    Jdk(String name, Project project) {
+        this.name = name;
+        this.configuration = project.getConfigurations().create("jdk_" + name);
+        this.version = project.getObjects().property(String.class);
+        this.platform = project.getObjects().property(String.class);
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getVersion() {
+        return version.get();
+    }
+
+    public void setVersion(String version) {
+        if (VERSION_PATTERN.matcher(version).matches() == false) {
+            throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
+        }
+        this.version.set(version);
+    }
+
+    public String getPlatform() {
+        return platform.get();
+    }
+
+    public void setPlatform(String platform) {
+        if (ALLOWED_PLATFORMS.contains(platform) == false) {
+            throw new IllegalArgumentException(
+                "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS);
+        }
+        this.platform.set(platform);
+    }
+
+    // pkg private, for internal use
+    Configuration getConfiguration() {
+        return configuration;
+    }
+
+    @Override
+    public String toString() {
+        return configuration.getSingleFile().toString();
+    }
+
+    @Override
+    public TaskDependency getBuildDependencies() {
+        return configuration.getBuildDependencies();
+    }
+
+    // internal, make this jdks configuration unmodifiable
+    void finalizeValues() {
+        if (version.isPresent() == false) {
+            throw new IllegalArgumentException("version not specified for jdk [" + name + "]");
+        }
+        if (platform.isPresent() == false) {
+            throw new IllegalArgumentException("platform not specified for jdk [" + name + "]");
+        }
+        version.finalizeValue();
+        platform.finalizeValue();
+    }
+
+    @Override
+    public Iterator<File> iterator() {
+        return configuration.iterator();
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

@@ -0,0 +1,184 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Action;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.UnknownTaskException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ConfigurationContainer;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.artifacts.dsl.RepositoryHandler;
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
+import org.gradle.api.file.CopySpec;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.file.RelativePath;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.TaskProvider;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+
+public class JdkDownloadPlugin implements Plugin<Project> {
+
+    private static final String REPO_NAME_PREFIX = "jdk_repo_";
+
+    @Override
+    public void apply(Project project) {
+        NamedDomainObjectContainer<Jdk> jdksContainer = project.container(Jdk.class, name ->
+            new Jdk(name, project)
+        );
+        project.getExtensions().add("jdks", jdksContainer);
+
+        project.afterEvaluate(p -> {
+            for (Jdk jdk : jdksContainer) {
+                jdk.finalizeValues();
+                String version = jdk.getVersion();
+                String platform = jdk.getPlatform();
+
+                // depend on the jdk directory "artifact" from the root project
+                DependencyHandler dependencies = project.getDependencies();
+                Map<String, Object> depConfig = new HashMap<>();
+                depConfig.put("path", ":"); // root project
+                depConfig.put("configuration", configName("extracted_jdk", version, platform));
+                dependencies.add(jdk.getConfiguration().getName(), dependencies.project(depConfig));
+
+                // ensure a root level jdk download task exists
+                setupRootJdkDownload(project.getRootProject(), platform, version);
+            }
+        });
+
+        // all other repos should ignore the special jdk artifacts
+        project.getRootProject().getRepositories().all(repo -> {
+            if (repo.getName().startsWith(REPO_NAME_PREFIX) == false) {
+                repo.content(content -> content.excludeGroup("jdk"));
+            }
+        });
+    }
+
+    private static void setupRootJdkDownload(Project rootProject, String platform, String version) {
+        String extractTaskName = "extract" + capitalize(platform) + "Jdk" + version;
+        // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task
+        try {
+            rootProject.getTasks().named(extractTaskName);
+            // already setup this version
+            return;
+        } catch (UnknownTaskException e) {
+            // fall through: register the task
+        }
+
+        // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build]
+        // Note the "patch" version is not yet handled here, as it has not yet been used by java.
+        Matcher jdkVersionMatcher = Jdk.VERSION_PATTERN.matcher(version);
+        if (jdkVersionMatcher.matches() == false) {
+            throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+        }
+        String jdkVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
+        String jdkMajor = jdkVersionMatcher.group(1);
+        String jdkBuild = jdkVersionMatcher.group(3);
+        String hash = jdkVersionMatcher.group(5);
+
+        // add fake ivy repo for jdk url
+        String repoName = REPO_NAME_PREFIX + version;
+        RepositoryHandler repositories = rootProject.getRepositories();
+        if (rootProject.getRepositories().findByName(repoName) == null) {
+            if (hash != null) {
+                // current pattern since 12.0.1
+                repositories.ivy(ivyRepo -> {
+                    ivyRepo.setName(repoName);
+                    ivyRepo.setUrl("https://download.oracle.com");
+                    ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+                    ivyRepo.patternLayout(layout -> layout.artifact(
+                        "java/GA/jdk" + jdkVersion + "/" + hash + "/" + jdkBuild + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"));
+                    ivyRepo.content(content -> content.includeGroup("jdk"));
+                });
+            } else {
+                // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
+                repositories.ivy(ivyRepo -> {
+                    ivyRepo.setName(repoName);
+                    ivyRepo.setUrl("https://download.oracle.com");
+                    ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+                    ivyRepo.patternLayout(layout ->
+                        layout.artifact("java/GA/jdk" + jdkMajor + "/" + jdkBuild + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"));
+                    ivyRepo.content(content -> content.includeGroup("jdk"));
+                });
+            }
+        }
+
+        // add the jdk as a "dependency"
+        final ConfigurationContainer configurations = rootProject.getConfigurations();
+        String remoteConfigName = configName("openjdk", version, platform);
+        String localConfigName = configName("extracted_jdk", version, platform);
+        Configuration jdkConfig = configurations.findByName(remoteConfigName);
+        if (jdkConfig == null) {
+            jdkConfig = configurations.create(remoteConfigName);
+            configurations.create(localConfigName);
+        }
+        String extension = platform.equals("windows") ? "zip" : "tar.gz";
+        String jdkDep = "jdk:" + (platform.equals("darwin") ? "osx" : platform) + ":" + jdkVersion + "@" + extension;
+        rootProject.getDependencies().add(configName("openjdk", version, platform), jdkDep);
+
+        // add task for extraction
+        // TODO: look into doing this as an artifact transform, which are cacheable starting in gradle 5.3
+        int rootNdx = platform.equals("darwin") ? 2 : 1;
+        Action<CopySpec> removeRootDir = copy -> {
+            // remove extra unnecessary directory levels
+            copy.eachFile(details -> {
+                String[] pathSegments = details.getRelativePath().getSegments();
+                String[] newPathSegments = Arrays.copyOfRange(pathSegments, rootNdx, pathSegments.length);
+                details.setRelativePath(new RelativePath(true, newPathSegments));
+            });
+            copy.setIncludeEmptyDirs(false);
+        };
+        // delay resolving jdkConfig until runtime
+        Supplier<File> jdkArchiveGetter = jdkConfig::getSingleFile;
+        final Callable<FileTree> fileGetter;
+        if (extension.equals("zip")) {
+            fileGetter = () -> rootProject.zipTree(jdkArchiveGetter.get());
+        } else {
+            fileGetter = () -> rootProject.tarTree(rootProject.getResources().gzip(jdkArchiveGetter.get()));
+        }
+        String extractDir = rootProject.getBuildDir().toPath().resolve("jdks/openjdk-" + jdkVersion + "_" + platform).toString();
+        TaskProvider<Copy> extractTask = rootProject.getTasks().register(extractTaskName, Copy.class, copyTask -> {
+            copyTask.doFirst(t -> rootProject.delete(extractDir));
+            copyTask.into(extractDir);
+            copyTask.from(fileGetter, removeRootDir);
+        });
+        rootProject.getArtifacts().add(localConfigName,
+            rootProject.getLayout().getProjectDirectory().dir(extractDir),
+            artifact -> artifact.builtBy(extractTask));
+    }
+
+    private static String configName(String prefix, String version, String platform) {
+        return prefix + "_" + version + "_" + platform;
+    }
+
+    private static String capitalize(String s) {
+        return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java

@@ -123,8 +123,7 @@ public boolean wait(int durationInMs) throws GeneralSecurityException, Interrupt
             if (System.nanoTime() < waitUntil) {
                 Thread.sleep(sleep);
             } else {
-                logger.error("Failed to access url [{}]", url, failure);
-                return false;
+                throw failure;
             }
         }
     }