Merge branch 'main' into retention-leases-not-null

.buildkite/pipelines/dra-workflow.yml

@@ -2,6 +2,7 @@ steps:
   - command: .buildkite/scripts/dra-workflow.sh
     env:
       USE_DRA_CREDENTIALS: "true"
+      USE_PROD_DOCKER_CREDENTIALS: "true"
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2204
@@ -18,4 +19,5 @@ steps:
       branch: "${BUILDKITE_BRANCH}"
       env:
         DRA_WORKFLOW: staging
+        USE_PROD_DOCKER_CREDENTIALS: "true"
     if: build.env('DRA_WORKFLOW') == 'staging'

.buildkite/pipelines/periodic-packaging.template.yml

@@ -27,7 +27,8 @@ steps:
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
           machineType: n1-standard-8
-        env: {}
+        env:
+          USE_PROD_DOCKER_CREDENTIALS: "true"
   - group: packaging-tests-upgrade
     steps: $BWC_STEPS
   - group: packaging-tests-windows

.buildkite/pipelines/periodic-packaging.yml

@@ -28,7 +28,8 @@ steps:
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
           machineType: n1-standard-8
-        env: {}
+        env:
+          USE_PROD_DOCKER_CREDENTIALS: "true"
   - group: packaging-tests-upgrade
     steps:
       - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade"

.buildkite/pipelines/pull-request/packaging-tests-unix-sample.yml

@@ -24,4 +24,5 @@ steps:
           diskSizeGb: 350
           machineType: custom-16-32768
         env:
+          USE_PROD_DOCKER_CREDENTIALS: "true"
           PACKAGING_TASK: "{{matrix.PACKAGING_TASK}}"

build-tools-internal/build.gradle

@@ -384,6 +384,15 @@ tasks.named("jar") {
   exclude("classpath.index")
 }
 
+spotless {
+  java {
+    // IDEs can sometimes run annotation processors that leave files in
+    // here, causing Spotless to complain. Even though this path ought not
+    // to exist, exclude it anyway in order to avoid spurious failures.
+    toggleOffOn()
+  }
+}
+
 def resolveMainWrapperVersion() {
   new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim()
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -29,11 +29,13 @@ public enum DockerBase {
     CLOUD_ESS(null, "-cloud-ess", "apt-get"),
 
     // Chainguard based wolfi image with latest jdk
-    WOLFI(
-        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
+    // This is usually updated via renovatebot
+    // spotless:off
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
         "-wolfi",
         "apk"
     ),
+    // spotless:on
 
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the Wolfi image.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -163,7 +163,13 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                     // So, if you change this such that the artifact will have a slash/directory in it, you'll need to update the logic
                     // below as well
                     pb.directory(uploadFileDir);
-                    pb.start().waitFor();
+                    try {
+                        // we are very generious here, as the upload can take
+                        // a long time depending on its size
+                        pb.start().waitFor(30, java.util.concurrent.TimeUnit.MINUTES);
+                    } catch (InterruptedException e) {
+                        System.out.println("Failed to upload buildkite artifact " + e.getMessage());
+                    }
 
                     System.out.println("Generating buildscan link for artifact...");
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -185,8 +185,8 @@ public void execute(Task t) {
             });
 
             if (OS.current().equals(OS.WINDOWS) && System.getProperty("tests.timeoutSuite") == null) {
-                // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man
-                test.systemProperty("tests.timeoutSuite", "2400000!");
+                // override the suite timeout to 60 mins for windows, because it has the most inefficient filesystem known to man
+                test.systemProperty("tests.timeoutSuite", "3600000!");
             }
 
             /*

build-tools-internal/src/main/resources/changelog-schema.json

@@ -284,6 +284,7 @@
         "Cluster and node setting",
         "Command line tool",
         "CRUD",
+        "ES|QL",
         "Index setting",
         "Ingest",
         "JVM option",

distribution/tools/entitlement-agent/README.md

@@ -1,6 +1,6 @@
 ### Entitlement Agent
 
-This is a java agent that instruments sensitive class library methods with calls into the `entitlement-runtime` module to check for permissions granted under the _entitlements_ system.
+This is a java agent that instruments sensitive class library methods with calls into the `entitlement-bridge` module to check for permissions granted under the _entitlements_ system.
 
 The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
 With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers.

distribution/tools/entitlement-agent/build.gradle

@@ -7,21 +7,44 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import static java.util.stream.Collectors.joining
+
 apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.embedded-providers'
+
+embeddedProviders {
+  impl 'entitlement-agent', project(':distribution:tools:entitlement-agent:impl')
+}
 
 configurations {
-  entitlementRuntime
+  entitlementBridge
 }
 
 dependencies {
-  entitlementRuntime project(":libs:elasticsearch-entitlement-runtime")
-  implementation project(":libs:elasticsearch-entitlement-runtime")
+  entitlementBridge project(":distribution:tools:entitlement-bridge")
+  compileOnly project(":libs:elasticsearch-core")
+  compileOnly project(":distribution:tools:entitlement-runtime")
   testImplementation project(":test:framework")
+  testImplementation project(":distribution:tools:entitlement-bridge")
+  testImplementation project(":distribution:tools:entitlement-agent:impl")
 }
 
 tasks.named('test').configure {
+  systemProperty "tests.security.manager", "false"
   dependsOn('jar')
-  jvmArgs "-javaagent:${ tasks.named('jar').flatMap{ it.archiveFile }.get()}"
+
+  // Register an argument provider to avoid eager resolution of configurations
+  jvmArgumentProviders.add(new CommandLineArgumentProvider() {
+    @Override
+    Iterable<String> asArguments() {
+      return ["-javaagent:${tasks.jar.archiveFile.get()}", "-Des.entitlements.bridgeJar=${configurations.entitlementBridge.singleFile}"]
+    }
+  })
+
+
+  // The Elasticsearch build plugin automatically adds all compileOnly deps as testImplementation.
+  // We must not add the bridge this way because it is also on the boot classpath, and that would lead to jar hell.
+  classpath -= files(configurations.entitlementBridge)
 }
 
 tasks.named('jar').configure {

distribution/tools/entitlement-agent/impl/build.gradle

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+apply plugin: 'elasticsearch.build'
+
+dependencies {
+  compileOnly project(':distribution:tools:entitlement-agent')
+  implementation 'org.ow2.asm:asm:9.7'
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+

distribution/tools/entitlement-agent/impl/licenses/asm-LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright (c) 2012 France Télécom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.

distribution/tools/entitlement-agent/impl/licenses/asm-NOTICE.txt

@@ -0,0 +1 @@
+

distribution/tools/entitlement-agent/impl/src/main/java/module-info.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import org.elasticsearch.entitlement.instrumentation.InstrumentationService;
+import org.elasticsearch.entitlement.instrumentation.impl.InstrumentationServiceImpl;
+
+module org.elasticsearch.entitlement.agent.impl {
+    requires org.objectweb.asm;
+    requires org.elasticsearch.entitlement.agent;
+
+    provides InstrumentationService with InstrumentationServiceImpl;
+}

distribution/tools/entitlement-agent/impl/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.instrumentation.impl;
+
+import org.elasticsearch.entitlement.instrumentation.InstrumentationService;
+import org.elasticsearch.entitlement.instrumentation.Instrumenter;
+import org.elasticsearch.entitlement.instrumentation.MethodKey;
+import org.objectweb.asm.Type;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Map;
+import java.util.stream.Stream;
+
+public class InstrumentationServiceImpl implements InstrumentationService {
+    @Override
+    public Instrumenter newInstrumenter(String classNameSuffix, Map<MethodKey, Method> instrumentationMethods) {
+        return new InstrumenterImpl(classNameSuffix, instrumentationMethods);
+    }
+
+    /**
+     * @return a {@link MethodKey} suitable for looking up the given {@code targetMethod} in the entitlements trampoline
+     */
+    public MethodKey methodKeyForTarget(Method targetMethod) {
+        Type actualType = Type.getMethodType(Type.getMethodDescriptor(targetMethod));
+        return new MethodKey(
+            Type.getInternalName(targetMethod.getDeclaringClass()),
+            targetMethod.getName(),
+            Stream.of(actualType.getArgumentTypes()).map(Type::getInternalName).toList(),
+            Modifier.isStatic(targetMethod.getModifiers())
+        );
+    }
+
+}