Merge branch 'main' into refactor-extensions-yml

.github/ISSUE_TEMPLATE/failed_check.md

@@ -1,6 +1,6 @@
 ---
-title: Gradle Check Failure.
-labels: >test-failure bug
+title: '[AUTOCUT] Gradle Check Failure.'
+labels: '>test-failure, bug'
 ---
 
 A gradle check workflow has failed after merge.

CHANGELOG.md

@@ -5,12 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 3.0]
 ### Added
-- Hardened token permissions in GitHub workflows ([#4587](https://github.com/opensearch-project/OpenSearch/pull/4587))
 - Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
 - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add support for ppc64le architecture ([#5459](https://github.com/opensearch-project/OpenSearch/pull/5459))
-- Add support to disallow search request with preference parameter with strict weighted shard routing([#5874](https://github.com/opensearch-project/OpenSearch/pull/5874))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
@@ -38,6 +36,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps `wiremock-jre8-standalone` from 2.33.2 to 2.35.0
 - Bumps `gson` from 2.10 to 2.10.1
 - Bumps `json-schema-validator` from 1.0.73 to 1.0.76
+- Bumps `jna` from 5.11.0 to 5.13.0
+- Bumps `joni` from 2.1.44 to 2.1.45
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -79,23 +79,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add query for initialized extensions ([#5658](https://github.com/opensearch-project/OpenSearch/pull/5658))
 - Add update-index-settings allowlist for searchable snapshot ([#5907](https://github.com/opensearch-project/OpenSearch/pull/5907))
 - Replace latches with CompletableFutures for extensions ([#5646](https://github.com/opensearch-project/OpenSearch/pull/5646))
-- Add GeoTile and GeoHash Grid aggregations on GeoShapes ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589))
+- Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589))
+- Add support to disallow search request with preference parameter with strict weighted shard routing([#5874](https://github.com/opensearch-project/OpenSearch/pull/5874))
+- Changing ExtensionActionRequest streaminput constructor to be public ([#6094](https://github.com/opensearch-project/OpenSearch/pull/6094))
 - Adds support for minimum compatible version for extensions ([#6003](https://github.com/opensearch-project/OpenSearch/pull/6003))
 
 ### Dependencies
 - Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704))
 - Bumps `reactor-netty` from 1.1.1 to 1.1.2 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/5878))
+- OpenJDK Update (January 2023 Patch releases) ([#6074](https://github.com/opensearch-project/OpenSearch/pull/6074))
+- Bumps `Mockito` from 4.7.0 to 5.1.0, `ByteBuddy` from 1.12.18 to 1.12.22 ([#6076](https://github.com/opensearch-project/OpenSearch/pull/6076))
+- Bumps `joda` from 2.10.13 to 2.12.2 ([#6083](https://github.com/opensearch-project/OpenSearch/pull/6083))
+- Upgrade to Lucene 9.5.0 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/6078))
 
 ### Changed
 - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725))
 - [Refactor] Use local opensearch.common.SetOnce instead of lucene's utility class ([#5947](https://github.com/opensearch-project/OpenSearch/pull/5947))
+- Cluster health call to throw decommissioned exception for local decommissioned node([#6008](https://github.com/opensearch-project/OpenSearch/pull/6008))
+- [Refactor] core.common to new opensearch-common library ([#5976](https://github.com/opensearch-project/OpenSearch/pull/5976))
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
 - [Segment Replication] Fix for peer recovery ([#5344](https://github.com/opensearch-project/OpenSearch/pull/5344))
+- Fix weighted shard routing state across search requests([#6004](https://github.com/opensearch-project/OpenSearch/pull/6004))
 
 ### Security
 

buildSrc/build.gradle

@@ -110,7 +110,7 @@ dependencies {
   api 'com.netflix.nebula:gradle-info-plugin:12.0.0'
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.7'
-  api "net.java.dev.jna:jna:5.11.0"
+  api "net.java.dev.jna:jna:5.13.0"
   api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
@@ -191,6 +191,7 @@ if (project != rootProject) {
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:windows-zip')
 
+    integTestRuntimeOnly(project(":libs:opensearch-common"))
     integTestRuntimeOnly(project(":libs:opensearch-core"))
   }
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/JarHellPrecommitPlugin.java

@@ -44,11 +44,11 @@ public class JarHellPrecommitPlugin extends PrecommitPlugin {
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {
         Configuration jarHellConfig = project.getConfigurations().create("jarHell");
-        if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) {
+        if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-common") == false) {
             // External plugins will depend on this already via transitive dependencies.
             // Internal projects are not all plugins, so make sure the check is available
             // we are not doing this for this project itself to avoid jar hell with itself
-            project.getDependencies().add("jarHell", project.project(":libs:opensearch-core"));
+            project.getDependencies().add("jarHell", project.project(":libs:opensearch-common"));
         }
 
         TaskProvider<JarHellTask> jarHell = project.getTasks().register("jarHell", JarHellTask.class);

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -75,9 +75,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.17+8";
+    private static final String SYSTEM_JDK_VERSION = "11.0.18+10";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.5+8";
+    private static final String GRADLE_JDK_VERSION = "17.0.6+10";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -1,8 +1,8 @@
 opensearch        = 3.0.0
-lucene            = 9.5.0-snapshot-0878271
+lucene            = 9.5.0
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 19.0.1+10
+bundled_jdk = 19.0.2+7
 
 
 # optional dependencies
@@ -26,8 +26,8 @@ antlr4            = 4.11.1
 jna               = 5.5.0
 
 netty             = 4.1.86.Final
-joda              = 2.10.13
-
+joda              = 2.12.2
+    
 # client dependencies
 httpclient5       = 5.1.4
 httpcore5         = 5.1.5
@@ -48,10 +48,9 @@ bouncycastle=1.70
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
 hamcrest          = 2.1
-# Update to 4.8.0 is using reflection without SecurityManager checks (fails with java.security.AccessControlException)
-mockito           = 4.7.0
+mockito           = 5.1.0
 objenesis         = 3.2
-bytebuddy         = 1.12.18
+bytebuddy         = 1.12.22
 
 # benchmark dependencies
 jmh               = 1.35

client/rest/build.gradle

@@ -54,6 +54,7 @@ dependencies {
   testImplementation "org.mockito:mockito-core:${versions.mockito}"
   testImplementation "org.objenesis:objenesis:${versions.objenesis}"
   testImplementation "net.bytebuddy:byte-buddy:${versions.bytebuddy}"
+  testImplementation "net.bytebuddy:byte-buddy-agent:${versions.bytebuddy}"
   testImplementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   testImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
   testImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"

client/sniffer/build.gradle

@@ -50,6 +50,7 @@ dependencies {
   testImplementation "org.mockito:mockito-core:${versions.mockito}"
   testImplementation "org.objenesis:objenesis:${versions.objenesis}"
   testImplementation "net.bytebuddy:byte-buddy:${versions.bytebuddy}"
+  testImplementation "net.bytebuddy:byte-buddy-agent:${versions.bytebuddy}"
 }
 
 tasks.named('forbiddenApisMain').configure {

gradle/missing-javadoc.gradle

@@ -96,7 +96,6 @@ configure([
   project(":client:rest-high-level"),
   project(":client:test"),
   project(":libs:opensearch-cli"),
-  project(":libs:opensearch-core"),
   project(":libs:opensearch-dissect"),
   project(":libs:opensearch-geo"),
   project(":libs:opensearch-grok"),
@@ -162,7 +161,11 @@ configure([
   }
 }
 
-configure(project(":server")) {
+configure([
+  project(":libs:opensearch-common"),
+  project(":libs:opensearch-core"),
+  project(":server")
+]) {
   project.tasks.withType(MissingJavadocTask) {
     // TODO: bump to variable missing level after increasing javadoc coverage
     javadocMissingLevel = "class"

libs/build.gradle

@@ -44,11 +44,12 @@ subprojects {
       dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
         Project depProject = dep.dependencyProject
         if (depProject != null
-          && false == depProject.path.equals(':libs:opensearch-core')
+          && (false == depProject.path.equals(':libs:opensearch-core') &&
+              false == depProject.path.equals(':libs:opensearch-common'))
           && depProject.path.startsWith(':libs')) {
           throw new InvalidUserDataException("projects in :libs "
             + "may not depend on other projects libs except "
-            + ":libs:opensearch-core but "
+            + ":libs:opensearch-core or :libs:opensearch-common but "
             + "${project.path} depends on ${depProject.path}")
         }
       }

libs/cli/build.gradle

@@ -33,6 +33,7 @@ apply plugin: 'opensearch.publish'
 
 dependencies {
   api 'net.sf.jopt-simple:jopt-simple:5.0.4'
+  api project(':libs:opensearch-common')
   api project(':libs:opensearch-core')
 }
 

libs/common/build.gradle

@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import org.opensearch.gradle.info.BuildParams
+
+apply plugin: 'opensearch.publish'
+
+archivesBaseName = 'opensearch-common'
+
+dependencies {
+  // This dependency is used only by :libs:core for null-checking interop with other tools
+  compileOnly "com.google.code.findbugs:jsr305:3.0.2"
+
+  testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.opensearch', module: 'opensearch-common'
+  }
+}
+
+tasks.named('forbiddenApisMain').configure {
+  // :libs:opensearch-common does not depend on server
+  // TODO: Need to decide how we want to handle for forbidden signatures with the changes to server
+  replaceSignatureFiles 'jdk-signatures'
+}

libs/core/src/main/java/org/opensearch/bootstrap/JarHell.java → libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java

@@ -71,6 +71,8 @@
  *   <li>Checks any {@code X-Compile-OpenSearch-Version} value in
  *       the jar manifest is compatible with the current ES</li>
  * </ul>
+ *
+ * @opensearch.internal
  */
 public class JarHell {
 

libs/core/src/main/java/org/opensearch/bootstrap/JdkJarHellCheck.java → libs/common/src/main/java/org/opensearch/bootstrap/JdkJarHellCheck.java

@@ -44,6 +44,11 @@
 import java.util.HashSet;
 import java.util.Set;
 
+/**
+ * Checks for jdk jar hell
+ *
+ * @opensearch.internal
+ */
 public class JdkJarHellCheck {
 
     private Set<String> detected = new HashSet<>();

libs/common/src/main/java/org/opensearch/bootstrap/package-info.java

@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Contains JarHell Classes */
+package org.opensearch.bootstrap;

libs/core/src/main/java/org/opensearch/common/Booleans.java → libs/common/src/main/java/org/opensearch/common/Booleans.java

@@ -32,6 +32,11 @@
 
 package org.opensearch.common;
 
+/**
+ * Boolean class utilities
+ *
+ * @opensearch.api
+ */
 public final class Booleans {
     private Booleans() {
         throw new AssertionError("No instances intended");

libs/core/src/main/java/org/opensearch/common/CharArrays.java → libs/common/src/main/java/org/opensearch/common/CharArrays.java

@@ -40,6 +40,8 @@
 
 /**
  * Helper class similar to Arrays to handle conversions for Char arrays
+ *
+ * @opensearch.api
  */
 public final class CharArrays {
 

libs/core/src/main/java/org/opensearch/common/CheckedConsumer.java → libs/common/src/main/java/org/opensearch/common/CheckedConsumer.java

@@ -36,6 +36,8 @@
 
 /**
  * A {@link Consumer}-like interface which allows throwing checked exceptions.
+ *
+ * @opensearch.api
  */
 @FunctionalInterface
 public interface CheckedConsumer<T, E extends Exception> {

libs/core/src/main/java/org/opensearch/common/CheckedFunction.java → libs/common/src/main/java/org/opensearch/common/CheckedFunction.java

@@ -36,6 +36,8 @@
 
 /**
  * A {@link Function}-like interface which allows throwing checked exceptions.
+ *
+ * @opensearch.api
  */
 @FunctionalInterface
 public interface CheckedFunction<T, R, E extends Exception> {

libs/core/src/main/java/org/opensearch/common/CheckedRunnable.java → libs/common/src/main/java/org/opensearch/common/CheckedRunnable.java

@@ -34,6 +34,8 @@
 
 /**
  * A {@link Runnable}-like interface which allows throwing checked exceptions.
+ *
+ * @opensearch.api
  */
 @FunctionalInterface
 public interface CheckedRunnable<E extends Exception> {

libs/core/src/main/java/org/opensearch/common/Glob.java → libs/common/src/main/java/org/opensearch/common/Glob.java

@@ -34,6 +34,8 @@
 
 /**
  * Utility class for glob-like matching
+ *
+ * @opensearch.api
  */
 public class Glob {
 

libs/core/src/main/java/org/opensearch/common/MemoizedSupplier.java → libs/common/src/main/java/org/opensearch/common/MemoizedSupplier.java

@@ -34,6 +34,11 @@
 
 import java.util.function.Supplier;
 
+/**
+ * A base supplier using memoization optimization technique
+ *
+ * @opensearch.api
+ */
 public class MemoizedSupplier<T> implements Supplier<T> {
     private Supplier<T> supplier;
     private T value;

libs/core/src/main/java/org/opensearch/common/Nullable.java → libs/common/src/main/java/org/opensearch/common/Nullable.java

@@ -45,7 +45,7 @@
  * {@code null} is an acceptable value for that parameter.  It should not be
  * used for parameters of primitive types.
  *
- *
+ * @opensearch.api
  */
 @Documented
 @TypeQualifierNickname

libs/core/src/main/java/org/opensearch/common/SuppressForbidden.java → libs/common/src/main/java/org/opensearch/common/SuppressForbidden.java

@@ -38,6 +38,8 @@
 
 /**
  * Annotation to suppress forbidden-apis errors inside a whole class, a method, or a field.
+ *
+ * @opensearch.api
  */
 @Retention(RetentionPolicy.CLASS)
 @Target({ ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE })

libs/core/src/main/java/org/opensearch/common/collect/List.java → libs/common/src/main/java/org/opensearch/common/collect/List.java

@@ -34,6 +34,13 @@
 
 import java.util.Collection;
 
+/**
+ * Java 9 List
+ *
+ * todo: deprecate and remove w/ min jdk upgrade to 11?
+ *
+ * @opensearch.internal
+ */
 public class List {
 
     /**

libs/core/src/main/java/org/opensearch/common/collect/Map.java → libs/common/src/main/java/org/opensearch/common/collect/Map.java

@@ -32,6 +32,13 @@
 
 package org.opensearch.common.collect;
 
+/**
+ * Java 9 Map
+ *
+ * todo: deprecate and remove w/ min jdk upgrade to 11?
+ *
+ * @opensearch.internal
+ */
 public class Map {
 
     /**

libs/core/src/main/java/org/opensearch/common/collect/Set.java → libs/common/src/main/java/org/opensearch/common/collect/Set.java

@@ -34,6 +34,13 @@
 
 import java.util.Collection;
 
+/**
+ * Java 9 Set
+ *
+ * todo: deprecate and remove w/ min jdk upgrade to 11?
+ *
+ * @opensearch.internal
+ */
 public class Set {
 
     /**