Merge branch '2.x' into backport/backport-13784-to-2.x

Signed-off-by: Peter Alfonsi <[email protected]>

.ci/bwcVersions

@@ -86,4 +86,5 @@ BWC_VERSION:
   - "2.12.1"
   - "2.13.0"
   - "2.13.1"
-
+  - "2.14.0"
+  - "2.14.1"

.github/CODEOWNERS

@@ -1,5 +1,5 @@
 # CODEOWNERS manages notifications, not PR approvals
-# For PR approvals see /.github/workflows/maintainer-approval.yml 
+# For PR approvals see /.github/workflows/maintainer-approval.yml
 
 # Files have a single rule applied, the last match decides the owner
 # If you would like to more specifically apply ownership, include existing owner in new sub fields
@@ -24,4 +24,4 @@
 
 /.github/ @peternied
 
-/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah

.github/workflows/assemble.yml

@@ -17,8 +17,23 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: temurin
       - name: Setup docker (missing on MacOS)
+        id: setup_docker
         if: runner.os == 'macos'
         uses: douglascamata/setup-docker-macos-action@main
+        continue-on-error: true
+        with:
+          upgrade-qemu: true
+          colima:  v0.6.8
+      - name: Run Gradle (assemble)
+        if: runner.os == 'macos' && steps.setup_docker.outcome != 'success'
+        run: |
+          # Report success even if previous step failed (Docker on MacOS runner is very unstable)
+          exit 0;
+      - name: Run Gradle (assemble)
+        if: runner.os != 'macos'
+        run: |
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
       - name: Run Gradle (assemble)
+        if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
         run: |
           ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE

.github/workflows/dco.yml

@@ -0,0 +1,19 @@
+name: Developer Certificate of Origin Check
+
+on: [pull_request]
+
+jobs:
+  dco-check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Get PR Commits
+      id: 'get-pr-commits'
+      uses: tim-actions/[email protected]
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: DCO Check
+      uses: tim-actions/[email protected]
+      with:
+        commits: ${{ steps.get-pr-commits.outputs.commits }}
+

.github/workflows/gradle-check.yml

@@ -12,13 +12,28 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
+  check-files:
+    runs-on: ubuntu-latest
+    outputs: 
+      RUN_GRADLE_CHECK: ${{ steps.changed-files-specific.outputs.any_changed }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get changed files
+      id: changed-files-specific
+      uses: tj-actions/changed-files@v44
+      with:
+        files_ignore: |
+          release-notes/*.md
+          .github/**
+          *.md
+    
   gradle-check:
-    if: github.repository == 'opensearch-project/OpenSearch'
+    needs: check-files
+    if: github.repository == 'opensearch-project/OpenSearch' && needs.check-files.outputs.RUN_GRADLE_CHECK == 'true'
     permissions:
       contents: read # to fetch code (actions/checkout)
       pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
       issues: write # To create an issue if check fails on push.
-
     runs-on: ubuntu-latest
     timeout-minutes: 130
     steps:
@@ -30,11 +45,31 @@ jobs:
       - name: Setup environment variables (PR)
         if: github.event_name == 'pull_request_target'
         run: |
+          echo "event_name=pull_request_target" >> $GITHUB_ENV
+          echo "branch_name=$(jq --raw-output .pull_request.base.ref $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_owner=$(jq --raw-output .pull_request.user.login $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_or_commit_description=$(jq --ascii-output .pull_request.body $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "post_merge_action=false" >> $GITHUB_ENV
+
+      # to get the PR data that can be used for post merge actions
+      - uses: actions/github-script@v7
+        if: github.event_name == 'push'
+        id: get_pr_data
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            return (
+              await github.rest.repos.listPullRequestsAssociatedWithCommit({
+                commit_sha: context.sha,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+              })
+            ).data[0];
 
       - name: Setup environment variables (Push)
         if: github.event_name == 'push'
@@ -43,11 +78,15 @@ jobs:
           ref_id=$(git rev-parse HEAD)
           branch_name=$(git rev-parse --abbrev-ref HEAD)
           echo "branch_name=$branch_name" >> $GITHUB_ENV
+          echo "event_name=push" >> $GITHUB_ENV
           echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
           echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
           echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
           echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
-          echo "pr_number=Null" >> $GITHUB_ENV
+          echo "pr_owner=$(jq --raw-output '.commits[0].author.username' $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo 'pr_number=${{ fromJson(steps.get_pr_data.outputs.result).number }}' >> $GITHUB_ENV
+          echo "pr_or_commit_description=$(jq --ascii-output .head_commit.message $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "post_merge_action=true" >> $GITHUB_ENV
 
       - name: Checkout opensearch-build repo
         uses: actions/checkout@v4
@@ -127,3 +166,12 @@ jobs:
         with:
           assignees: ${{ github.event.head_commit.author.username }}, ${{ github.triggering_actor }}
           filename: .github/ISSUE_TEMPLATE/failed_check.md
+
+  check-result:
+    needs: [check-files, gradle-check]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if gradle-check fails
+        if: ${{ needs.check-files.outputs.RUN_GRADLE_CHECK && needs.gradle-check.result == 'failure' }}
+        run: exit 1

CHANGELOG.md

@@ -8,26 +8,44 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add support for Azure Managed Identity in repository-azure ([#12423](https://github.com/opensearch-project/OpenSearch/issues/12423))
 - Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478))
 - Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293))
+- [Remote Store] Upload translog checkpoint as object metadata to translog.tlog([#13637](https://github.com/opensearch-project/OpenSearch/pull/13637))
+- [Remote Store] Add dynamic cluster settings to set timeout for segments upload to Remote Store ([#13679](https://github.com/opensearch-project/OpenSearch/pull/13679))
+- Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819))
 
 ### Dependencies
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))
 - Bump `org.apache.commons:commons-text` from 1.11.0 to 1.12.0 ([#13557](https://github.com/opensearch-project/OpenSearch/pull/13557))
 - Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
-- Bump `com.gradle.enterprise` from 3.17.2 to 3.17.3 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641))
+- Bump `com.gradle.enterprise` from 3.17.2 to 3.17.4 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641), [#13753](https://github.com/opensearch-project/OpenSearch/pull/13753))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.6 to 3.4.0 ([#13642](https://github.com/opensearch-project/OpenSearch/pull/13642))
+- Bump `mockito` from 5.11.0 to 5.12.0 ([#13665](https://github.com/opensearch-project/OpenSearch/pull/13665))
+- Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#13752](https://github.com/opensearch-project/OpenSearch/pull/13752))
+- Bump `ch.qos.logback:logback-core` from 1.5.3 to 1.5.6 ([#13756](https://github.com/opensearch-project/OpenSearch/pull/13756))
+- Bump `netty` from 4.1.109.Final to 4.1.110.Final ([#13802](https://github.com/opensearch-project/OpenSearch/pull/13802))
+- Bump `jackson` from 2.17.0 to 2.17.1 ([#13817](https://github.com/opensearch-project/OpenSearch/pull/13817))
+- Bump `reactor` from 3.5.15 to 3.5.17 ([#13825](https://github.com/opensearch-project/OpenSearch/pull/13825))
+- Bump `reactor-netty` from 1.1.17 to 1.1.19 ([#13825](https://github.com/opensearch-project/OpenSearch/pull/13825))
+- Bump `commons-cli:commons-cli` from 1.7.0 to 1.8.0 ([#13840](https://github.com/opensearch-project/OpenSearch/pull/13840))
+- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.0 to 5.2.1 ([#13839](https://github.com/opensearch-project/OpenSearch/pull/13839))
+- Bump `actions/checkout` from 3 to 4 ([#13935](https://github.com/opensearch-project/OpenSearch/pull/13935))
+- Bump `com.netflix.nebula.ospackage-base` from 11.9.0 to 11.9.1 ([#13933](https://github.com/opensearch-project/OpenSearch/pull/13933))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
 - Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481))
+- Adds support to inject telemetry instances to plugins ([#13636](https://github.com/opensearch-project/OpenSearch/pull/13636))
 
 ### Deprecated
 
 ### Removed
 - Remove handling of index.mapper.dynamic in AutoCreateIndex([#13067](https://github.com/opensearch-project/OpenSearch/pull/13067))
 
 ### Fixed
-- Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
 - Fix get field mapping API returns 404 error in mixed cluster with multiple versions ([#13624](https://github.com/opensearch-project/OpenSearch/pull/13624))
+- Allow clearing `remote_store.compatibility_mode` setting ([#13646](https://github.com/opensearch-project/OpenSearch/pull/13646))
+- Painless: ensure type "UnmodifiableMap" for params ([#13885](https://github.com/opensearch-project/OpenSearch/pull/13885))
+- Pass parent filter to inner hit query ([#13903](https://github.com/opensearch-project/OpenSearch/pull/13903))
+- Fix NPE on restore searchable snapshot ([#13911](https://github.com/opensearch-project/OpenSearch/pull/13911))
 
 ### Security
 

MAINTAINERS.md

@@ -14,6 +14,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Dan Widdis               | [dbwiddis](https://github.com/dbwiddis)                 | Amazon      |
 | Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock)                     | Amazon      |
 | Gaurav Bafna             | [gbbafna](https://github.com/gbbafna)                   | Amazon      |
+| Jay Deng                 | [jed326](https://github.com/jed326)                     | Amazon      |
 | Kunal Kotwani            | [kotwanikunal](https://github.com/kotwanikunal)         | Amazon      |
 | Marc Handalian           | [mch2](https://github.com/mch2)                         | Amazon      |
 | Michael Froh             | [msfroh](https://github.com/msfroh)                     | Amazon      |

build.gradle

@@ -55,7 +55,6 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.25.0" apply false
-  id "org.gradle.test-retry" version "1.5.8" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }
@@ -71,6 +70,13 @@ apply from: 'gradle/run.gradle'
 apply from: 'gradle/missing-javadoc.gradle'
 apply from: 'gradle/code-coverage.gradle'
 
+// Disable unconditional publishing of build scans
+develocity {
+  buildScan {
+    publishing.onlyIf { false }
+  }
+}
+
 // common maven publishing configuration
 allprojects {
   group = 'org.opensearch'
@@ -454,9 +460,8 @@ gradle.projectsEvaluated {
 
 // test retry configuration
 subprojects {
-  apply plugin: "org.gradle.test-retry"
   tasks.withType(Test).configureEach {
-    retry {
+    develocity.testRetry {
       if (BuildParams.isCi()) {
         maxRetries = 3
         maxFailures = 10

buildSrc/version.properties

@@ -7,8 +7,8 @@ bundled_jdk = 21.0.3+9
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.17.0
-jackson_databind  = 2.17.0
+jackson           = 2.17.1
+jackson_databind  = 2.17.1
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0
@@ -27,12 +27,12 @@ google_http_client = 1.44.1
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.109.Final
+netty             = 4.1.110.Final
 joda              = 2.12.7
 
 # project reactor
-reactor_netty     = 1.1.17
-reactor           = 3.5.15
+reactor_netty     = 1.1.19
+reactor           = 3.5.17
 
 # client dependencies
 httpclient        = 4.5.14
@@ -55,7 +55,7 @@ bouncycastle=1.78
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
 hamcrest          = 2.1
-mockito           = 5.11.0
+mockito           = 5.12.0
 objenesis         = 3.2
 bytebuddy         = 1.14.9
 

client/sniffer/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.9.0"
+  id "com.netflix.nebula.ospackage-base" version "11.9.1"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.1.jar.sha1

@@ -0,0 +1 @@
+fca7ef6192c9ad05d07bc50da991bf937a84af3a

distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.1.jar.sha1

@@ -0,0 +1 @@
+0524dcbcccdde7d45a679dfc333e4763feb09079

gradle.properties

@@ -22,7 +22,7 @@ org.gradle.jvmargs=-Xmx3g -XX:+HeapDumpOnOutOfMemoryError -Xss2m \
 options.forkOptions.memoryMaximumSize=3g
 
 # Disable Gradle Enterprise Gradle plugin's test retry
-systemProp.gradle.enterprise.testretry.enabled=false
+systemProp.develocity.testretry.enabled.enabled=false
 
 # Disable duplicate project id detection
 # See https://docs.gradle.org/current/userguide/upgrading_version_6.html#duplicate_project_names_may_cause_publication_to_fail

libs/core/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilder.java

@@ -157,6 +157,9 @@ public static XContentBuilder builder(XContent xContent, Set<String> includes, S
 
     /**
      * Returns a string representation of the builder (only applicable for text based xcontent).
+     * Note: explicitly or implicitly (from debugger) calling toString() could cause XContentBuilder
+     * to close which is a side effect done by @see BytesReference#bytes().
+     * Trying to write more contents after toString() will cause NPE. Use it with caution.
      */
     @Override
     public String toString() {

libs/x-content/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

libs/x-content/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1

@@ -0,0 +1 @@
+ba5d8e6ecc62aa0e49c0ce935b8696352dbebc71

libs/x-content/licenses/jackson-dataformat-smile-2.17.1.jar.sha1

@@ -0,0 +1 @@
+89683ac4f0a0c2c4f69ea56b90480ed40266dac8

libs/x-content/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1

@@ -0,0 +1 @@
+b4c7b8a9ea3f398116a75c146b982b22afebc4ee

modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java → modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheIT.java

@@ -65,7 +65,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(TieredSpilloverCachePlugin.class, MockDiskCachePlugin.class);
     }
 
-    private Settings defaultSettings(String onHeapCacheSizeInBytesOrPecentage) {
+    static Settings defaultSettings(String onHeapCacheSizeInBytesOrPercentage) {
         return Settings.builder()
             .put(FeatureFlags.PLUGGABLE_CACHE, "true")
             .put(
@@ -88,7 +88,7 @@ private Settings defaultSettings(String onHeapCacheSizeInBytesOrPecentage) {
                 OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
                     .get(MAXIMUM_SIZE_IN_BYTES_KEY)
                     .getKey(),
-                onHeapCacheSizeInBytesOrPecentage
+                onHeapCacheSizeInBytesOrPercentage
             )
             .build();
     }