Merge branch '2.x' into backport-12488-2.x

Signed-off-by: Rishikesh <[email protected]>

.ci/bwcVersions

@@ -54,6 +54,9 @@ BWC_VERSION:
   - "1.3.15"
   - "1.3.16"
   - "1.3.17"
+  - "1.3.18"
+  - "1.3.19"
+  - "1.3.20"
   - "2.0.0"
   - "2.0.1"
   - "2.0.2"
@@ -88,3 +91,8 @@ BWC_VERSION:
   - "2.13.1"
   - "2.14.0"
   - "2.14.1"
+  - "2.15.0"
+  - "2.15.1"
+  - "2.16.0"
+  - "2.16.1"
+  - "2.17.0"

.ci/java-versions.properties

@@ -13,7 +13,8 @@
 # build and test OpenSearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-OPENSEARCH_BUILD_JAVA=openjdk11
+# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
+OPENSEARCH_BUILD_JAVA=openjdk17
 OPENSEARCH_RUNTIME_JAVA=java11
 GRADLE_TASK=build
 GRADLE_EXTRA_ARGS=

.github/CODEOWNERS

@@ -11,17 +11,27 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
+* @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jainankitk @kotwanikunal @linuxpi @mch2 @msfroh @nknize @owaiskazi19  @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
+/modules/lang-painless/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/modules/parent-join/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 /modules/transport-netty4/ @peternied
 
 /plugins/identity-shiro/ @peternied
 
+/server/src/internalClusterTest/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/internalClusterTest/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+
 /server/src/main/java/org/opensearch/extensions/ @peternied
 /server/src/main/java/org/opensearch/identity/ @peternied
-/server/src/main/java/org/opensearch/threadpool/ @peternied
+/server/src/main/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/threadpool/ @jed326 @peternied
 /server/src/main/java/org/opensearch/transport/ @peternied
 
-/.github/ @peternied
+/server/src/test/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/test/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+
+/.github/ @jed326 @peternied
 
-/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gaobinlong @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah

.github/pull_request_template.md

@@ -1,18 +1,21 @@
+<!--  Thanks for sending a pull request, here are some tips:
+
+1. If this is a fix for an undisclosed security vulnerability, please STOP. All security vulnerability reporting and fixes should be done as per our security policy https://github.com/opensearch-project/OpenSearch/security/policy
+2. If this is your first time, please read our contributor guidelines: https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md and developer guide https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md
+3. Ensure you have added or ran the appropriate tests for your PR: https://github.com/opensearch-project/OpenSearch/blob/main/TESTING.md
+-->
+
 ### Description
 [Describe what this change achieves]
 
-### Issues Resolved
-[List any issues this PR will resolve]
+### Related Issues
+Resolves #[Issue number to be closed when this PR is merged]
+<!-- List any other related issues here -->
 
 ### Check List
-- [ ] New functionality includes testing.
-  - [ ] All tests pass
-- [ ] New functionality has been documented.
-  - [ ] New functionality has javadoc added
-- [ ] Failing checks are inspected and point to the corresponding known issue(s) (See: [Troubleshooting Failing Builds](../blob/main/CONTRIBUTING.md#troubleshooting-failing-builds))
-- [ ] Commits are signed per the DCO using --signoff
-- [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
-- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose)
+- [ ] Functionality includes testing.
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md), if applicable.
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose), if applicable.
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
 For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/assemble.yml

@@ -16,9 +16,33 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin
+      - name: Set up JDK 17
+        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
+        if: matrix.java == 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: temurin
+      - name: Set JAVA${{ matrix.java }}_HOME
+        shell: bash
+        run: |
+          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Setup docker (missing on MacOS)
+        id: setup_docker
         if: runner.os == 'macos'
-        uses: douglascamata/setup-docker-macos-action@main
+        run: |
+          exit 0;
+      - name: Run Gradle (assemble)
+        if: runner.os == 'macos' && steps.setup_docker.outcome != 'success'
+        run: |
+          # Report success even if previous step failed (Docker on MacOS runner is very unstable)
+          exit 0;
+      - name: Run Gradle (assemble)
+        shell: bash
+        if: runner.os != 'macos'
+        run: |
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} 
       - name: Run Gradle (assemble)
+        if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
+          exit 0;

.github/workflows/dco.yml

@@ -0,0 +1,19 @@
+name: Developer Certificate of Origin Check
+
+on: [pull_request]
+
+jobs:
+  dco-check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Get PR Commits
+      id: 'get-pr-commits'
+      uses: tim-actions/[email protected]
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: DCO Check
+      uses: tim-actions/[email protected]
+      with:
+        commits: ${{ steps.get-pr-commits.outputs.commits }}
+

.github/workflows/dependabot_pr.yml

@@ -22,6 +22,13 @@ jobs:
         with:
           token: ${{ steps.github_app_token.outputs.token }}
 
+      # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: temurin
+
       - name: Update Gradle SHAs
         run: |
           ./gradlew updateSHAs

.github/workflows/gradle-check.yml

@@ -12,13 +12,28 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
+  check-files:
+    runs-on: ubuntu-latest
+    outputs:
+      RUN_GRADLE_CHECK: ${{ steps.changed-files-specific.outputs.any_changed }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get changed files
+      id: changed-files-specific
+      uses: tj-actions/changed-files@v45
+      with:
+        files_ignore: |
+          release-notes/*.md
+          .github/**
+          *.md
+
   gradle-check:
-    if: github.repository == 'opensearch-project/OpenSearch'
+    needs: check-files
+    if: github.repository == 'opensearch-project/OpenSearch' && needs.check-files.outputs.RUN_GRADLE_CHECK == 'true'
     permissions:
       contents: read # to fetch code (actions/checkout)
       pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
       issues: write # To create an issue if check fails on push.
-
     runs-on: ubuntu-latest
     timeout-minutes: 130
     steps:
@@ -98,6 +113,7 @@ jobs:
         if: success()
         uses: codecov/codecov-action@v4
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: ./codeCoverage.xml
 
       - name: Create Comment Success
@@ -143,11 +159,11 @@ jobs:
 
             Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green. Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
 
-      - name: Create Issue On Push Failure
-        if: ${{ github.event_name == 'push' && failure() }}
-        uses: dblock/create-a-github-issue@v3
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          assignees: ${{ github.event.head_commit.author.username }}, ${{ github.triggering_actor }}
-          filename: .github/ISSUE_TEMPLATE/failed_check.md
+  check-result:
+    needs: [check-files, gradle-check]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if gradle-check fails
+        if: ${{ needs.check-files.outputs.RUN_GRADLE_CHECK && needs.gradle-check.result == 'failure' }}
+        run: exit 1

.github/workflows/precommit.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         java: [ 11, 17, 21 ]
-        os: [ubuntu-latest, windows-latest, macos-13]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
@@ -17,6 +17,18 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
+      - name: Set up JDK 17
+        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
+        if: matrix.java == 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: temurin
+      - name: Set JAVA${{ matrix.java }}_HOME
+        shell: bash
+        run: |
+          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Run Gradle (precommit)
+        shell: bash
         run: |
-          ./gradlew javadoc precommit --parallel
+          ./gradlew javadoc precommit --parallel -Druntime.java=${{ matrix.java }}

.github/workflows/version.yml

@@ -127,3 +127,32 @@ jobs:
           title: '[AUTO] [main] Add bwc version ${{ env.NEXT_VERSION }}.'
           body: |
             I've noticed that a new tag ${{ env.TAG }} was pushed, and added a bwc version ${{ env.NEXT_VERSION }}.
+
+      - name: Create tracking issue
+        id: create-issue
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const body = `
+             ### Description
+             A new version of OpenSearch was released, to prepare for the next release new version numbers need to be updated in all active branches of development.
+
+             ### Exit Criteria
+             Review and merged the following pull requests
+             - [ ] ${{ steps.base_pr.outputs.pull-request-url }}
+             - [ ] ${{ steps.base_x_pr.outputs.pull-request-url }}
+             - [ ] ${{ steps.main_pr.outputs.pull-request-url }}
+
+             ### Additional Context
+             See project wide guidance on branching and versions [[link]](https://github.com/opensearch-project/.github/blob/main/RELEASING.md).
+            `
+            const { data: issue }= await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ["Build"],
+              title: "Increment version for ${{ env.NEXT_VERSION }}",
+              body: body
+            });
+            console.error(JSON.stringify(issue));
+            return issue.number;
+          result-encoding: string

.whitesource

@@ -0,0 +1,45 @@
+{
+  "scanSettings": {
+    "configMode": "AUTO",
+    "configExternalURL": "",
+    "projectToken": "",
+    "baseBranches": []
+  },
+  "scanSettingsSAST": {
+    "enableScan": false,
+    "scanPullRequests": false,
+    "incrementalScan": true,
+    "baseBranches": [],
+    "snippetSize": 10
+  },
+  "checkRunSettings": {
+    "vulnerableCheckRunConclusionLevel": "failure",
+    "displayMode": "diff",
+    "useMendCheckNames": true
+  },
+  "checkRunSettingsSAST": {
+    "checkRunConclusionLevel": "failure",
+    "severityThreshold": "high"
+  },
+  "issueSettings": {
+    "minSeverityLevel": "LOW",
+    "issueType": "DEPENDENCY"
+  },
+  "issueSettingsSAST": {
+    "minSeverityLevel": "high",
+    "issueType": "repo"
+  },
+  "remediateSettings": {
+    "workflowRules": {
+      "enabled": true
+    }
+  },
+   "imageSettings":{
+       "imageTracing":{
+           "enableImageTracingPR": false,
+           "addRepositoryCoordinate": false,
+           "addDockerfilePath": false,
+           "addMendIdentifier": false
+       }
+   }
+}