Merge branch 'main' of github.com:opensearch-project/OpenSearch

.ci/bwcVersions

@@ -19,3 +19,7 @@ BWC_VERSION:
   - "2.7.0"
   - "2.7.1"
   - "2.8.0"
+  - "2.8.1"
+  - "2.9.0"
+  - "2.9.1"
+  - "2.10.0"

.github/CODEOWNERS

@@ -1 +1 @@
-*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis
+*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale @sohami @msfroh

.github/dependabot.yml

@@ -7,6 +7,14 @@ updates:
     labels:
       - "dependabot"
       - "dependencies"
+  - directory: /
+    open-pull-requests-limit: 1
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "dependabot"
+      - "dependencies"
   - directory: /benchmarks/
     open-pull-requests-limit: 1
     package-ecosystem: gradle

.github/workflows/add-untriaged.yml

@@ -6,6 +6,7 @@ on:
 
 jobs:
   apply-label:
+    if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/github-script@v6

.github/workflows/backport.yml

@@ -33,7 +33,8 @@ jobs:
           installation_id: 22958780
 
       - name: Backport
-        uses: VachaShah/backport@v2.1.0
+        uses: VachaShah/backport@v2.2.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
           head_template: backport/backport-<%= number %>-to-<%= base %>
+          failure_labels: backport-failed

.github/workflows/changelog_verifier.yml

@@ -6,6 +6,7 @@ on:
 jobs:
   # Enforces the update of a changelog file on every pull request
   verify-changelog:
+    if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/check-compatibility.yml

@@ -0,0 +1,50 @@
+---
+name: Check Compatibility
+
+on:
+  pull_request_target
+
+jobs:
+  check-compatibility:
+    if: github.repository == 'opensearch-project/OpenSearch'
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Run compatibility task
+        run: ./gradlew checkCompatibility -i | tee $HOME/gradlew-check.out
+
+      - name: Get results
+        run: |
+          echo '## Compatibility status:' > "${{ github.workspace }}/results.txt"
+          echo "Checks if related components are compatible with change $(git rev-parse --short HEAD)" >> "${{ github.workspace }}/results.txt"
+          echo "### Incompatible components" >> "${{ github.workspace }}/results.txt" && grep -e 'Incompatible component' $HOME/gradlew-check.out | sed -e 's/Incompatible component: \[\(.*\)\]/- \1/' >> "${{ github.workspace }}/results.txt"
+          echo "### Skipped components"      >> "${{ github.workspace }}/results.txt" && grep -e 'Skipped component'      $HOME/gradlew-check.out | sed -e 's/Skipped component: \[\(.*\)\]/- \1/'      >> "${{ github.workspace }}/results.txt"
+          echo "### Compatible components"   >> "${{ github.workspace }}/results.txt" && grep -e 'Compatible component'   $HOME/gradlew-check.out | sed -e 's/Compatible component: \[\(.*\)\]/- \1/'   >> "${{ github.workspace }}/results.txt"
+
+      - name: Upload results
+        uses: actions/upload-artifact@v3
+        with:
+          name: results.txt
+          path: ${{ github.workspace }}/results.txt
+
+  add-comment:
+    needs: [check-compatibility]
+    permissions:
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download results
+        uses: actions/download-artifact@v3
+        with:
+          name: results.txt
+
+      - name: Add comment on the PR
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          issue-number: ${{ github.event.number }}
+          body-path: results.txt

.github/workflows/gradle-check.yml

@@ -13,6 +13,7 @@ permissions:
 
 jobs:
   gradle-check:
+    if: github.repository == 'opensearch-project/OpenSearch'
     permissions:
       contents: read # to fetch code (actions/checkout)
       pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
@@ -22,52 +23,52 @@ jobs:
     timeout-minutes: 130
     steps:
       - name: Checkout OpenSearch repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Setup environment variables (PR)
         if: github.event_name == 'pull_request_target'
         run: |
-            echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-            echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-            echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-            echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-            echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
 
       - name: Setup environment variables (Push)
         if: github.event_name == 'push'
         run: |
-            repo_url="https://github.com/opensearch-project/OpenSearch"
-            ref_id=$(git rev-parse HEAD)
-            branch_name=$(git rev-parse --abbrev-ref HEAD)
-            echo "branch_name=$branch_name" >> $GITHUB_ENV
-            echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
-            echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
-            echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
-            echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
-            echo "pr_number=Null" >> $GITHUB_ENV
+          repo_url="https://github.com/opensearch-project/OpenSearch"
+          ref_id=$(git rev-parse HEAD)
+          branch_name=$(git rev-parse --abbrev-ref HEAD)
+          echo "branch_name=$branch_name" >> $GITHUB_ENV
+          echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
+          echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
+          echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
+          echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
+          echo "pr_number=Null" >> $GITHUB_ENV
 
       - name: Checkout opensearch-build repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: opensearch-project/opensearch-build
           ref: main
           path: opensearch-build
 
       - name: Trigger jenkins workflow to run gradle check
         run: |
-            set -e
-            set -o pipefail
-            bash opensearch-build/scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log
+          set -e
+          set -o pipefail
+          bash opensearch-build/scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log
 
       - name: Setup Result Status
         if: always()
         run: |
-            WORKFLOW_URL=`cat gradle-check.log | grep 'WORKFLOW_URL' | awk '{print $2}'`
-            RESULT=`cat gradle-check.log | grep 'Result:' | awk '{print $2}'`
-            echo "workflow_url=$WORKFLOW_URL" >> $GITHUB_ENV
-            echo "result=$RESULT" >> $GITHUB_ENV
+          WORKFLOW_URL=`cat gradle-check.log | grep 'WORKFLOW_URL' | awk '{print $2}'`
+          RESULT=`cat gradle-check.log | grep 'Result:' | awk '{print $2}'`
+          echo "workflow_url=$WORKFLOW_URL" >> $GITHUB_ENV
+          echo "result=$RESULT" >> $GITHUB_ENV
 
       - name: Upload Coverage Report
         if: success()
@@ -81,25 +82,25 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-              ### Gradle Check (Jenkins) Run Completed with:
-              * **RESULT:** ${{ env.result }} :white_check_mark:
-              * **URL:** ${{ env.workflow_url }}
-              * **CommitID:** ${{ env.pr_from_sha }}
+            ### Gradle Check (Jenkins) Run Completed with:
+            * **RESULT:** ${{ env.result }} :white_check_mark:
+            * **URL:** ${{ env.workflow_url }}
+            * **CommitID:** ${{ env.pr_from_sha }}
 
       - name: Extract Test Failure
         if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
         run: |
-            TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
-            if [[ "$TEST_FAILURES" != "" ]]
-            then
-              echo "test_failures<<EOF" >> $GITHUB_ENV
-              echo "" >> $GITHUB_ENV
-              echo "* **TEST FAILURES:**" >> $GITHUB_ENV
-              echo '```' >> $GITHUB_ENV
-              echo "$TEST_FAILURES" >> $GITHUB_ENV
-              echo '```' >> $GITHUB_ENV
-              echo "EOF" >> $GITHUB_ENV
-            fi
+          TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
+          if [[ "$TEST_FAILURES" != "" ]]
+          then
+            echo "test_failures<<EOF" >> $GITHUB_ENV
+            echo "" >> $GITHUB_ENV
+            echo "* **TEST FAILURES:**" >> $GITHUB_ENV
+            echo '```' >> $GITHUB_ENV
+            echo "$TEST_FAILURES" >> $GITHUB_ENV
+            echo '```' >> $GITHUB_ENV
+            echo "EOF" >> $GITHUB_ENV
+          fi
 
       - name: Create Comment Flaky
         if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }}
@@ -119,12 +120,12 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-              ### Gradle Check (Jenkins) Run Completed with:
-              * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
-              * **URL:** ${{ env.workflow_url }}
-              * **CommitID:** ${{ env.pr_from_sha }}
-              Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
-              Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
+            ### Gradle Check (Jenkins) Run Completed with:
+            * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
+            * **URL:** ${{ env.workflow_url }}
+            * **CommitID:** ${{ env.pr_from_sha }}
+            Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
+            Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
 
       - name: Create Issue On Push Failure
         if: ${{ github.event_name == 'push' && failure() }}

.github/workflows/links.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: lychee Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v1.2.0
+        uses: lycheeverse/lychee-action@v1.8.0
         with:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true

.github/workflows/lucene-snapshots.yml

@@ -13,6 +13,7 @@ on:
 
 jobs:
   publish-snapshots:
+    if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
     # These permissions are needed to interact with GitHub's OIDC Token endpoint.
     permissions:
@@ -22,7 +23,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 17
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           java-version: '17'
           distribution: 'adopt'
@@ -49,7 +50,7 @@ jobs:
         run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ steps.version.outputs.REVISION }}
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_ROLE }}
           aws-region: us-west-2

.github/workflows/poc-checklist.yml

@@ -0,0 +1,49 @@
+name: Add comment
+on:
+  issues:
+    types:
+      - labeled
+jobs:
+  add-comment:
+    if: github.event.label.name == 'poc'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - name: Add comment
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ github.event.issue.number }}
+          body: |
+            ### POC Checklist:
+
+            Please go through the following checklist to ensure these items are taken into account while designing the POC.
+
+            - [ ] Supports safe upgrade paths from all supported BWC versions to the current version
+            - [ ] Supports compatibility with all plugins
+                - [ ] opensearch-alerting
+                - [ ] opensearch-anomaly-detection
+                - [ ] opensearch-asynchronous-search
+                - [ ] opensearch-cross-cluster-replication
+                - [ ] opensearch-geospatial
+                - [ ] opensearch-index-management
+                - [ ] opensearch-job-scheduler
+                - [ ] opensearch-knn
+                - [ ] opensearch-ml
+                - [ ] opensearch-notifications
+                - [ ] opensearch-notifications-core
+                - [ ] opensearch-observability
+                - [ ] opensearch-performance-analyzer
+                - [ ] opensearch-reports-scheduler
+                - [ ] opensearch-security
+                - [ ] opensearch-sql
+            - [ ] Supports lucene upgrades across minor lucene versions
+            - [ ] Supports lucene upgrades across major lucene versions
+            - [ ] Supports lucene upgrades across underlying lucene codec bumps (Eg: Lucene95Codec -> Lucene96Codec)
+            - [ ] Supports wire compatibility of OpenSearch
+            - [ ] Plan to measure performance degradation/improvement (if any)
+            - [ ] Plan to document any user facing changes introduced by this feature
+            - [ ] Ensure working and passing CI
+
+            Thank you for your contribution!
+

.github/workflows/precommit.yml

@@ -1,19 +1,34 @@
-name: Gradle Precommit
+name: Gradle Precommit and Asssemble
 on: [pull_request]
- 
+
 jobs:
   precommit:
+    if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest] 
+        os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up JDK 11
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           java-version: 11
-          distribution: adopt
-      - name: Run Gradle
+          distribution: temurin
+          cache: gradle
+      - name: Run Gradle (precommit)
         run: |
           ./gradlew javadoc precommit --parallel
+      - name: Setup docker (missing on MacOS)
+        if: runner.os == 'macos'
+        run: |
+          # Workaround for https://github.com/actions/runner-images/issues/8104
+          brew remove --ignore-dependencies qemu
+          curl -o ./qemu.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f88e30b3a23ef3735580f9b05535ce5a0a03c9e3/Formula/qemu.rb
+          brew install ./qemu.rb
+          brew install docker
+          colima start
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+      - name: Run Gradle (assemble)
+        run: |
+          ./gradlew assemble --parallel

.github/workflows/publish-maven-snapshots.yml

@@ -5,11 +5,12 @@ on:
   push:
       branches:
         - main
-        - '1.3'
-        - 2.x
+        - '[0-9]+.[0-9]+'
+        - '[0-9]+.x'
 
 jobs:
   build-and-publish-snapshots:
+    if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
 
     permissions:
@@ -25,7 +26,7 @@ jobs:
           java-version: 17
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
           aws-region: us-east-1