forked from elastic/elasticsearch
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'origin/feature/esql' into function/sqrt
- Loading branch information
Showing
733 changed files
with
19,093 additions
and
7,247 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
@ECHO OFF | ||
|
||
FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i | ||
|
||
SET JAVA_HOME=%USERPROFILE%\.java\%ES_BUILD_JAVA% | ||
SET JAVA16_HOME=%USERPROFILE%\.java\openjdk16 | ||
|
||
SET GRADLEW=./gradlew --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ | ||
SET GRADLEW_BAT=./gradlew.bat --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ | ||
|
||
(if not exist "%USERPROFILE%/.gradle" mkdir "%USERPROFILE%/.gradle") && (echo. >> "%USERPROFILE%/.gradle/gradle.properties" && echo org.gradle.daemon=false >> "%USERPROFILE%/.gradle/gradle.properties") | ||
|
||
set WORKSPACE=%cd% | ||
set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER% | ||
set COMPOSE_HTTP_TIMEOUT=120 | ||
set JOB_BRANCH=%BUILDKITE_BRANCH% | ||
|
||
set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache | ||
set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache | ||
|
||
exit /b 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
steps: | ||
- trigger: apache-lucene-build-snapshot | ||
label: Trigger pipeline to build lucene snapshot | ||
key: lucene-build | ||
if: build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == "" | ||
- wait | ||
- label: Upload and update lucene snapshot | ||
command: .buildkite/scripts/lucene-snapshot/upload-snapshot.sh | ||
timeout_in_minutes: 15 | ||
env: | ||
USE_LUCENE_SNAPSHOT_CREDS: "true" | ||
UPDATE_ES_LUCENE_SNAPSHOT: "true" | ||
agents: | ||
provider: gcp | ||
image: family/elasticsearch-ubuntu-2004 | ||
machineType: custom-32-98304 | ||
buildDirectory: /dev/shm/bk | ||
- wait | ||
- trigger: "elasticsearch-lucene-snapshot-tests" | ||
build: | ||
branch: "${BUILDKITE_BRANCH}" | ||
async: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
steps: | ||
- label: sanity-check | ||
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit | ||
timeout_in_minutes: 300 | ||
agents: | ||
provider: gcp | ||
image: family/elasticsearch-ubuntu-2004 | ||
machineType: custom-32-98304 | ||
buildDirectory: /dev/shm/bk | ||
- wait: null | ||
- label: part1 | ||
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 | ||
timeout_in_minutes: 300 | ||
agents: | ||
provider: gcp | ||
image: family/elasticsearch-ubuntu-2004 | ||
machineType: custom-32-98304 | ||
buildDirectory: /dev/shm/bk | ||
- label: part2 | ||
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 | ||
timeout_in_minutes: 300 | ||
agents: | ||
provider: gcp | ||
image: family/elasticsearch-ubuntu-2004 | ||
machineType: custom-32-98304 | ||
buildDirectory: /dev/shm/bk | ||
- label: part3 | ||
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 | ||
timeout_in_minutes: 300 | ||
agents: | ||
provider: gcp | ||
image: family/elasticsearch-ubuntu-2004 | ||
machineType: custom-32-98304 | ||
buildDirectory: /dev/shm/bk | ||
- group: bwc-snapshots | ||
steps: | ||
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots" | ||
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest | ||
timeout_in_minutes: 300 | ||
matrix: | ||
setup: | ||
BWC_VERSION: | ||
- 7.17.13 | ||
- 8.9.1 | ||
- 8.10.0 | ||
agents: | ||
provider: gcp | ||
image: family/elasticsearch-ubuntu-2004 | ||
machineType: custom-32-98304 | ||
buildDirectory: /dev/shm/bk | ||
env: | ||
BWC_VERSION: "{{matrix.BWC_VERSION}}" | ||
- label: rest-compat | ||
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat | ||
timeout_in_minutes: 300 | ||
agents: | ||
provider: gcp | ||
image: family/elasticsearch-ubuntu-2004 | ||
machineType: custom-32-98304 | ||
buildDirectory: /dev/shm/bk |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
steps: | ||
- label: Update lucene-snapshot branch | ||
command: .buildkite/scripts/lucene-snapshot/update-branch.sh | ||
timeout_in_minutes: 15 | ||
- wait | ||
- trigger: "elasticsearch-lucene-snapshot-tests" | ||
build: | ||
branch: "${BUILDKITE_BRANCH}" | ||
async: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable. | ||
# It should really only be used inside .buildkite/hooks/pre-command | ||
|
||
VAULT_ROLE_ID=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials) | ||
VAULT_SECRET_ID=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials) | ||
VAULT_ADDR=https://secrets.elastic.co:8200 | ||
|
||
unset VAULT_TOKEN | ||
VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) | ||
export VAULT_TOKEN | ||
|
||
vault read -format=json aws-elastic/creds/lucene-snapshots |
13 changes: 13 additions & 0 deletions
13
.buildkite/scripts/lucene-snapshot/remove-verification-metadata.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
import re | ||
|
||
regex = re.compile( | ||
r'<component group="org.apache.lucene" name="lucene-.*?</component>\s*', | ||
re.MULTILINE | re.DOTALL, | ||
) | ||
|
||
with open("gradle/verification-metadata.xml", "r+") as f: | ||
text = f.read() | ||
text = regex.sub("", text) | ||
f.seek(0) | ||
f.truncate() | ||
f.write(text) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then | ||
echo "Error: This script should only be run on the lucene_snapshot branch" | ||
exit 1 | ||
fi | ||
|
||
echo --- Updating lucene_snapshot branch with main | ||
|
||
git config --global user.name elasticsearchmachine | ||
git config --global user.email '[email protected]' | ||
|
||
git checkout lucene_snapshot | ||
git fetch origin main | ||
git merge --no-edit origin/main | ||
git push origin lucene_snapshot |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then | ||
echo "Error: This script should only be run on the lucene_snapshot branch" | ||
exit 1 | ||
fi | ||
|
||
echo --- Update Lucene snapshot in Elasticsearch | ||
|
||
LUCENE_SNAPSHOT_VERSION=${LUCENE_SNAPSHOT_VERSION:-} | ||
|
||
if [[ -z "$LUCENE_SNAPSHOT_VERSION" ]]; then | ||
LUCENE_SNAPSHOT_VERSION=$(buildkite-agent meta-data get lucene-snapshot-version) | ||
fi | ||
|
||
echo "Lucene Snapshot Version: $LUCENE_SNAPSHOT_VERSION" | ||
|
||
git checkout "$BUILDKITE_BRANCH" | ||
git pull --ff-only origin "$BUILDKITE_BRANCH" | ||
|
||
# Replace `lucene = <version>` string in version.properties and maintain the same indentation | ||
sed -E "s/^(lucene *= *[^ ]* *).*\$/\1$LUCENE_SNAPSHOT_VERSION/" build-tools-internal/version.properties > new-version.properties | ||
mv new-version.properties build-tools-internal/version.properties | ||
|
||
# Remove stale verification metadata, because generating them just appends new ones | ||
python3 .buildkite/scripts/lucene-snapshot/remove-verification-metadata.py | ||
./gradlew --write-verification-metadata sha256 | ||
|
||
# 9.8.0-snapshot-1f25c68 -> 9.8.0 | ||
VERSION=$(echo "$LUCENE_SNAPSHOT_VERSION" | cut -f 1 -d '-') | ||
VERSION_SNAKE=$(echo "$VERSION" | sed -E 's/\./_/g') | ||
|
||
sed -E "s/^(:lucene_version: *).*\$/\1$VERSION/" docs/Versions.asciidoc > docs/Versions.asciidoc.new | ||
sed -E "s/^(:lucene_version_path: *).*\$/\1$VERSION_SNAKE/" docs/Versions.asciidoc.new > docs/Versions.asciidoc | ||
rm -f docs/Versions.asciidoc.new | ||
|
||
if git diff-index --quiet HEAD --; then | ||
echo 'No changes to commit.' | ||
else | ||
git config --global user.name elasticsearchmachine | ||
git config --global user.email '[email protected]' | ||
|
||
git add build-tools-internal/version.properties | ||
git add gradle/verification-metadata.xml | ||
git add docs/Versions.asciidoc | ||
|
||
git commit -m "[Automated] Update Lucene snapshot to $LUCENE_SNAPSHOT_VERSION" | ||
git push origin "$BUILDKITE_BRANCH" | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
LUCENE_BUILD_ID=${LUCENE_BUILD_ID:-} | ||
|
||
if [[ -z "$LUCENE_BUILD_ID" ]]; then | ||
build_json=$(curl -sH "Authorization: Bearer $BUILDKITE_API_TOKEN" "https://api.buildkite.com/v2/organizations/elastic/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER") | ||
LUCENE_BUILD_ID=$(jq -r '.jobs[] | select(.step_key == "lucene-build").triggered_build.id' <<< "$build_json") | ||
fi | ||
|
||
export LUCENE_BUILD_ID | ||
|
||
LUCENE_SHA=$(buildkite-agent meta-data get --build "$LUCENE_BUILD_ID" lucene-snapshot-sha) | ||
export LUCENE_SHA | ||
|
||
LUCENE_SNAPSHOT_VERSION=$(buildkite-agent meta-data get --build "$LUCENE_BUILD_ID" lucene-snapshot-version) | ||
export LUCENE_SNAPSHOT_VERSION | ||
|
||
echo --- Downloading lucene snapshot | ||
|
||
mkdir lucene-snapshot | ||
cd lucene-snapshot | ||
buildkite-agent artifact download --build "$LUCENE_BUILD_ID" lucene-snapshot.tar.gz . | ||
tar -xvf lucene-snapshot.tar.gz | ||
cd - | ||
|
||
echo --- Upload lucene snapshot to S3 | ||
|
||
if ! which aws; then | ||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | ||
unzip awscliv2.zip | ||
sudo ./aws/install | ||
rm -rf awscliv2.zip aws | ||
fi | ||
|
||
aws s3 sync lucene-snapshot/ "s3://download.elasticsearch.org/lucenesnapshots/$LUCENE_SHA/" --acl public-read | ||
|
||
if [[ "${UPDATE_ES_LUCENE_SNAPSHOT:-}" ]]; then | ||
.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh | ||
fi |
Oops, something went wrong.