Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/feature/esql' into function/sqrt
Browse files Browse the repository at this point in the history
  • Loading branch information
kkrik-es committed Aug 16, 2023
2 parents 78a51cf + 2f016e5 commit e9d097f
Show file tree
Hide file tree
Showing 733 changed files with 19,093 additions and 7,247 deletions.
15 changes: 15 additions & 0 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,18 @@ export GRADLE_BUILD_CACHE_USERNAME

GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
export GRADLE_BUILD_CACHE_PASSWORD

BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
export BUILDKITE_API_TOKEN

if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
data=$(.buildkite/scripts/lucene-snapshot/get-credentials.sh)

AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key)
export AWS_ACCESS_KEY_ID

AWS_SECRET_ACCESS_KEY=$(echo "$data" | jq -r .data.secret_key)
export AWS_SECRET_ACCESS_KEY

unset data
fi
21 changes: 21 additions & 0 deletions .buildkite/hooks/pre-command.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
@ECHO OFF

FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i

SET JAVA_HOME=%USERPROFILE%\.java\%ES_BUILD_JAVA%
SET JAVA16_HOME=%USERPROFILE%\.java\openjdk16

SET GRADLEW=./gradlew --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
SET GRADLEW_BAT=./gradlew.bat --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/

(if not exist "%USERPROFILE%/.gradle" mkdir "%USERPROFILE%/.gradle") && (echo. >> "%USERPROFILE%/.gradle/gradle.properties" && echo org.gradle.daemon=false >> "%USERPROFILE%/.gradle/gradle.properties")

set WORKSPACE=%cd%
set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
set COMPOSE_HTTP_TIMEOUT=120
set JOB_BRANCH=%BUILDKITE_BRANCH%

set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache

exit /b 0
22 changes: 22 additions & 0 deletions .buildkite/pipelines/lucene-snapshot/build-snapshot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
steps:
- trigger: apache-lucene-build-snapshot
label: Trigger pipeline to build lucene snapshot
key: lucene-build
if: build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == ""
- wait
- label: Upload and update lucene snapshot
command: .buildkite/scripts/lucene-snapshot/upload-snapshot.sh
timeout_in_minutes: 15
env:
USE_LUCENE_SNAPSHOT_CREDS: "true"
UPDATE_ES_LUCENE_SNAPSHOT: "true"
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- wait
- trigger: "elasticsearch-lucene-snapshot-tests"
build:
branch: "${BUILDKITE_BRANCH}"
async: true
60 changes: 60 additions & 0 deletions .buildkite/pipelines/lucene-snapshot/run-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
steps:
- label: sanity-check
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- wait: null
- label: part1
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: part2
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: part3
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- group: bwc-snapshots
steps:
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION:
- 7.17.13
- 8.9.1
- 8.10.0
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: "{{matrix.BWC_VERSION}}"
- label: rest-compat
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
9 changes: 9 additions & 0 deletions .buildkite/pipelines/lucene-snapshot/update-branch.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
steps:
- label: Update lucene-snapshot branch
command: .buildkite/scripts/lucene-snapshot/update-branch.sh
timeout_in_minutes: 15
- wait
- trigger: "elasticsearch-lucene-snapshot-tests"
build:
branch: "${BUILDKITE_BRANCH}"
async: true
47 changes: 47 additions & 0 deletions .buildkite/pipelines/periodic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,50 @@ steps:
env:
ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
- group: packaging-tests-windows
steps:
- label: "{{matrix.image}} / packaging-tests-windows"
command: |
.\.buildkite\scripts\run-script.ps1 .\.ci\scripts\packaging-test.ps1
timeout_in_minutes: 180
matrix:
setup:
image:
- windows-2016
- windows-2019
- windows-2022
agents:
provider: gcp
image: family/brian-elasticsearch-{{matrix.image}}
imageProject: elastic-images-qa
machineType: custom-32-98304
diskType: pd-ssd
diskSizeGb: 350
env: {}
- group: platform-support-windows
steps:
- label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-windows"
command: |
.\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
timeout_in_minutes: 360
matrix:
setup:
image:
- windows-2016
- windows-2019
- windows-2022
GRADLE_TASK:
- checkPart1
- checkPart2
- checkPart3
- bwcTestSnapshots
- checkRestCompat
agents:
provider: gcp
image: family/brian-elasticsearch-{{matrix.image}}
imageProject: elastic-images-qa
machineType: custom-32-98304
diskType: pd-ssd
diskSizeGb: 350
env:
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
16 changes: 16 additions & 0 deletions .buildkite/scripts/lucene-snapshot/get-credentials.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/bin/bash

set -euo pipefail

# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable.
# It should really only be used inside .buildkite/hooks/pre-command

VAULT_ROLE_ID=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
VAULT_SECRET_ID=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
VAULT_ADDR=https://secrets.elastic.co:8200

unset VAULT_TOKEN
VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
export VAULT_TOKEN

vault read -format=json aws-elastic/creds/lucene-snapshots
13 changes: 13 additions & 0 deletions .buildkite/scripts/lucene-snapshot/remove-verification-metadata.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import re

regex = re.compile(
r'<component group="org.apache.lucene" name="lucene-.*?</component>\s*',
re.MULTILINE | re.DOTALL,
)

with open("gradle/verification-metadata.xml", "r+") as f:
text = f.read()
text = regex.sub("", text)
f.seek(0)
f.truncate()
f.write(text)
18 changes: 18 additions & 0 deletions .buildkite/scripts/lucene-snapshot/update-branch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/bin/bash

set -euo pipefail

if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
echo "Error: This script should only be run on the lucene_snapshot branch"
exit 1
fi

echo --- Updating lucene_snapshot branch with main

git config --global user.name elasticsearchmachine
git config --global user.email '[email protected]'

git checkout lucene_snapshot
git fetch origin main
git merge --no-edit origin/main
git push origin lucene_snapshot
51 changes: 51 additions & 0 deletions .buildkite/scripts/lucene-snapshot/update-es-snapshot.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/bash

set -euo pipefail

if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
echo "Error: This script should only be run on the lucene_snapshot branch"
exit 1
fi

echo --- Update Lucene snapshot in Elasticsearch

LUCENE_SNAPSHOT_VERSION=${LUCENE_SNAPSHOT_VERSION:-}

if [[ -z "$LUCENE_SNAPSHOT_VERSION" ]]; then
LUCENE_SNAPSHOT_VERSION=$(buildkite-agent meta-data get lucene-snapshot-version)
fi

echo "Lucene Snapshot Version: $LUCENE_SNAPSHOT_VERSION"

git checkout "$BUILDKITE_BRANCH"
git pull --ff-only origin "$BUILDKITE_BRANCH"

# Replace `lucene = <version>` string in version.properties and maintain the same indentation
sed -E "s/^(lucene *= *[^ ]* *).*\$/\1$LUCENE_SNAPSHOT_VERSION/" build-tools-internal/version.properties > new-version.properties
mv new-version.properties build-tools-internal/version.properties

# Remove stale verification metadata, because generating them just appends new ones
python3 .buildkite/scripts/lucene-snapshot/remove-verification-metadata.py
./gradlew --write-verification-metadata sha256

# 9.8.0-snapshot-1f25c68 -> 9.8.0
VERSION=$(echo "$LUCENE_SNAPSHOT_VERSION" | cut -f 1 -d '-')
VERSION_SNAKE=$(echo "$VERSION" | sed -E 's/\./_/g')

sed -E "s/^(:lucene_version: *).*\$/\1$VERSION/" docs/Versions.asciidoc > docs/Versions.asciidoc.new
sed -E "s/^(:lucene_version_path: *).*\$/\1$VERSION_SNAKE/" docs/Versions.asciidoc.new > docs/Versions.asciidoc
rm -f docs/Versions.asciidoc.new

if git diff-index --quiet HEAD --; then
echo 'No changes to commit.'
else
git config --global user.name elasticsearchmachine
git config --global user.email '[email protected]'

git add build-tools-internal/version.properties
git add gradle/verification-metadata.xml
git add docs/Versions.asciidoc

git commit -m "[Automated] Update Lucene snapshot to $LUCENE_SNAPSHOT_VERSION"
git push origin "$BUILDKITE_BRANCH"
fi
41 changes: 41 additions & 0 deletions .buildkite/scripts/lucene-snapshot/upload-snapshot.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#!/bin/bash

set -euo pipefail

LUCENE_BUILD_ID=${LUCENE_BUILD_ID:-}

if [[ -z "$LUCENE_BUILD_ID" ]]; then
build_json=$(curl -sH "Authorization: Bearer $BUILDKITE_API_TOKEN" "https://api.buildkite.com/v2/organizations/elastic/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER")
LUCENE_BUILD_ID=$(jq -r '.jobs[] | select(.step_key == "lucene-build").triggered_build.id' <<< "$build_json")
fi

export LUCENE_BUILD_ID

LUCENE_SHA=$(buildkite-agent meta-data get --build "$LUCENE_BUILD_ID" lucene-snapshot-sha)
export LUCENE_SHA

LUCENE_SNAPSHOT_VERSION=$(buildkite-agent meta-data get --build "$LUCENE_BUILD_ID" lucene-snapshot-version)
export LUCENE_SNAPSHOT_VERSION

echo --- Downloading lucene snapshot

mkdir lucene-snapshot
cd lucene-snapshot
buildkite-agent artifact download --build "$LUCENE_BUILD_ID" lucene-snapshot.tar.gz .
tar -xvf lucene-snapshot.tar.gz
cd -

echo --- Upload lucene snapshot to S3

if ! which aws; then
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
rm -rf awscliv2.zip aws
fi

aws s3 sync lucene-snapshot/ "s3://download.elasticsearch.org/lucenesnapshots/$LUCENE_SHA/" --acl public-read

if [[ "${UPDATE_ES_LUCENE_SNAPSHOT:-}" ]]; then
.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh
fi
Loading

0 comments on commit e9d097f

Please sign in to comment.