diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 34321b61161..798939bbf32 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -1,5 +1,137 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + steps: - - label: "Example test" - command: echo "Hello!" + - group: "Auditbeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":ubuntu: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Auditbeat: linux/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":rhel: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Auditbeat: rhel/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows:-{{matrix.image}} Unit Tests" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Auditbeat: windows/Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":linux: Crosscompile" + command: + - ".buildkite/auditbeat/scripts/crosscompile.sh" + env: + GOX_FLAGS: "-arch amd64" + notify: + - github_commit_status: + context: "Auditbeat: Crosscompile" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - group: "Extended Testing" + key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for extended support" + + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Auditbeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" + artifact_paths: "auditbeat/build/*.xml" + + - label: ":mac: MacOS Unit Tests" + key: "macos-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Auditbeat/Extended: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "auditbeat/build/*.xml" + + - group: "Windows Extended Testing" + key: "extended-tests-win" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Auditbeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + + steps: + - label: Package pipeline + commands: ".buildkite/auditbeat/scripts/package-step.sh" diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh new file mode 100755 index 00000000000..866d6be4223 --- /dev/null +++ b/.buildkite/auditbeat/scripts/crosscompile.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Crosscompile" +make -C auditbeat crosscompile diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh new file mode 100755 index 00000000000..cb06895879a --- /dev/null +++ b/.buildkite/auditbeat/scripts/package-step.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +changeset="^auditbeat/ +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/auditbeat/" + +if are_files_changed "$changeset"; then + bk_pipeline=$(cat <<-YAML + steps: + - label: ":ubuntu: Packaging Linux X86" + key: "package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + command: + - ".buildkite/auditbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Auditbeat/Packaging: Linux X86" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: + - ".buildkite/auditbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Auditbeat/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" +YAML +) + echo "${bk_pipeline}" | buildkite-agent pipeline upload +else + buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' + exit 0 +fi diff --git a/.buildkite/auditbeat/scripts/package.sh b/.buildkite/auditbeat/scripts/package.sh new file mode 100755 index 00000000000..71872ca15a3 --- /dev/null +++ b/.buildkite/auditbeat/scripts/package.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Docker Version: $(docker --version)" + +echo "--- Start Packaging" +cd auditbeat +umask 0022 +mage package + diff --git a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 new file mode 100644 index 00000000000..200627d518f --- /dev/null +++ b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually + +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF() { + Write-Host "--- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function withGolang() { + Write-Host "--- Install golang $GoVersion --" + choco install golang -y --version $GoVersion + + $choco = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$choco\helpers\chocolateyProfile.psm1" + refreshenv + go version + go env +} + +function installGoDependencies() { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "github.com/tebeka/go2xunit" + ) + foreach ($pkg in $installPackages) { + go install "$pkg" + } +} + +fixCRLF + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path auditbeat +New-Item -ItemType Directory -Force -Path "build" +withGolang +installGoDependencies + +mage build unitTest + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh new file mode 100755 index 00000000000..c1f5685c77f --- /dev/null +++ b/.buildkite/auditbeat/scripts/unit-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Running Unit Tests" +sudo chmod -R go-w auditbeat/ + +cd auditbeat +umask 0022 +mage build unitTest diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh new file mode 100644 index 00000000000..4dfc01bafc3 --- /dev/null +++ b/.buildkite/env-scripts/env.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +SETUP_GVM_VERSION="v0.5.1" +WORKSPACE="$(pwd)" +BIN="${WORKSPACE}/bin" +HW_TYPE="$(uname -m)" +PLATFORM_TYPE="$(uname)" +REPO="beats" +TMP_FOLDER="tmp.${REPO}" +DOCKER_REGISTRY="docker.elastic.co" + +export SETUP_GVM_VERSION +export WORKSPACE +export BIN +export HW_TYPE +export PLATFORM_TYPE +export REPO +export TMP_FOLDER +export DOCKER_REGISTRY diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh new file mode 100644 index 00000000000..1365aaace4a --- /dev/null +++ b/.buildkite/env-scripts/linux-env.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +DEBIAN_FRONTEND="noninteractive" + +sudo mkdir -p /etc/needrestart +echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + +if [[ $PLATFORM_TYPE == "Linux" ]]; then + # Remove this code once beats specific agent is set up + if grep -q 'Ubuntu' /etc/*release; then + export DEBIAN_FRONTEND + + echo "--- Ubuntu - Installing libs" + sudo apt-get update + sudo apt-get install -y libsystemd-dev + sudo apt install -y python3-pip + sudo apt-get install -y python3-venv + fi + + # Remove this code once beats specific agent is set up + if grep -q 'Red Hat' /etc/*release; then + echo "--- RHL - Installing libs" + sudo yum update -y + sudo yum install -y systemd-devel + sudo yum install -y python3-pip + sudo yum install -y python3 + pip3 install virtualenv + fi +fi + +if [[ $PLATFORM_TYPE == Darwin* ]]; then + echo "--- Setting larger ulimit on MacOS" + # To bypass file descriptor errors like "Too many open files error" on MacOS + ulimit -Sn 50000 + echo "--- ULIMIT: $(ulimit -n)" +fi + +echo "--- Setting up environment" +add_bin_path +with_go +with_mage diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh new file mode 100644 index 00000000000..6a5c36bcd04 --- /dev/null +++ b/.buildkite/env-scripts/util.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +set -euo pipefail + +add_bin_path() { + echo "Adding PATH to the environment variables..." + create_bin + export PATH="${PATH}:${BIN}" +} + +with_go() { + local go_version="${GOLANG_VERSION}" + echo "Setting up the Go environment..." + create_bin + check_platform_architecture + retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}" + export PATH="${PATH}:${BIN}" + chmod +x ${BIN}/gvm + eval "$(gvm "$go_version")" + go version + which go + export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin" +} + +with_mage() { + local install_packages=( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "gotest.tools/gotestsum" + ) + create_bin + for pkg in "${install_packages[@]}"; do + go install "${pkg}@latest" + done +} + +create_bin() { + if [[ ! -d "${BIN}" ]]; then + mkdir -p ${BIN} + fi +} + +check_platform_architecture() { +# for downloading the GVM and Terraform packages + case "${HW_TYPE}" in + "x86_64") + arch_type="amd64" + ;; + "aarch64") + arch_type="arm64" + ;; + "arm64") + arch_type="arm64" + ;; + *) + echo "The current platform/OS type is unsupported yet" + ;; + esac +} + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +are_files_changed() { + local changeset=$1 + + if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then + return 0; + else + return 1; + fi +} + +cleanup() { + echo "Deleting temporary files..." + rm -rf ${BIN}/${TMP_FOLDER}.* + echo "Done." +} + +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then + unset "$var" + fi + done +} diff --git a/.buildkite/env-scripts/win-env.sh b/.buildkite/env-scripts/win-env.sh new file mode 100644 index 00000000000..931051d550a --- /dev/null +++ b/.buildkite/env-scripts/win-env.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if [[ ${PLATFORM_TYPE} = MINGW* ]]; then + echo "--- Installing Python on Win" + choco install mingw -y + choco install python --version=3.11.0 -y +fi diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 34321b61161..e811d286953 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -1,5 +1,140 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + steps: - - label: "Example test" - command: echo "Hello!" + - group: "Filebeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":ubuntu: Unit Tests" + command: + - ".buildkite/filebeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Filebeat: linux/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Go Integration Tests" + command: + - ".buildkite/filebeat/scripts/integration-gotests.sh" + notify: + - github_commit_status: + context: "Filebeat: Go Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Python Integration Tests" + command: + - ".buildkite/filebeat/scripts/integration-pytests.sh" + notify: + - github_commit_status: + context: "Filebeat: Python Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":windows:-{{matrix.image}} Unit Tests" + command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Filebeat: windows/Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - group: "Extended Testing" + key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support" + + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: + - ".buildkite/filebeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Filebeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" + artifact_paths: "filebeat/build/*.xml" + + - label: ":mac: MacOS Unit Tests" + key: "macos-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: + - ".buildkite/filebeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Filebeat/Extended: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "filebeat/build/*.xml" + + - group: "Windows Extended Testing" + key: "extended-tests-win" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Filebeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + + steps: + - label: Package pipeline + commands: ".buildkite/filebeat/scripts/package-step.sh" diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh new file mode 100755 index 00000000000..d64ce7c98eb --- /dev/null +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Integration Tests" +sudo chmod -R go-w filebeat/ + +cd filebeat +umask 0022 +mage goIntegTest diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh new file mode 100755 index 00000000000..b51e8ae18a6 --- /dev/null +++ b/.buildkite/filebeat/scripts/integration-pytests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Integration Tests" +sudo chmod -R go-w filebeat/ + +cd filebeat +umask 0022 +mage pythonIntegTest diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh new file mode 100755 index 00000000000..f8fa02db81d --- /dev/null +++ b/.buildkite/filebeat/scripts/package-step.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +changeset="^filebeat/ +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/filebeat/" + +if are_files_changed "$changeset"; then + bk_pipeline=$(cat <<-YAML + steps: + - label: ":ubuntu: Packaging Linux X86" + key: "package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + command: + - ".buildkite/filebeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Filebeat/Packaging: Linux X86" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: + - ".buildkite/filebeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Filebeat/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" +YAML +) + echo "${bk_pipeline}" | buildkite-agent pipeline upload +else + buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' + exit 0 +fi diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh new file mode 100755 index 00000000000..0bb03250348 --- /dev/null +++ b/.buildkite/filebeat/scripts/package.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Start Packaging" +cd filebeat +umask 0022 +mage package diff --git a/.buildkite/filebeat/scripts/unit-tests-win.ps1 b/.buildkite/filebeat/scripts/unit-tests-win.ps1 new file mode 100644 index 00000000000..8990eb30a09 --- /dev/null +++ b/.buildkite/filebeat/scripts/unit-tests-win.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually + +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF() { + Write-Host "-- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function withGolang() { + Write-Host "-- Install golang $GoVersion --" + choco install golang -y --version $GoVersion + + $choco = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$choco\helpers\chocolateyProfile.psm1" + refreshenv + go version + go env +} + +function installGoDependencies() { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "github.com/tebeka/go2xunit" + ) + foreach ($pkg in $installPackages) { + go install "$pkg" + } +} + +fixCRLF + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path filebeat +New-Item -ItemType Directory -Force -Path "build" +withGolang +installGoDependencies + +mage build unitTest + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh new file mode 100755 index 00000000000..08ce9d4ea1c --- /dev/null +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Unit Tests" +sudo chmod -R go-w filebeat/ + +umask 0022 +mage -d filebeat unitTest diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 34321b61161..bf645a2b295 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -1,5 +1,142 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + steps: - - label: "Example test" - command: echo "Hello!" + - group: "Heartbeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":linux: Unit Tests / {{matrix.image}}" + command: + - ".buildkite/heartbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Heartbeat: linux/Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + matrix: + setup: + image: + - "${IMAGE_UBUNTU_X86_64}" + - "${IMAGE_RHEL9}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":windows: Unit Tests / {{matrix.image}}" + command: + - ".buildkite/heartbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Heartbeat: windows/Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":ubuntu: Go Integration Tests" + command: + - ".buildkite/heartbeat/scripts/integration-gotests.sh" + notify: + - github_commit_status: + context: "Heartbeat: Go Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":ubuntu: Python Integration Tests" + command: + - ".buildkite/heartbeat/scripts/integration-pytests.sh" + notify: + - github_commit_status: + context: "Heartbeat: Python Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - group: "Extended Testing" + key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for extended support" + + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: + - ".buildkite/heartbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Heartbeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" + artifact_paths: "heartbeat/build/*.xml" + + - label: ":mac: MacOS Unit Tests" + key: "macos-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: + - ".buildkite/heartbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Heartbeat/Extended: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "heartbeat/build/*.xml" + + - group: "Windows Extended Testing" + key: "extended-tests-win" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/heartbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Heartbeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_type: "pd-ssd" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + + steps: + - label: Package pipeline + commands: ".buildkite/heartbeat/scripts/package-step.sh" diff --git a/.buildkite/heartbeat/scripts/integration-gotests.sh b/.buildkite/heartbeat/scripts/integration-gotests.sh new file mode 100755 index 00000000000..8eab0e8b5d8 --- /dev/null +++ b/.buildkite/heartbeat/scripts/integration-gotests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Remove when custom image is set up +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Integration Tests" +# Remove when custom image is set up +sudo chmod -R go-w heartbeat/ + +cd heartbeat +# Remove when custom image is set up +umask 0022 +mage goIntegTest diff --git a/.buildkite/heartbeat/scripts/integration-pytests.sh b/.buildkite/heartbeat/scripts/integration-pytests.sh new file mode 100755 index 00000000000..729df5ae6f6 --- /dev/null +++ b/.buildkite/heartbeat/scripts/integration-pytests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Remove when custom image is set up +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Integration Tests" +# Remove when custom image is set up +sudo chmod -R go-w heartbeat/ + +cd heartbeat +# Remove when custom image is set up +umask 0022 +mage pythonIntegTest diff --git a/.buildkite/heartbeat/scripts/package-step.sh b/.buildkite/heartbeat/scripts/package-step.sh new file mode 100755 index 00000000000..03790edfa5f --- /dev/null +++ b/.buildkite/heartbeat/scripts/package-step.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +changeset="^heartbeat/ +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/heartbeat/" + +if are_files_changed "$changeset"; then + bk_pipeline=$(cat <<-YAML + steps: + - label: ":ubuntu: Packaging Linux X86" + key: "package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + command: + - ".buildkite/heartbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "heartbeat/Packaging: Linux X86" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: + - ".buildkite/heartbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "heartbeat/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" +YAML +) + echo "${bk_pipeline}" | buildkite-agent pipeline upload +else + buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' + exit 0 +fi diff --git a/.buildkite/heartbeat/scripts/package.sh b/.buildkite/heartbeat/scripts/package.sh new file mode 100755 index 00000000000..7f51a6b5ca1 --- /dev/null +++ b/.buildkite/heartbeat/scripts/package.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Docker Version: $(docker --version)" + +echo "--- Start Packaging" +cd heartbeat +umask 0022 +mage package + diff --git a/.buildkite/heartbeat/scripts/unit-tests-win.ps1 b/.buildkite/heartbeat/scripts/unit-tests-win.ps1 new file mode 100644 index 00000000000..17282813e13 --- /dev/null +++ b/.buildkite/heartbeat/scripts/unit-tests-win.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually + +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF() { + Write-Host "--- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function withGolang() { + Write-Host "--- Install golang $GoVersion --" + choco install golang -y --version $GoVersion + + $choco = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$choco\helpers\chocolateyProfile.psm1" + refreshenv + go version + go env +} + +function installGoDependencies() { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "github.com/tebeka/go2xunit" + ) + foreach ($pkg in $installPackages) { + go install "$pkg" + } +} + +fixCRLF + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path heartbeat +New-Item -ItemType Directory -Force -Path "build" +withGolang +installGoDependencies + +mage build unitTest + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/heartbeat/scripts/unit-tests.sh b/.buildkite/heartbeat/scripts/unit-tests.sh new file mode 100755 index 00000000000..4b746da2d57 --- /dev/null +++ b/.buildkite/heartbeat/scripts/unit-tests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Remove when custom image is set up +source .buildkite/env-scripts/linux-env.sh + +echo "--- Running Unit Tests" +# Remove when custom image is set up +sudo chmod -R go-w heartbeat/ + +cd heartbeat +# Remove when custom image is set up +umask 0022 +mage build unitTest diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout new file mode 100644 index 00000000000..b6cc7ad60bd --- /dev/null +++ b/.buildkite/hooks/post-checkout @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -euo pipefail + +checkout_merge() { + local target_branch=$1 + local pr_commit=$2 + local merge_branch=$3 + + if [[ -z "${target_branch}" ]]; then + echo "No pull request target branch" + exit 1 + fi + + git fetch -v origin "${target_branch}" + git checkout FETCH_HEAD + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # create temporal branch to merge the PR with the target branch + git checkout -b ${merge_branch} + echo "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # set author identity so it can be run git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit "${BUILDKITE_COMMIT}" || { + local merge_result=$? + echo "Merge failed: ${merge_result}" + git merge --abort + exit ${merge_result} + } +} + +pull_request="${BUILDKITE_PULL_REQUEST:-false}" + +if [[ "${pull_request}" == "false" ]]; then + echo "Not a pull request, skipping" + exit 0 +fi + +TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}" +PR_COMMIT="${BUILDKITE_COMMIT}" +PR_ID=${BUILDKITE_PULL_REQUEST} +MERGE_BRANCH="pr_merge_${PR_ID}" + +checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" + +echo "Commit information" +git --no-pager log --format=%B -n 1 + +# Ensure buildkite groups are rendered +echo "" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 00000000000..0ac7c51099c --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then + source .buildkite/env-scripts/env.sh + source .buildkite/env-scripts/util.sh + source .buildkite/env-scripts/win-env.sh + + if [[ -z "${GOLANG_VERSION-""}" ]]; then + export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") + fi +fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then + source .buildkite/scripts/setenv.sh + if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then + echo "Skipped pre-command when running the Upload pipeline" + exit 0 + fi +fi diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 34321b61161..83fda5662c7 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -1,5 +1,45 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-libbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "libbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Libbeat - run_libbeat" + key: "run_libbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Libbeat - run_libbeat_arm_tests" + key: "run_libbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic Libbeat pipeline" + key: "libbeat-pipeline" + command: ".buildkite/scripts/generate_libbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 34321b61161..04f3b44575e 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -1,5 +1,52 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-metricbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + BEATS_PROJECT_NAME: "metricbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "run_metricbeat" + fields: + - select: "Metricbeat - run_metricbeat" + key: "run_metricbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Metricbeat - run_metricbeat_macos_tests" + key: "run_metricbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic metricbeat pipeline" + key: "metricbeat-pipeline" + command: ".buildkite/scripts/generate_metricbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 34321b61161..8e606e16622 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -1,5 +1,60 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-packetbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_RHEL9_X86_64: "family/core-rhel-9" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "packetbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Packetbeat - run_packetbeat" + key: "run_packetbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_packetbeat_arm_tests" + key: "run_packetbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_packetbeat_macos_tests" + key: "run_packetbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic packetbeat pipeline" + key: "packetbeat-pipeline" + command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index cc8ff9ab7a5..8018411a743 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -25,12 +25,12 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test filebeat$", - "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test filebeat$", + "trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", + "always_trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/package filebeat$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ ] + "always_require_ci_on_changed": ["^filebeat/.*", ".buildkite/filebeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*" ] }, { "enabled": true, @@ -57,8 +57,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/test auditbeat$", - "always_trigger_comment_regex": "^/test auditbeat$", + "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$", + "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], @@ -73,8 +73,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/test heartbeat$", - "always_trigger_comment_regex": "^/test heartbeat$", + "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$", + "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], @@ -127,6 +127,102 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-elastic-agent", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test elastic-agent$", + "always_trigger_comment_regex": "^/test elastic-agent$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ], + "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-winlogbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test winlogbeat$", + "always_trigger_comment_regex": "^/test winlogbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-winlogbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/winlogbeat$", + "always_trigger_comment_regex": "^/test x-pack/winlogbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/winlogbeat/.*", ".buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-packetbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/packetbeat$", + "always_trigger_comment_regex": "^/test x-pack/packetbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-libbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/libbeat$", + "always_trigger_comment_regex": "^/test x-pack/libbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-xpack-metricbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test x-pack/metricbeat$", + "always_trigger_comment_regex": "^/test x-pack/metricbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/metricbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] } diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh new file mode 100755 index 00000000000..b797ec718aa --- /dev/null +++ b/.buildkite/scripts/common.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash +set -euo pipefail + +WORKSPACE=${WORKSPACE:-"$(pwd)"} +BIN="${WORKSPACE}/bin" +platform_type="$(uname)" +platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') +arch_type="$(uname -m)" +GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} +GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} +ONLY_DOCS=${ONLY_DOCS:-"true"} +[ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" +[ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" +[ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" +[ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" +[ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" +[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" +[ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" +[ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" + +metricbeat_changeset=( + "^metricbeat/.*" + ) + +libbeat_changeset=( + "^libbeat/.*" + ) + +packetbeat_changeset=( + "^packetbeat/.*" + ) + +winlogbeat_changeset=( + "^winlogbeat/.*" + ) + +xpack_libbeat_changeset=( + "^x-pack/libbeat/.*" + ) + +xpack_metricbeat_changeset=( + "^x-pack/metricbeat/.*" + ) + +xpack_packetbeat_changeset=( + "^x-pack/packetbeat/.*" + ) + +xpack_winlogbeat_changeset=( + "^x-pack/winlogbeat/.*" + ) + +ci_changeset=( + "^.buildkite/.*" + ) + +go_mod_changeset=( + "^go.mod" + ) + +oss_changeset=( + "^go.mod" + "^pytest.ini" + "^dev-tools/.*" + "^libbeat/.*" + "^testing/.*" +) + +xpack_changeset=( + "${xpack_libbeat_changeset[@]}" + "${oss_changeset[@]}" +) + +docs_changeset=( + ".*\\.(asciidoc|md)" + "deploy/kubernetes/.*-kubernetes\\.yaml" + ) + +packaging_changeset=( + "^dev-tools/packaging/.*" + ".go-version" + ) + +check_and_set_beat_vars() { + if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then + BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - + BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _ + BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels + BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} + TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" + TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" + declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset" + echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" + mandatory_changeset=( + "${BEAT_CHANGESET_REFERENCE[@]}" + "${xpack_changeset[@]}" + "${ci_changeset[@]}" + ) + else + BEATS_GH_LABEL=${BEATS_PROJECT_NAME} + TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" + TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" + declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset" + echo "Beats project name is $BEATS_PROJECT_NAME" + mandatory_changeset=( + "${BEAT_CHANGESET_REFERENCE[@]}" + "${oss_changeset[@]}" + "${ci_changeset[@]}" + ) + fi + BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" + BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" + BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" + BAETS_GH_MACOS_LABEL="macOS" + BAETS_GH_ARM_LABEL="arm" +} + +with_docker_compose() { + local version=$1 + echo "Setting up the Docker-compose environment..." + create_workspace + retry 3 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" + chmod +x ${BIN}/docker-compose + export PATH="${BIN}:${PATH}" + docker-compose version +} + +create_workspace() { + if [[ ! -d "${BIN}" ]]; then + mkdir -p "${BIN}" + fi +} + +add_bin_path() { + echo "Adding PATH to the environment variables..." + create_workspace + export PATH="${BIN}:${PATH}" +} + +check_platform_architeture() { + case "${arch_type}" in + "x86_64") + go_arch_type="amd64" + ;; + "aarch64") + go_arch_type="arm64" + ;; + "arm64") + go_arch_type="arm64" + ;; + *) + echo "The current platform or OS type is unsupported yet" + ;; + esac +} + +with_mage() { + local install_packages=( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "gotest.tools/gotestsum" + ) + create_workspace + for pkg in "${install_packages[@]}"; do + go install "${pkg}@latest" + done +} + +with_go() { + echo "Setting up the Go environment..." + create_workspace + check_platform_architeture + retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${go_arch_type}" + chmod +x "${BIN}/gvm" + eval "$(gvm $GO_VERSION)" + go version + which go + local go_path="$(go env GOPATH):$(go env GOPATH)/bin" + export PATH="${go_path}:${PATH}" +} + +checkLinuxType() { + if [ "${platform_type}" == "Linux" ]; then + if grep -q 'ubuntu' /etc/os-release; then + echo "ubuntu" + elif grep -q 'rhel' /etc/os-release; then + echo "rhel" + else + echo "Unsupported Linux" + fi + else + echo "This is not a Linux" + fi +} + +with_python() { + local linuxType="$(checkLinuxType)" + echo "${linuxType}" + if [ "${platform_type}" == "Linux" ]; then + if [ "${linuxType}" = "ubuntu" ]; then + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv + elif [ "${linuxType}" = "rhel" ]; then + sudo dnf update -y + sudo dnf install -y python3 python3-pip + pip3 install virtualenv + fi + elif [ "${platform_type}" == "Darwin" ]; then + brew update + pip3 install virtualenv + ulimit -Sn 10000 + fi +} + +with_dependencies() { + local linuxType="$(checkLinuxType)" + echo "${linuxType}" + if [ "${platform_type}" == "Linux" ]; then + if [ "${linuxType}" = "ubuntu" ]; then + sudo apt-get update + sudo apt-get install -y libsystemd-dev libpcap-dev + elif [ "${linuxType}" = "rhel" ]; then + # sudo dnf update -y + sudo dnf install -y systemd-devel + wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image + sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image + fi + elif [ "${platform_type}" == "Darwin" ]; then + pip3 install libpcap + fi +} + +config_git() { + if [ -z "$(git config --get user.email)" ]; then + git config --global user.email "beatsmachine@users.noreply.github.com" + git config --global user.name "beatsmachine" + fi +} + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +are_paths_changed() { + local patterns=("${@}") + local changelist=() + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +are_changed_only_paths() { + local patterns=("${@}") + local changelist=() + local changed_files=$(git diff --name-only HEAD@{1} HEAD) + if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + return 0 + fi + return 1 +} + +are_conditions_met_mandatory_tests() { + if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then + return 0 + fi + return 1 +} + +are_conditions_met_arm_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then + return 0 + fi + fi + fi + return 1 +} + +are_conditions_met_macos_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + fi + fi + fi + return 1 +} + +are_conditions_met_packaging() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then + return 0 + fi + fi + return 1 +} + +config_git() { + if [ -z "$(git config --get user.email)" ]; then + git config --global user.email "beatsmachine@users.noreply.github.com" + git config --global user.name "beatsmachine" + fi +} + +if ! are_changed_only_paths "${docs_changeset[@]}" ; then + ONLY_DOCS="false" + echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." +else + echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." +fi + +if are_paths_changed "${go_mod_changeset[@]}" ; then + GO_MOD_CHANGES="true" +fi + +if are_paths_changed "${packaging_changeset[@]}" ; then + PACKAGING_CHANGES="true" +fi + +check_and_set_beat_vars diff --git a/.buildkite/scripts/crosscompile.sh b/.buildkite/scripts/crosscompile.sh new file mode 100755 index 00000000000..12f0f6574ca --- /dev/null +++ b/.buildkite/scripts/crosscompile.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Crosscompile for $BEATS_PROJECT_NAME" +make -C "${BEATS_PROJECT_NAME}" crosscompile diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh new file mode 100755 index 00000000000..0674d0b186b --- /dev/null +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.libbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-cross-compile" + command: ".buildkite/scripts/crosscompile.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":testengine: Stress Tests" + key: "mandatory-stress-test" + command: ".buildkite/scripts/stress_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":linux: Arm64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh new file mode 100755 index 00000000000..e91896eb70c --- /dev/null +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.metricbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":go: Go Intergration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-cross-compile" + command: ".buildkite/scripts/crosscompile.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# echo "Add the extended windows tests into the pipeline" +# TODO: ADD conditions from the main pipeline + + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +YAML + +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh new file mode 100755 index 00000000000..89ea7a33e20 --- /dev/null +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.packetbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":rhel: RHEL-9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - group: "Extended Windowds Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + + +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh new file mode 100755 index 00000000000..1eb1b459c92 --- /dev/null +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.winlogbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-cross-compile" + command: ".buildkite/scripts/crosscompile.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2019}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# echo "Add the extended windows tests into the pipeline" +# TODO: ADD conditions from the main pipeline + + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh new file mode 100755 index 00000000000..66f0750ab6f --- /dev/null +++ b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.libbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":linux: Arm64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh new file mode 100755 index 00000000000..b4c519f4512 --- /dev/null +++ b/.buildkite/scripts/go_int_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Go Intergration Tests for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage goIntegTest + +popd > /dev/null diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh new file mode 100755 index 00000000000..80e70ae96c5 --- /dev/null +++ b/.buildkite/scripts/install_tools.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +echo "--- Env preparation" + +# Temporary solution to fix the issues with "sudo apt get...." https://elastic.slack.com/archives/C0522G6FBNE/p1706003603442859?thread_ts=1706003209.424539&cid=C0522G6FBNE +# It could be removed when we use our own image for the BK agent. +if [ "${platform_type}" == "Linux" ]; then + if [ "${platform_type}" == "Linux" ]; then + if [ $(checkLinuxType) = "ubuntu" ]; then + DEBIAN_FRONTEND="noninteractive" + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo mkdir -p /etc/needrestart + echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + fi + fi +fi + +add_bin_path + +if command -v docker-compose &> /dev/null +then + echo "Found docker-compose. Checking version.." + FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version | awk '{print $4}'|sed s/\,//) + if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then + echo "Versions match. No need to install docker-compose. Exiting." + elif [[ "${platform_type}" == "Linux" && "${arch_type}" == "aarch64" ]]; then + with_docker_compose "${DOCKER_COMPOSE_VERSION_AARCH64}" + elif [[ "${platform_type}" == "Linux" && "${arch_type}" == "x86_64" ]]; then + with_docker_compose "${DOCKER_COMPOSE_VERSION}" + fi +else + with_docker_compose "${DOCKER_COMPOSE_VERSION}" +fi + +with_go "${GO_VERSION}" +with_mage +with_python +with_dependencies +config_git +mage dumpVariables + +#sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) +sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #TODO: Remove when the issue is solved https://github.com/elastic/beats/issues/37838 + +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +#TODO "umask 0022" has to be removed after our own image is ready (it has to be moved to the image) +umask 0022 # fix the filesystem permissions issue like this: https://buildkite.com/elastic/beats-metricbeat/builds/1329#018d3179-25a9-475b-a2c8-64329dfe092b/320-1696 + +popd > /dev/null diff --git a/.buildkite/scripts/packaging.sh b/.buildkite/scripts/packaging.sh new file mode 100755 index 00000000000..1539d3ab430 --- /dev/null +++ b/.buildkite/scripts/packaging.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Packaging for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage package + +popd > /dev/null diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh new file mode 100755 index 00000000000..19fa8796c3e --- /dev/null +++ b/.buildkite/scripts/py_int_tests.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Python Intergration Tests for $BEATS_PROJECT_NAME" + +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage pythonIntegTest + +popd > /dev/null diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh new file mode 100755 index 00000000000..25121de212f --- /dev/null +++ b/.buildkite/scripts/setenv.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SETUP_GVM_VERSION="v0.5.1" +DOCKER_COMPOSE_VERSION="1.21.0" +DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" +SETUP_WIN_PYTHON_VERSION="3.11.0" +NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) +GO_VERSION=$(cat .go-version) + +export SETUP_GVM_VERSION +export DOCKER_COMPOSE_VERSION +export DOCKER_COMPOSE_VERSION_AARCH64 +export SETUP_WIN_PYTHON_VERSION +export NMAP_WIN_VERSION +export GO_VERSION + +exportVars() { + local platform_type="$(uname)" + local arch_type="$(uname -m)" + if [ "${arch_type}" == "x86_64" ]; then + case "${platform_type}" in + Linux|Darwin) + export GOX_FLAGS="-arch amd64" + export testResults="**/build/TEST*.xml" + export artifacts="**/build/TEST*.out" + ;; + MINGW*) + export GOX_FLAGS="-arch 386" + export testResults="**\\build\\TEST*.xml" + export artifacts="**\\build\\TEST*.out" + ;; + esac + elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then + export GOX_FLAGS="-arch arm" + export testResults="**/build/TEST*.xml" + export artifacts="**/build/TEST*.out" + else + echo "Unsupported OS" + fi +} + + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + exportVars + export RACE_DETECTOR="true" + export TEST_COVERAGE="true" + export DOCKER_PULL="0" +fi diff --git a/.buildkite/scripts/stress_tests.sh b/.buildkite/scripts/stress_tests.sh new file mode 100755 index 00000000000..b177eb53ea6 --- /dev/null +++ b/.buildkite/scripts/stress_tests.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Stress Tests for $BEATS_PROJECT_NAME" + +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' stress-tests + +popd > /dev/null diff --git a/.buildkite/scripts/unit_tests.sh b/.buildkite/scripts/unit_tests.sh new file mode 100755 index 00000000000..059b4166e29 --- /dev/null +++ b/.buildkite/scripts/unit_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Unit Tests" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage build unitTest + +popd > /dev/null diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 new file mode 100644 index 00000000000..b3c5c58fac0 --- /dev/null +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -0,0 +1,154 @@ +$ErrorActionPreference = "Stop" # set -e +$WorkFolder = $env:BEATS_PROJECT_NAME +$WORKSPACE = Get-Location +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF { + Write-Host "-- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function retry { + param( + [int]$retries, + [ScriptBlock]$scriptBlock + ) + $count = 0 + while ($count -lt $retries) { + $count++ + try { + & $scriptBlock + return + } catch { + $exitCode = $_.Exception.ErrorCode + Write-Host "Retry $count/$retries exited $exitCode, retrying..." + Start-Sleep -Seconds ([Math]::Pow(2, $count)) + } + } + Write-Host "Retry $count/$retries exited, no more retries left." +} + +function verifyFileChecksum { + param ( + [string]$filePath, + [string]$checksumFilePath + ) + $actualHash = (Get-FileHash -Algorithm SHA256 -Path $filePath).Hash + $checksumData = Get-Content -Path $checksumFilePath + $expectedHash = ($checksumData -split "\s+")[0] + if ($actualHash -eq $expectedHash) { + Write-Host "CheckSum is checked. File is correct. Original checkSum is: $expectedHash " + return $true + } else { + Write-Host "CheckSum is wrong. File can be corrupted or modified. Current checksum is: $actualHash, the original checksum is: $expectedHash" + return $false + } +} + +function withGolang($version) { + Write-Host "-- Installing Go $version --" + $goDownloadPath = Join-Path $env:TEMP "go_installer.msi" + $goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -Uri $goInstallerUrl -OutFile $goDownloadPath + } + Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $goDownloadPath /quiet" -Wait + $env:GOPATH = "${env:ProgramFiles}\Go" + $env:GOBIN = "${env:GOPATH}\bin" + $env:Path += ";$env:GOPATH;$env:GOBIN" + go version + installGoDependencies +} + +function withPython($version) { + Write-Host "-- Installing Python $version --" + [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" + $pyDownloadPath = Join-Path $env:TEMP "python-$version-amd64.exe" + $pyInstallerUrl = "https://www.python.org/ftp/python/$version/python-$version-amd64.exe" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -UseBasicParsing -Uri $pyInstallerUrl -OutFile $pyDownloadPath + } + Start-Process -FilePath $pyDownloadPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0" -Wait + $pyBinPath = "${env:ProgramFiles}\Python311" + $env:Path += ";$pyBinPath" + python --version +} + +function withMinGW { + Write-Host "-- Installing MinGW --" + [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" + $gwInstallerUrl = "https://github.com/brechtsanders/winlibs_mingw/releases/download/12.1.0-14.0.6-10.0.0-ucrt-r3/winlibs-x86_64-posix-seh-gcc-12.1.0-llvm-14.0.6-mingw-w64ucrt-10.0.0-r3.zip" + $gwInstallerCheckSumUrl = "$gwInstallerUrl.sha256" + $gwDownloadPath = "$env:TEMP\winlibs-x86_64.zip" + $gwDownloadCheckSumPath = "$env:TEMP\winlibs-x86_64.zip.sha256" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -Uri $gwInstallerUrl -OutFile $gwDownloadPath + Invoke-WebRequest -Uri $gwInstallerCheckSumUrl -OutFile $gwDownloadCheckSumPath + } + $comparingResult = verifyFileChecksum -filePath $gwDownloadPath -checksumFilePath $gwDownloadCheckSumPath + if ($comparingResult) { + Expand-Archive -Path $gwDownloadPath -DestinationPath "$env:TEMP" + $gwBinPath = "$env:TEMP\mingw64\bin" + $env:Path += ";$gwBinPath" + } else { + exit 1 + } + +} +function installGoDependencies { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report/v2" + "gotest.tools/gotestsum" + ) + foreach ($pkg in $installPackages) { + go install "$pkg@latest" + } +} + +function withNmap($version) { + Write-Host "-- Installing Nmap $version --" + [Net.ServicePointManager]::SecurityProtocol = "tls, tls11, tls12, ssl3" + $nmapInstallerUrl = "https://nmap.org/dist/nmap-$version-setup.exe" + $nmapDownloadPath = "$env:TEMP\nmap-$version-setup.exe" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -UseBasicParsing -Uri $nmapInstallerUrl -OutFile $nmapDownloadPath + } + Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait +} + +fixCRLF + +withGolang $env:GO_VERSION + +withPython $env:SETUP_WIN_PYTHON_VERSION + +withMinGW + +if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-packetbeat") { + withNmap $env:NMAP_WIN_VERSION +} + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path $WorkFolder + +$magefile = "$WORKSPACE\$WorkFolder\.magefile" +$env:MAGEFILE_CACHE = $magefile + +New-Item -ItemType Directory -Force -Path "build" + +if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { + mage -w reader/etw build goUnitTest +} else { + mage build unitTest +} + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml new file mode 100644 index 00000000000..7c5ee4d1609 --- /dev/null +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -0,0 +1,40 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-winlogbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + BEATS_PROJECT_NAME: "winlogbeat" + +steps: + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Winlogbeat - run_winlogbeat" + key: "run_winlogbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic winlogbeat pipeline" + key: "winlogbeat-pipeline" + command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.yml new file mode 100644 index 00000000000..58d61a367a4 --- /dev/null +++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +# This pipeline is only for 7.17 branch. See catalog-info.yml +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml new file mode 100644 index 00000000000..01695fa4fb6 --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -0,0 +1,50 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-libbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/libbeat" + +steps: + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Packetbeat - run_xpack_libbeat" + key: "run_xpack_libbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_xpack_libbeat_arm_tests" + key: "run_xpack_libbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic packetbeat pipeline" + key: "packetbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml new file mode 100644 index 00000000000..34321b61161 --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml new file mode 100644 index 00000000000..34321b61161 --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml new file mode 100644 index 00000000000..34321b61161 --- /dev/null +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d3e40d854f5..c10616bd3d6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,7 +18,7 @@ CHANGELOG* /go.sum @elastic/beats-tech-leads /NOTICE.txt @elastic/beats-tech-leads -/.ci/ @elastic/elastic-agent-data-plane +/.ci/ @elastic/elastic-agent-data-plane @elastic/ingest-eng-prod /.github/ @elastic/elastic-agent-data-plane /auditbeat/ @elastic/sec-linux-platform /deploy/ @elastic/elastic-agent-data-plane @@ -112,6 +112,7 @@ CHANGELOG* /x-pack/filebeat/input/cel/ @elastic/security-service-integrations /x-pack/filebeat/input/cometd/ @elastic/obs-infraobs-integrations /x-pack/filebeat/input/entityanalytics/ @elastic/security-service-integrations +/x-pack/filebeat/input/etw/ @elastic/sec-windows-platform /x-pack/filebeat/input/gcppubsub/ @elastic/security-service-integrations /x-pack/filebeat/input/gcs/ @elastic/security-service-integrations /x-pack/filebeat/input/http_endpoint/ @elastic/security-service-integrations @@ -121,6 +122,7 @@ CHANGELOG* /x-pack/filebeat/input/lumberjack/ @elastic/security-service-integrations /x-pack/filebeat/input/netflow/ @elastic/sec-deployment-and-devices /x-pack/filebeat/input/o365audit/ @elastic/security-service-integrations +/x-pack/filebeat/input/websocket/ @elastic/security-service-integrations /x-pack/filebeat/module/activemq @elastic/obs-infraobs-integrations /x-pack/filebeat/module/aws @elastic/obs-cloud-monitoring /x-pack/filebeat/module/awsfargate @elastic/obs-cloud-monitoring @@ -173,6 +175,8 @@ CHANGELOG* /x-pack/filebeat/modules.d/zoom.yml.disabled @elastic/security-service-integrations /x-pack/filebeat/processors/decode_cef/ @elastic/sec-deployment-and-devices /x-pack/heartbeat/ @elastic/obs-ds-hosted-services +/x-pack/libbeat/reader/parquet/ @elastic/security-service-integrations +/x-pack/libbeat/reader/etw/ @elastic/sec-windows-platform /x-pack/metricbeat/ @elastic/elastic-agent-data-plane /x-pack/metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. /x-pack/metricbeat/module/activemq @elastic/obs-infraobs-integrations @@ -219,4 +223,9 @@ CHANGELOG* /x-pack/osquerybeat/ @elastic/sec-deployment-and-devices /x-pack/packetbeat/ @elastic/sec-linux-platform /x-pack/winlogbeat/ @elastic/sec-windows-platform -/x-pack/libbeat/reader/parquet/ @elastic/security-service-integrations + +# Ownership of CI or related files by the Ingest Eng Prod team +/.buildkite @elastic/ingest-eng-prod +/catalog-info.yml @elastic/ingest-eng-prod +/libbeat/scripts @elastic/ingest-eng-prod +/metricbeat/tests @elastic/ingest-eng-prod diff --git a/.github/workflows/check-audtibeat.yml b/.github/workflows/check-auditbeat.yml similarity index 100% rename from .github/workflows/check-audtibeat.yml rename to .github/workflows/check-auditbeat.yml diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 140b9ce302e..f97bb2029a7 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -32,6 +32,9 @@ jobs: with: go-version-file: .go-version + - name: Install Apt Package + run: sudo apt-get update && sudo apt-get install -y libpcap-dev + - name: golangci-lint env: GOOS: ${{ matrix.GOOS }} diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml index 4cdb1e2197e..84a6209ff2c 100644 --- a/.github/workflows/opentelemetry.yml +++ b/.github/workflows/opentelemetry.yml @@ -1,46 +1,16 @@ +--- +# Look up results at https://ela.st/oblt-ci-cd-stats. +# There will be one service per GitHub repository, including the org name, and one Transaction per Workflow. name: OpenTelemetry Export Trace on: workflow_run: - workflows: - - bump-elastic-stack-snapshot - - bump-golang - - check-auditbeat - - check-default - - check-dev-tools - - check-docs - - check-filebeat - - check-heartbeat - - check-libbeat - - check-metricbeat - - check-packetbeat - - check-winlogbeat - - check-x-pack-auditbeat - - check-x-pack-dockerlogbeat - - check-x-pack-filebeat - - check-x-pack-functionbeat - - check-x-pack-heartbeat - - check-x-pack-libbeat - - check-x-pack-metricbeat - - check-x-pack-osquerybeat - - check-x-pack-packetbeat - - check-x-pack-winlogbeat - - golangci-lint - - notify-stalled-snapshots - - auditbeat - - filebeat - - heartbeat - - metricbeat - - packetbeat - - x-pack-auditbeat - - x-pack-filebeat - - x-pack-functionbeat - - x-pack-heartbeat - - x-pack-metricbeat - - x-pack-osquerybeat - - x-pack-packetbeat + workflows: [ "*" ] types: [completed] +permissions: + contents: read + jobs: otel-export-trace: runs-on: ubuntu-latest diff --git a/.go-version b/.go-version index c262b1f0dfd..8819d012cee 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.21.6 +1.21.7 diff --git a/.golangci.yml b/.golangci.yml index 03a01e24c4f..79b77eab0d1 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -114,7 +114,7 @@ linters-settings: gosimple: # Select the Go version to target. The default is '1.13'. - go: "1.21.6" + go: "1.21.7" nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 @@ -132,19 +132,19 @@ linters-settings: staticcheck: # Select the Go version to target. The default is '1.13'. - go: "1.21.6" + go: "1.21.7" checks: ["all"] stylecheck: # Select the Go version to target. The default is '1.13'. - go: "1.21.6" + go: "1.21.7" # Disabled: # ST1005: error strings should not be capitalized checks: ["all", "-ST1005"] unused: # Select the Go version to target. The default is '1.13'. - go: "1.21.6" + go: "1.21.7" gosec: excludes: diff --git a/.mergify.yml b/.mergify.yml index b354e123967..9a4af750903 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -302,3 +302,16 @@ pull_request_rules: labels: - "backport" title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" + - name: backport patches to 8.13 branch + conditions: + - merged + - label=backport-v8.13.0 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.13" + labels: + - "backport" + title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 4e650a193d1..14901ead1bc 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -87,6 +87,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757] - Renamed an httpjson input metric to follow naming conventions. `httpjson_interval_pages_total` was renamed to `httpjson_interval_pages` because the `_total` suffix is reserved for counters. {issue}35933[35933] {pull}36169[36169] - Fixed some race conditions in tests {pull}36185[36185] +- Fix Stringer implementation of fingerprint processor {issue}35174[35174] - Re-enable HTTPJSON fixed flakey test. {issue}34929[34929] {pull}36525[36525] - Make winlogbeat/sys/wineventlog follow the unsafe.Pointer rules. {pull}36650[36650] - Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714] @@ -178,7 +179,8 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Add initial infrastructure for a caching enrichment processor. {pull}36619[36619] - Add file-backed cache for cache enrichment processor. {pull}36686[36686] {pull}36696[36696] - Elide retryable HTTP client construction in Filebeat HTTPJSON and CEL inputs if not needed. {pull}36916[36916] -- Allow assignment of packetbeat protocols to interfaces. {issue}36574[36564] {pull}[] +- Allow assignment of packetbeat protocols to interfaces. {issue}36574[36564] {pull}36852[36852] +- Add Active Directory entity collector for Filebeat entity analytics. {pull}37854[37854] ==== Deprecated diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 071357c104a..a25cb6baddb 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,57 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.12.1]] +=== Beats version 8.12.1 +https://github.com/elastic/beats/compare/v8.12.0\...v8.12.1[View commits] + +==== Known Issues + +*Affecting all Beats* + +Performance regression in AWS S3 inputs using SQS notification. + +In 8.12 the default memory queue flush interval was raised from 1 second to 10 seconds. In many configurations this improves performance because it allows the output to batch more events per round trip, which improves efficiency. However, the SQS input has an extra bottleneck that interacts badly with the new value. For more details see {issue}37754[37754]. + +If you are using the Elasticsearch output, and your output configuration uses a performance preset, switch it to `preset: latency`. If you use no preset or use `preset: custom`, then set `queue.mem.flush.timeout: 1s` in your queue or output configuration. + +==== Breaking changes + +*Affecting all Beats* + +- add_cloud_metadata processor: `huawei` provider is now treated as `openstack`. Huawei cloud runs on OpenStack +platform, and when viewed from a metadata API standpoint, it is impossible to differentiate it from OpenStack. If you +know that your deployments run on Huawei Cloud exclusively, and you wish to have `cloud.provider` value as `huawei`, +you can achieve this by overwriting the value using an `add_fields` processor. {pull}35184[35184] + +==== Bugfixes + +*Affecting all Beats* + +- aws: Add credential caching for `AssumeRole` session tokens. {issue}37787[37787] +- Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments. {pull}[37816][37816] + +*Filebeat* + +- Fix nil pointer dereference in the httpjson input. {pull}37591[37591] +- Fix TCP/UDP metric queue length parsing base. {pull}37714[37714] +- Fix m365_defender cursor value and query building. {pull}37116[37116] +- Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799] + +*Heartbeat* + +- Fix setuid root when running under cgroups v2. {pull}37794[37794] + +*Metricbeat* + +- Fix Azure Resource Metrics missing metrics (min and max aggregations) after upgrade to 8.11.3. {issue}37642[37642] {pull}37643[37643] + +==== Added + +*Filebeat* + +- Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714] + [[release-notes-8.12.0]] === Beats version 8.12.0 https://github.com/elastic/beats/compare/v8.11.4\...v8.12.0[View commits] @@ -15,14 +66,20 @@ Performance regression in AWS S3 inputs using SQS notification. In 8.12 the default memory queue flush interval was raised from 1 second to 10 seconds. In many configurations this improves performance because it allows the output to batch more events per round trip, which improves efficiency. However, the SQS input has an extra bottleneck that interacts badly with the new value. For more details see {issue}37754[37754]. -If you are using the Elasticsearch output, and your output configuration uses a performance preset, switch it to `preset: latency`. If you use no preset or use `preset: custom`, then set `queue.mem.flush.timeout: 1` in your queue or output configuration. +If you are using the Elasticsearch output, and your output configuration uses a performance preset, switch it to `preset: latency`. If you use no preset or use `preset: custom`, then set `queue.mem.flush.timeout: 1s` in your queue or output configuration. -If you are not using the Elasticsearch output, set `queue.mem.flush.timeout: 1` in your queue or output configuration. +If you are not using the Elasticsearch output, set `queue.mem.flush.timeout: 1s` in your queue or output configuration. ==== Breaking changes +*Affecting all Beats* + +- Windows MSI installers now store configuration in C:\Program Files instead of C:\ProgramData. https://github.com/elastic/elastic-stack-installers/pull/209 + *Heartbeat* + - Decrease the ES default timeout to 10 for the load monitor state requests. +- Windows MSI installers now store configuration in C:\Program Files instead of C:\ProgramData. https://github.com/elastic/elastic-stack-installers/pull/209 *Osquerybeat* @@ -180,6 +237,7 @@ https://github.com/elastic/beats/compare/v8.11.1\...v8.11.2[View commits] - Enhance Azure Metrics metricset with refined grouping logic and resolved duplication issues for TSDB compatibility. {pull}36823[36823] - Fix unintended skip in metric collection on Azure Monitor. {issue}37204[37204] {pull}37203[37203] - Fix the "api-version query parameter (?api-version=) is required for all requests" error in Azure Billing. {pull}37158[37158] +- add_cloud_metadata: fix the `orchestrator` metadata for the aws cloud provider *Winlogbeat* diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5e256e1ed4d..8cddf0b5593 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -10,13 +10,24 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Affecting all Beats* -- Upgrade to Go 1.21.6. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}37615[37615] +- Upgrade to Go 1.21.7. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}37913[37913] +- add_cloud_metadata processor: `huawei` provider is now treated as `openstack`. Huawei cloud runs on OpenStack +platform, and when viewed from a metadata API standpoint, it is impossible to differentiate it from OpenStack. If you +know that your deployments run on Huawei Cloud exclusively, and you wish to have `cloud.provider` value as `huawei`, +you can achieve this by overwriting the value using an `add_fields` processor. {pull}35184[35184] +- In managed mode, Beats running under Elastic Agent will report the package +version of Elastic Agent as their own version. This includes all additional +fields added to events containing the Beats version. {pull}37553[37553] +- The behavior of `queue.mem.flush.min_events` has been simplified. It now serves as a simple maximum on the size of all event batches. There are no longer performance implications in its relationship to `bulk_max_size`. {pull}37795[37795] *Auditbeat* +- Add opt-in `KProbes` backend for file_integrity module. {pull}37796[37796] *Filebeat* +- Convert netflow input to API v2 and disable event normalisation {pull}37901[37901] + *Heartbeat* @@ -56,6 +67,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Support build of projects outside of beats directory {pull}36126[36126] - Support Elastic Agent control protocol chunking support {pull}37343[37343] - Upgrade elastic-agent-libs to v0.7.5. Removes obsolete "Treating the CommonName field on X.509 certificates as a host name..." deprecation warning for 8.0. {pull}37755[37755] +- aws: Add credential caching for `AssumeRole` session tokens. {issue}37787[37787] +- Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816] +- Set timeout of 1 minute for FQDN requests {pull}37756[37756] +- Fix the paths in the .cmd script added to the path by the Windows MSI to point to the new C:\Program Files installation location. https://github.com/elastic/elastic-stack-installers/pull/238 *Auditbeat* @@ -77,27 +92,15 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Added a fix for Crowdstrike pipeline handling process arrays {pull}36496[36496] - Fix m365_defender cursor value and query building. {pull}37116[37116] - Fix TCP/UDP metric queue length parsing base. {pull}37714[37714] +- Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799] +- [threatintel] MISP pagination fixes {pull}37898[37898] +- Fix file handle leak when handling errors in filestream {pull}37973[37973] *Heartbeat* -- Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] *Metricbeat* -- in module/windows/perfmon, changed collection method of the second counter value required to create a displayable value {pull}32305[32305] -- Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724] -- Add missing cluster metadata to k8s module metricsets {pull}32979[32979] {pull}33032[33032] -- Add GCP CloudSQL region filter {pull}32943[32943] -- Fix logstash cgroup mappings {pull}33131[33131] -- Remove unused `elasticsearch.node_stats.indices.bulk.avg_time.bytes` mapping {pull}33263[33263] -- Make generic SQL GA {pull}34637[34637] -- Collect missing remote_cluster in elasticsearch ccr metricset {pull}34957[34957] -- Add context with timeout in AWS API calls {pull}35425[35425] -- Fix EC2 host.cpu.usage {pull}35717[35717] -- Add option in SQL module to execute queries for all dbs. {pull}35688[35688] -- Add remaining dimensions for azure storage account to make them available for tsdb enablement. {pull}36331[36331] -- Add log error when statsd server fails to start {pull}36477[36477] -- Fix Azure Resource Metrics missing metrics (min and max aggregations) after upgrade to 8.11.3 {issue}37642[37642] {pull}37643[37643] - Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720] *Osquerybeat* @@ -105,11 +108,11 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Packetbeat* +- Fix interface device parsing for packetbeat protocols. {pull}37946[37946] *Winlogbeat* - *Elastic Logging Plugin* @@ -130,12 +133,16 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259] - Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544] - Make more selective the Pod autodiscovery upon node and namespace update events. {issue}37338[37338] {pull}37431[37431] +- Upgrade go-sysinfo from 1.12.0 to 1.13.1. {pull}37996[37996] *Auditbeat* +- Add linux capabilities to processes in the system/process. {pull}37453[37453] +- Add opt-in eBPF backend for file_integrity module. {pull}37223[37223] *Filebeat* +- Update SQL input documentation regarding Oracle DSNs {pull}37590[37590] - add documentation for decode_xml_wineventlog processor field mappings. {pull}32456[32456] - httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412] - Add cloudflare R2 to provider list in AWS S3 input. {pull}32620[32620] @@ -174,11 +181,22 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Update CEL extensions library to v1.8.0 to provide runtime error location reporting. {issue}37304[37304] {pull}37718[37718] - Add request trace logging for chained API requests. {issue}37551[36551] {pull}37682[37682] - Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714] +- Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772] +- Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836] +- Added experimental version of the Websocket Input. {pull}37774[37774] +- Add support for PEM-based Okta auth in CEL. {pull}37813[37813] +- Add ETW input. {pull}36915[36915] +- Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971] +- Add logging for cache processor file reads and writes. {pull}38052[38052] +- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738] *Auditbeat* *Libbeat* +- Add watcher that can be used to monitor Linux kernel events. {pull}37833[37833] + +- Added support for ETW reader. {pull}36914[36914] *Heartbeat* - Added status to monitor run log report. @@ -197,6 +215,10 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each metricset instance. {pull}36971[36971] - Add linux IO metrics to system/process {pull}37213[37213] - Add new memory/cgroup metrics to Kibana module {pull}37232[37232] + + +*Metricbeat* + - Update `getOpTimestamp` in `replstatus` to fix sort and temp files generation issue in mongodb. {pull}37688[37688] *Osquerybeat* @@ -205,6 +227,8 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d *Packetbeat* - Bump Windows Npcap version to v1.79. {pull}37733[37733] +- Add metrics for TCP flags. {issue}36992[36992] {pull}36975[36975] +- Add support for pipeline loading. {pull}37291[37291] *Packetbeat* @@ -292,6 +316,9 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d + + + diff --git a/NOTICE.txt b/NOTICE.txt index c803ff33e8e..c038c7027e3 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12255,6 +12255,32 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/ebpfevents +Version: v0.4.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.4.0/LICENSE.txt: + +The https://github.com/elastic/ebpfevents repository contains source code under +various licenses: + +- Source code in the 'headers/bpf' directory, is dual-licensed under the GNU Lesser General + Public License version 2.1 (LICENSES/LGPL-2.1-only.txt) OR BSD-2-Clause license + (LICENSES/BSD-2-Clause.txt) + +- Source code in the 'ebpf' submodule is licensed with multiple licenses. Read more at + https://github.com/elastic/ebpf/blob/main/LICENSE.txt. + +- The binary files 'bpf_bpfel_x86.o' and 'bpf_bpfel_amd64.o' are compiled + from dual-licensed GPL-2.0-only OR BSD-2-Clause licensed code, and are distributed with + the GPL-2.0-only License (LICENSES/GPL-2.0-only.txt). + +- Source code not listed in the previous points is licensed under the Apache License, + version 2 (LICENSES/Apache-2.0.txt). + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover Version: v0.6.7 @@ -12468,11 +12494,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 -Version: v7.6.0 +Version: v7.8.0 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.6.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.8.0/LICENSE.txt: ELASTIC LICENSE AGREEMENT @@ -14955,11 +14981,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-sysinfo -Version: v1.11.2 +Version: v1.13.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.11.2/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.13.1/LICENSE.txt: Apache License @@ -15589,11 +15615,11 @@ limitations under the License. -------------------------------------------------------------------------------- Dependency : github.com/elastic/mito -Version: v1.8.0 +Version: v1.9.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v1.8.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v1.9.0/LICENSE: Apache License @@ -15799,6 +15825,218 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v1.8.0/LIC limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/tk-btf +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/tk-btf@v0.1.0/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/toutoumomoma Version: v0.0.0-20221026030040-594ef30cb640 @@ -16179,6 +16417,38 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/go-ldap/ldap/v3 +Version: v3.4.6 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-ldap/ldap/v3@v3.4.6/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com) +Portions copyright (c) 2015-2016 go-ldap Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/go-sql-driver/mysql Version: v1.6.0 @@ -18421,6 +18691,38 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/gorilla/websocket +Version: v1.4.2 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gorilla/websocket@v1.4.2/LICENSE: + +Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/h2non/filetype Version: v1.1.1 @@ -20363,11 +20665,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/lestrrat-go/jwx/v2 -Version: v2.0.11 +Version: v2.0.19 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/lestrrat-go/jwx/v2@v2.0.11/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/lestrrat-go/jwx/v2@v2.0.19/LICENSE: The MIT License (MIT) @@ -25520,11 +25822,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.58.3/LIC -------------------------------------------------------------------------------- Dependency : google.golang.org/protobuf -Version: v1.31.0 +Version: v1.32.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.31.0/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.32.0/LICENSE: Copyright (c) 2018 The Go Authors. All rights reserved. @@ -30918,6 +31220,37 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/trac limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/Azure/go-ntlmssp +Version: v0.0.0-20221128193559-754e69321358 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!azure/go-ntlmssp@v0.0.0-20221128193559-754e69321358/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2016 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/AzureAD/microsoft-authentication-library-for-go Version: v0.9.0 @@ -31277,6 +31610,43 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/alexbrainman/sspi +Version: v0.0.0-20210105120005-909beea2cc74 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/alexbrainman/sspi@v0.0.0-20210105120005-909beea2cc74/LICENSE: + +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/andybalholm/brotli Version: v1.0.5 @@ -36167,6 +36537,39 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/cilium/ebpf +Version: v0.12.3 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/cilium/ebpf@v0.12.3/LICENSE: + +MIT License + +Copyright (c) 2017 Nathan Sweet +Copyright (c) 2018, 2019 Cloudflare +Copyright (c) 2019 Authors of Cilium + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/codegangsta/inject Version: v0.0.0-20150114235600-33e0aa1cb7c0 @@ -38172,11 +38575,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/frankban/quicktest -Version: v1.14.3 +Version: v1.14.5 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.5/LICENSE: MIT License @@ -38201,6 +38604,69 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/go-asn1-ber/asn1-ber +Version: v1.5.5 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-asn1-ber/asn1-ber@v1.5.5/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com) +Portions copyright (c) 2015-2016 go-asn1-ber Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-faker/faker/v4 +Version: v4.2.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-faker/faker/v4@v4.2.0/LICENSE: + +MIT License + +Copyright (c) 2017 Iman Tumorang + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/go-logfmt/logfmt Version: v0.5.1 @@ -41504,38 +41970,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/gorilla/websocket -Version: v1.4.2 -Licence type (autodetected): BSD-2-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/gorilla/websocket@v1.4.2/LICENSE: - -Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/hashicorp/cronexpr Version: v1.1.0 @@ -45643,11 +46077,11 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/kr/pretty -Version: v0.3.0 +Version: v0.3.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.3.0/License: +Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.3.1/License: Copyright 2012 Keith Rarick @@ -45913,11 +46347,11 @@ Contents of probable licence file $GOMODCACHE/github.com/kylelemons/godebug@v1.1 -------------------------------------------------------------------------------- Dependency : github.com/lestrrat-go/blackmagic -Version: v1.0.1 +Version: v1.0.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/lestrrat-go/blackmagic@v1.0.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/lestrrat-go/blackmagic@v1.0.2/LICENSE: MIT License diff --git a/auditbeat/.gitignore b/auditbeat/.gitignore index 3cd551fd506..7c8dbc05501 100644 --- a/auditbeat/.gitignore +++ b/auditbeat/.gitignore @@ -6,4 +6,3 @@ module/*/_meta/config.yml /auditbeat /auditbeat.test /docs/html_docs - diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 59eb4860b40..df038d2edf8 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.6 +FROM golang:1.21.7 RUN \ apt-get update \ diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 4ea656f174e..a68f7e1094a 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -30,6 +30,24 @@ stages: unitTest: mage: "mage build unitTest" stage: mandatory + integTest: + mage: "mage build integTest" + when: + comments: + - "/test auditbeat integTest" + branches: false + tags: false + stage: extended + integTest-arm: + mage: "mage build integTest" + platforms: + - "ubuntu-2204-aarch64" + when: + comments: + - "/test auditbeat integTest arm" + branches: false + tags: false + stage: extended crosscompile: make: "make -C auditbeat crosscompile" stage: mandatory diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 883760ab410..e9a23ca6ac0 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -92,6 +92,11 @@ auditbeat.modules: # Auditbeat will ignore files unless they match a pattern. #include_files: #- '/\.ssh($|/)' + # Select the backend which will be used to source events. + # "fsnotify" doesn't have the ability to associate user data to file events. + # Valid values: auto, fsnotify, kprobes, ebpf. + # Default: fsnotify. + backend: fsnotify # Scan over the configured file paths at startup and send events for new or # modified files since the last time Auditbeat was running. @@ -1351,9 +1356,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/auditbeat/docker-compose.yml b/auditbeat/docker-compose.yml index adf33888988..e0479381eff 100644 --- a/auditbeat/docker-compose.yml +++ b/auditbeat/docker-compose.yml @@ -14,11 +14,15 @@ services: - KIBANA_PORT=5601 volumes: - ${PWD}/..:/go/src/github.com/elastic/beats/ + - /sys:/sys command: make privileged: true pid: host cap_add: - AUDIT_CONTROL + - BPF + - PERFMON + - SYS_RESOURCE # This is a proxy used to block beats until all services are healthy. # See: https://github.com/docker/compose/issues/4369 diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index bd4db4ce5b6..9eee5f008fc 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -18925,6 +18925,28 @@ type: keyword -- +*`process.thread.capabilities.effective`*:: ++ +-- +This is the set of capabilities used by the kernel to perform permission checks for the thread. + +type: keyword + +example: ["CAP_BPF", "CAP_SYS_ADMIN"] + +-- + +*`process.thread.capabilities.permitted`*:: ++ +-- +This is a limiting superset for the effective capabilities that the thread may assume. + +type: keyword + +example: ["CAP_BPF", "CAP_SYS_ADMIN"] + +-- + [float] === hash diff --git a/auditbeat/docs/modules/file_integrity.asciidoc b/auditbeat/docs/modules/file_integrity.asciidoc index a12c4df47ca..5257099270b 100644 --- a/auditbeat/docs/modules/file_integrity.asciidoc +++ b/auditbeat/docs/modules/file_integrity.asciidoc @@ -28,8 +28,14 @@ to only send events for new or modified files. The operating system features that power this feature are as follows. -* Linux - `inotify` is used, and therefore the kernel must have inotify support. +* Linux - Multiple backends are supported: `auto`, `fsnotify`, `kprobes`, `ebpf`. +By default, `fsnotify` is used, and therefore the kernel must have inotify support. Inotify was initially merged into the 2.6.13 Linux kernel. +The eBPF backend uses modern eBPF features and supports 5.10.16+ kernels. +The `Kprobes` backend uses tracefs and supports 3.10+ kernels. +FSNotify doesn't have the ability to associate user data to file events. +The preferred backend can be selected by specifying the `backend` config option. +Since eBPF and Kprobes are in technical preview, `auto` will default to `fsnotify`. * macOS (Darwin) - Uses the `FSEvents` API, present since macOS 10.5. This API coalesces multiple changes to a file into a single event. {beatname_uc} translates this coalesced changes into a meaningful sequence of actions. However, @@ -144,6 +150,9 @@ of this directories are watched. If `recursive` is set to `true`, the `file_integrity` module will watch for changes on this directories and all their subdirectories. +*`backend`*:: (*Linux only*) Select the backend which will be used to +source events. Valid values: `auto`, `fsnotify`, `kprobes`, `ebpf`. Default: `fsnotify`. + include::{docdir}/auditbeat-options.asciidoc[] diff --git a/auditbeat/docs/running-on-kubernetes.asciidoc b/auditbeat/docs/running-on-kubernetes.asciidoc index 73ac5cdd70f..f5f4f0f4715 100644 --- a/auditbeat/docs/running-on-kubernetes.asciidoc +++ b/auditbeat/docs/running-on-kubernetes.asciidoc @@ -57,17 +57,17 @@ may want to change that behavior, so just edit the YAML file and modify them: ------------------------------------------------ [float] -===== Running {beatname_uc} on master nodes +===== Running {beatname_uc} on control plane nodes -Kubernetes master nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] -to limit the workloads that can run on them. To run {beatname_uc} on master nodes you may need to +Kubernetes control plane nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] +to limit the workloads that can run on them. To run {beatname_uc} on control plane nodes you may need to update the Daemonset spec to include proper tolerations: [source,yaml] ------------------------------------------------ spec: tolerations: - - key: node-role.kubernetes.io/master + - key: node-role.kubernetes.io/control-plane effect: NoSchedule ------------------------------------------------ diff --git a/auditbeat/module/file_integrity/_meta/config.yml.tmpl b/auditbeat/module/file_integrity/_meta/config.yml.tmpl index 588a6279eee..b3a3784d15c 100644 --- a/auditbeat/module/file_integrity/_meta/config.yml.tmpl +++ b/auditbeat/module/file_integrity/_meta/config.yml.tmpl @@ -55,6 +55,14 @@ #- '/\.ssh($|/)' {{- end }} + {{- if eq .GOOS "linux" }} + # Select the backend which will be used to source events. + # "fsnotify" doesn't have the ability to associate user data to file events. + # Valid values: auto, fsnotify, kprobes, ebpf. + # Default: fsnotify. + backend: fsnotify + {{- end }} + # Scan over the configured file paths at startup and send events for new or # modified files since the last time Auditbeat was running. scan_at_start: true diff --git a/auditbeat/module/file_integrity/_meta/docs.asciidoc b/auditbeat/module/file_integrity/_meta/docs.asciidoc index 0f32ef64f93..35031d8acea 100644 --- a/auditbeat/module/file_integrity/_meta/docs.asciidoc +++ b/auditbeat/module/file_integrity/_meta/docs.asciidoc @@ -21,8 +21,14 @@ to only send events for new or modified files. The operating system features that power this feature are as follows. -* Linux - `inotify` is used, and therefore the kernel must have inotify support. +* Linux - Multiple backends are supported: `auto`, `fsnotify`, `kprobes`, `ebpf`. +By default, `fsnotify` is used, and therefore the kernel must have inotify support. Inotify was initially merged into the 2.6.13 Linux kernel. +The eBPF backend uses modern eBPF features and supports 5.10.16+ kernels. +The `Kprobes` backend uses tracefs and supports 3.10+ kernels. +FSNotify doesn't have the ability to associate user data to file events. +The preferred backend can be selected by specifying the `backend` config option. +Since eBPF and Kprobes are in technical preview, `auto` will default to `fsnotify`. * macOS (Darwin) - Uses the `FSEvents` API, present since macOS 10.5. This API coalesces multiple changes to a file into a single event. {beatname_uc} translates this coalesced changes into a meaningful sequence of actions. However, @@ -137,4 +143,7 @@ of this directories are watched. If `recursive` is set to `true`, the `file_integrity` module will watch for changes on this directories and all their subdirectories. +*`backend`*:: (*Linux only*) Select the backend which will be used to +source events. Valid values: `auto`, `fsnotify`, `kprobes`, `ebpf`. Default: `fsnotify`. + include::{docdir}/auditbeat-options.asciidoc[] diff --git a/auditbeat/module/file_integrity/config.go b/auditbeat/module/file_integrity/config.go index e431e640766..db934b02789 100644 --- a/auditbeat/module/file_integrity/config.go +++ b/auditbeat/module/file_integrity/config.go @@ -18,10 +18,12 @@ package file_integrity import ( + "errors" "fmt" "math" "path/filepath" "regexp" + "runtime" "sort" "strings" @@ -72,6 +74,25 @@ const ( XXH64 HashType = "xxh64" ) +type Backend string + +const ( + BackendFSNotify Backend = "fsnotify" + BackendKprobes Backend = "kprobes" + BackendEBPF Backend = "ebpf" + BackendAuto Backend = "auto" +) + +func (b *Backend) Unpack(v string) error { + *b = Backend(v) + switch *b { + case BackendFSNotify, BackendKprobes, BackendEBPF, BackendAuto: + return nil + default: + return fmt.Errorf("invalid backend: %q", v) + } +} + // Config contains the configuration parameters for the file integrity // metricset. type Config struct { @@ -86,6 +107,7 @@ type Config struct { Recursive bool `config:"recursive"` // Recursive enables recursive monitoring of directories. ExcludeFiles []match.Matcher `config:"exclude_files"` IncludeFiles []match.Matcher `config:"include_files"` + Backend Backend `config:"backend"` } // Validate validates the config data and return an error explaining all the @@ -160,6 +182,11 @@ nextHash: if err != nil { errs = append(errs, fmt.Errorf("invalid scan_rate_per_sec value: %w", err)) } + + if c.Backend != "" && c.Backend != BackendAuto && runtime.GOOS != "linux" { + errs = append(errs, errors.New("backend can only be specified on linux")) + } + return errs.Err() } diff --git a/auditbeat/module/file_integrity/event.go b/auditbeat/module/file_integrity/event.go index fd4d68828a4..22813a47f22 100644 --- a/auditbeat/module/file_integrity/event.go +++ b/auditbeat/module/file_integrity/event.go @@ -65,11 +65,17 @@ const ( // SourceFSNotify identifies events triggered by a notification from the // file system. SourceFSNotify + // SourceEBPF identifies events triggered by an eBPF program. + SourceEBPF + // SourceKProbes identifies events triggered by KProbes. + SourceKProbes ) var sourceNames = map[Source]string{ SourceScan: "scan", SourceFSNotify: "fsnotify", + SourceEBPF: "ebpf", + SourceKProbes: "kprobes", } // Type identifies the file type (e.g. dir, file, symlink). @@ -91,12 +97,20 @@ const ( FileType DirType SymlinkType + CharDeviceType + BlockDeviceType + FIFOType + SocketType ) var typeNames = map[Type]string{ - FileType: "file", - DirType: "dir", - SymlinkType: "symlink", + FileType: "file", + DirType: "dir", + SymlinkType: "symlink", + CharDeviceType: "char_device", + BlockDeviceType: "block_device", + FIFOType: "fifo", + SocketType: "socket", } // Digest is an output of a hash function. @@ -189,36 +203,42 @@ func NewEventFromFileInfo( switch event.Info.Type { case FileType: - if event.Info.Size <= maxFileSize { - hashes, nbytes, err := hashFile(event.Path, maxFileSize, hashTypes...) - if err != nil { - event.errors = append(event.errors, err) - event.hashFailed = true - } else if hashes != nil { - // hashFile returns nil hashes and no error when: - // - There's no hashes configured. - // - File size at the time of hashing is larger than configured limit. - event.Hashes = hashes - event.Info.Size = nbytes - } - - if len(fileParsers) != 0 && event.ParserResults == nil { - event.ParserResults = make(mapstr.M) - } - for _, p := range fileParsers { - err = p.Parse(event.ParserResults, path) - if err != nil { - event.errors = append(event.errors, err) - } - } - } + fillHashes(&event, path, maxFileSize, hashTypes, fileParsers) case SymlinkType: - event.TargetPath, _ = filepath.EvalSymlinks(event.Path) + event.TargetPath, err = filepath.EvalSymlinks(event.Path) + if err != nil { + event.errors = append(event.errors, err) + } } return event } +func fillHashes(event *Event, path string, maxFileSize uint64, hashTypes []HashType, fileParsers []FileParser) { + if event.Info.Size <= maxFileSize { + hashes, nbytes, err := hashFile(event.Path, maxFileSize, hashTypes...) + if err != nil { + event.errors = append(event.errors, err) + event.hashFailed = true + } else if hashes != nil { + // hashFile returns nil hashes and no error when: + // - There's no hashes configured. + // - File size at the time of hashing is larger than configured limit. + event.Hashes = hashes + event.Info.Size = nbytes + } + + if len(fileParsers) != 0 && event.ParserResults == nil { + event.ParserResults = make(mapstr.M) + } + for _, p := range fileParsers { + if err = p.Parse(event.ParserResults, path); err != nil { + event.errors = append(event.errors, err) + } + } + } +} + // NewEvent creates a new Event. Any errors that occur are included in the // returned Event. func NewEvent( diff --git a/auditbeat/module/file_integrity/event_linux.go b/auditbeat/module/file_integrity/event_linux.go new file mode 100644 index 00000000000..7643d03a6b4 --- /dev/null +++ b/auditbeat/module/file_integrity/event_linux.go @@ -0,0 +1,199 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package file_integrity + +import ( + "os" + "os/user" + "path/filepath" + "strconv" + "time" + + "github.com/elastic/ebpfevents" +) + +// NewEventFromEbpfEvent creates a new Event from an ebpfevents.Event. +func NewEventFromEbpfEvent( + ee ebpfevents.Event, + maxFileSize uint64, + hashTypes []HashType, + fileParsers []FileParser, + isExcludedPath func(string) bool, +) (Event, bool) { + var ( + path, target string + action Action + metadata Metadata + err error + ) + switch ee.Type { + case ebpfevents.EventTypeFileCreate: + action = Created + + fileCreateEvent := ee.Body.(*ebpfevents.FileCreate) + path = fileCreateEvent.Path + if isExcludedPath(path) { + event := Event{Path: path} + return event, false + } + target = fileCreateEvent.SymlinkTargetPath + metadata, err = metadataFromFileCreate(fileCreateEvent) + case ebpfevents.EventTypeFileRename: + action = Moved + + fileRenameEvent := ee.Body.(*ebpfevents.FileRename) + path = fileRenameEvent.NewPath + if isExcludedPath(path) { + event := Event{Path: path} + return event, false + } + target = fileRenameEvent.SymlinkTargetPath + metadata, err = metadataFromFileRename(fileRenameEvent) + case ebpfevents.EventTypeFileDelete: + action = Deleted + + fileDeleteEvent := ee.Body.(*ebpfevents.FileDelete) + path = fileDeleteEvent.Path + if isExcludedPath(path) { + event := Event{Path: path} + return event, false + } + target = fileDeleteEvent.SymlinkTargetPath + case ebpfevents.EventTypeFileModify: + fileModifyEvent := ee.Body.(*ebpfevents.FileModify) + + switch fileModifyEvent.ChangeType { + case ebpfevents.FileChangeTypeContent: + action = Updated + case ebpfevents.FileChangeTypePermissions, ebpfevents.FileChangeTypeOwner, ebpfevents.FileChangeTypeXattrs: + action = AttributesModified + } + + path = fileModifyEvent.Path + if isExcludedPath(path) { + event := Event{Path: path} + return event, false + } + target = fileModifyEvent.SymlinkTargetPath + metadata, err = metadataFromFileModify(fileModifyEvent) + } + + event := Event{ + Timestamp: time.Now().UTC(), + Path: path, + TargetPath: target, + Info: &metadata, + Source: SourceEBPF, + Action: action, + errors: make([]error, 0), + } + if err != nil { + event.errors = append(event.errors, err) + } + + if event.Action == Deleted { + event.Info = nil + } else { + switch event.Info.Type { + case FileType: + fillHashes(&event, path, maxFileSize, hashTypes, fileParsers) + case SymlinkType: + var err error + event.TargetPath, err = filepath.EvalSymlinks(event.Path) + if err != nil { + event.errors = append(event.errors, err) + } + } + } + + return event, true +} + +func metadataFromFileCreate(evt *ebpfevents.FileCreate) (Metadata, error) { + var md Metadata + fillExtendedAttributes(&md, evt.Path) + err := fillFileInfo(&md, evt.Finfo) + return md, err +} + +func metadataFromFileRename(evt *ebpfevents.FileRename) (Metadata, error) { + var md Metadata + fillExtendedAttributes(&md, evt.NewPath) + err := fillFileInfo(&md, evt.Finfo) + return md, err +} + +func metadataFromFileModify(evt *ebpfevents.FileModify) (Metadata, error) { + var md Metadata + fillExtendedAttributes(&md, evt.Path) + err := fillFileInfo(&md, evt.Finfo) + return md, err +} + +func fillFileInfo(md *Metadata, finfo ebpfevents.FileInfo) error { + md.Inode = finfo.Inode + md.UID = finfo.Uid + md.GID = finfo.Gid + md.Size = finfo.Size + md.MTime = finfo.Mtime + md.CTime = finfo.Ctime + md.Type = typeFromEbpfType(finfo.Type) + md.Mode = finfo.Mode + md.SetUID = finfo.Mode&os.ModeSetuid != 0 + md.SetGID = finfo.Mode&os.ModeSetgid != 0 + + u, err := user.LookupId(strconv.FormatUint(uint64(finfo.Uid), 10)) + if err != nil { + md.Owner = "n/a" + md.Group = "n/a" + return err + } + md.Owner = u.Username + + g, err := user.LookupGroupId(strconv.FormatUint(uint64(finfo.Gid), 10)) + if err != nil { + md.Group = "n/a" + return err + } + md.Group = g.Name + + return nil +} + +func typeFromEbpfType(typ ebpfevents.FileType) Type { + switch typ { + case ebpfevents.FileTypeFile: + return FileType + case ebpfevents.FileTypeDir: + return DirType + case ebpfevents.FileTypeSymlink: + return SymlinkType + case ebpfevents.FileTypeCharDevice: + return CharDeviceType + case ebpfevents.FileTypeBlockDevice: + return BlockDeviceType + case ebpfevents.FileTypeNamedPipe: + return FIFOType + case ebpfevents.FileTypeSocket: + return SocketType + default: + return UnknownType + } +} diff --git a/auditbeat/module/file_integrity/event_linux_test.go b/auditbeat/module/file_integrity/event_linux_test.go new file mode 100644 index 00000000000..1a440afb8f1 --- /dev/null +++ b/auditbeat/module/file_integrity/event_linux_test.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package file_integrity + +import ( + "os" + "os/user" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/ebpfevents" +) + +func TestNewEventFromEbpfEvent(t *testing.T) { + ebpfEvent := ebpfevents.Event{ + Header: ebpfevents.Header{ + Type: ebpfevents.EventTypeFileCreate, + }, + Body: &ebpfevents.FileCreate{ + Finfo: ebpfevents.FileInfo{ + Type: ebpfevents.FileTypeFile, + Inode: 1234, + Mode: os.FileMode(0o644), + Size: 2345, + Uid: 3456, + Gid: 4567, + }, + Path: "/foo", + SymlinkTargetPath: "/bar", + }, + } + expectedEvent := Event{ + Action: Created, + Path: "/foo", + TargetPath: "/bar", + Info: &Metadata{ + Type: FileType, + Inode: 1234, + UID: 3456, + GID: 4567, + Size: 2345, + Owner: "n/a", + Group: "n/a", + Mode: os.FileMode(0o644), + }, + Source: SourceEBPF, + errors: []error{user.UnknownUserIdError(3456)}, + } + + event, ok := NewEventFromEbpfEvent( + ebpfEvent, 0, []HashType{}, []FileParser{}, func(path string) bool { return false }) + assert.True(t, ok) + event.Timestamp = expectedEvent.Timestamp + + assert.Equal(t, expectedEvent, event) +} diff --git a/auditbeat/module/file_integrity/eventreader_ebpf.go b/auditbeat/module/file_integrity/eventreader_ebpf.go new file mode 100644 index 00000000000..2fb452861e8 --- /dev/null +++ b/auditbeat/module/file_integrity/eventreader_ebpf.go @@ -0,0 +1,128 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package file_integrity + +import ( + "path/filepath" + "strings" + "time" + + "github.com/elastic/beats/v7/libbeat/ebpf" + "github.com/elastic/ebpfevents" + "github.com/elastic/elastic-agent-libs/logp" +) + +const clientName = "fim" + +type ebpfReader struct { + watcher *ebpf.Watcher + done <-chan struct{} + config Config + log *logp.Logger + eventC chan Event + parsers []FileParser + paths map[string]struct{} + + _records <-chan ebpfevents.Record +} + +func (r *ebpfReader) Start(done <-chan struct{}) (<-chan Event, error) { + watcher, err := ebpf.GetWatcher() + if err != nil { + return nil, err + } + r.watcher = watcher + r.done = done + + mask := ebpf.EventMask(ebpfevents.EventTypeFileCreate | ebpfevents.EventTypeFileRename | ebpfevents.EventTypeFileDelete | ebpfevents.EventTypeFileModify) + r._records = r.watcher.Subscribe(clientName, mask) + + go r.consumeEvents() + + r.log.Infow("started ebpf watcher", "file_path", r.config.Paths, "recursive", r.config.Recursive) + return r.eventC, nil +} + +func (r *ebpfReader) consumeEvents() { + defer close(r.eventC) + defer r.watcher.Unsubscribe(clientName) + + for { + select { + case rec := <-r._records: + if rec.Error != nil { + r.log.Errorf("ebpf watcher error: %v", rec.Error) + continue + } + + switch rec.Event.Type { + case ebpfevents.EventTypeFileCreate, ebpfevents.EventTypeFileRename, ebpfevents.EventTypeFileDelete, ebpfevents.EventTypeFileModify: + default: + r.log.Warnf("received unwanted ebpf event: %s", rec.Event.Type.String()) + continue + } + + start := time.Now() + e, ok := NewEventFromEbpfEvent( + *rec.Event, + r.config.MaxFileSizeBytes, + r.config.HashTypes, + r.parsers, + r.excludedPath, + ) + if !ok { + continue + } + e.rtt = time.Since(start) + + r.log.Debugw("received ebpf event", "file_path", e.Path) + r.eventC <- e + case <-r.done: + r.log.Debug("ebpf watcher terminated") + return + } + } +} + +func (r *ebpfReader) excludedPath(path string) bool { + dir, err := filepath.Abs(filepath.Dir(path)) + if err != nil { + r.log.Errorf("ebpf watcher error: resolve abs path %q: %v", path, err) + return true + } + + if r.config.IsExcludedPath(dir) { + return true + } + + if !r.config.Recursive { + if _, ok := r.paths[dir]; ok { + return false + } + } else { + for p := range r.paths { + if strings.HasPrefix(dir, p) { + return false + } + } + } + + return true +} diff --git a/auditbeat/module/file_integrity/eventreader_fsevents.go b/auditbeat/module/file_integrity/eventreader_fsevents.go index 8a5844b3eea..035b2bf90b4 100644 --- a/auditbeat/module/file_integrity/eventreader_fsevents.go +++ b/auditbeat/module/file_integrity/eventreader_fsevents.go @@ -31,7 +31,7 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -type fsreader struct { +type fsEventsReader struct { stream *fsevents.EventStream config Config eventC chan Event @@ -89,7 +89,7 @@ var flagNames = map[fsevents.EventFlags]string{ } // NewEventReader creates a new EventProducer backed by FSEvents macOS facility. -func NewEventReader(c Config) (EventProducer, error) { +func NewEventReader(c Config, logger *logp.Logger) (EventProducer, error) { stream := &fsevents.EventStream{ Paths: c.Paths, // NoDefer: Ignore Latency field and send events as fast as possible. @@ -108,28 +108,27 @@ func NewEventReader(c Config) (EventProducer, error) { stream.Flags |= fsevents.IgnoreSelf } - log := logp.NewLogger(moduleName) var dirs []os.FileInfo if !c.Recursive { for _, path := range c.Paths { if info, err := getFileInfo(path); err == nil { dirs = append(dirs, info) } else { - log.Warnw("Failed to get file info", "file_path", path, "error", err) + logger.Warnw("Failed to get file info", "file_path", path, "error", err) } } } - return &fsreader{ + return &fsEventsReader{ stream: stream, config: c, eventC: make(chan Event, 1), watchedDirs: dirs, - log: log, + log: logger, parsers: FileParsers(c), }, nil } -func (r *fsreader) Start(done <-chan struct{}) (<-chan Event, error) { +func (r *fsEventsReader) Start(done <-chan struct{}) (<-chan Event, error) { r.stream.Start() go r.consumeEvents(done) r.log.Infow("Started FSEvents watcher", @@ -138,7 +137,7 @@ func (r *fsreader) Start(done <-chan struct{}) (<-chan Event, error) { return r.eventC, nil } -func (r *fsreader) consumeEvents(done <-chan struct{}) { +func (r *fsEventsReader) consumeEvents(done <-chan struct{}) { defer close(r.eventC) defer r.stream.Stop() @@ -209,7 +208,7 @@ func getFileInfo(path string) (os.FileInfo, error) { return info, fmt.Errorf("failed to stat: %w", err) } -func (r *fsreader) isWatched(path string) bool { +func (r *fsEventsReader) isWatched(path string) bool { if r.config.Recursive { return true } diff --git a/auditbeat/module/file_integrity/eventreader_fsnotify.go b/auditbeat/module/file_integrity/eventreader_fsnotify.go index b49bb7b7905..0420d0f8f81 100644 --- a/auditbeat/module/file_integrity/eventreader_fsnotify.go +++ b/auditbeat/module/file_integrity/eventreader_fsnotify.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -type reader struct { +type fsNotifyReader struct { watcher monitor.Watcher config Config eventC chan Event @@ -41,16 +41,7 @@ type reader struct { parsers []FileParser } -// NewEventReader creates a new EventProducer backed by fsnotify. -func NewEventReader(c Config) (EventProducer, error) { - return &reader{ - config: c, - log: logp.NewLogger(moduleName), - parsers: FileParsers(c), - }, nil -} - -func (r *reader) Start(done <-chan struct{}) (<-chan Event, error) { +func (r *fsNotifyReader) Start(done <-chan struct{}) (<-chan Event, error) { watcher, err := monitor.New(r.config.Recursive, r.config.IsExcludedPath) if err != nil { return nil, err @@ -105,17 +96,18 @@ func (r *reader) Start(done <-chan struct{}) (<-chan Event, error) { return r.eventC, nil } -func (r *reader) enqueueEvents(done <-chan struct{}) (events []*Event) { +func (r *fsNotifyReader) enqueueEvents(done <-chan struct{}) []*Event { + events := make([]*Event, 0) for { ev := r.nextEvent(done) if ev == nil { - return + return events } events = append(events, ev) } } -func (r *reader) consumeEvents(done <-chan struct{}) { +func (r *fsNotifyReader) consumeEvents(done <-chan struct{}) { defer close(r.eventC) defer r.watcher.Close() @@ -129,7 +121,7 @@ func (r *reader) consumeEvents(done <-chan struct{}) { } } -func (r *reader) nextEvent(done <-chan struct{}) *Event { +func (r *fsNotifyReader) nextEvent(done <-chan struct{}) *Event { for { select { case <-done: diff --git a/auditbeat/module/file_integrity/eventreader_kprobes.go b/auditbeat/module/file_integrity/eventreader_kprobes.go new file mode 100644 index 00000000000..7cddd7f60cd --- /dev/null +++ b/auditbeat/module/file_integrity/eventreader_kprobes.go @@ -0,0 +1,182 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package file_integrity + +import ( + "errors" + "fmt" + "path/filepath" + "time" + + "github.com/elastic/beats/v7/auditbeat/module/file_integrity/kprobes" + + "github.com/elastic/elastic-agent-libs/logp" + + "golang.org/x/sys/unix" +) + +type kProbesReader struct { + watcher *kprobes.Monitor + config Config + eventC chan Event + log *logp.Logger + + parsers []FileParser +} + +func (r kProbesReader) Start(done <-chan struct{}) (<-chan Event, error) { + watcher, err := kprobes.New(r.config.Recursive) + if err != nil { + return nil, err + } + + r.watcher = watcher + if err := r.watcher.Start(); err != nil { + // Ensure that watcher is closed so that we don't leak watchers + r.watcher.Close() + return nil, fmt.Errorf("unable to start watcher: %w", err) + } + + queueDone := make(chan struct{}) + queueC := make(chan []*Event) + + // Launch a separate goroutine to fetch all events that happen while + // watches are being installed. + go func() { + defer close(queueC) + queueC <- r.enqueueEvents(queueDone) + }() + + // kProbes watcher needs to have the watched paths + // installed after the event consumer is started, to avoid a potential + // deadlock. Do it on all platforms for simplicity. + for _, p := range r.config.Paths { + if err := r.watcher.Add(p); err != nil { + if errors.Is(err, unix.EMFILE) { + r.log.Warnw("Failed to add watch (check the max number of "+ + "open files allowed with 'ulimit -a')", + "file_path", p, "error", err) + } else { + r.log.Warnw("Failed to add watch", "file_path", p, "error", err) + } + } + } + + close(queueDone) + events := <-queueC + + // Populate callee's event channel with the previously received events + r.eventC = make(chan Event, 1+len(events)) + for _, ev := range events { + r.eventC <- *ev + } + + go r.consumeEvents(done) + + r.log.Infow("Started kprobes watcher", + "file_path", r.config.Paths, + "recursive", r.config.Recursive) + return r.eventC, nil +} + +func (r kProbesReader) enqueueEvents(done <-chan struct{}) []*Event { + var events []*Event //nolint:prealloc //can't be preallocated as the number of events is unknown + for { + ev := r.nextEvent(done) + if ev == nil { + break + } + events = append(events, ev) + } + + return events +} + +func (r kProbesReader) consumeEvents(done <-chan struct{}) { + defer close(r.eventC) + defer r.watcher.Close() + + for { + ev := r.nextEvent(done) + if ev == nil { + r.log.Debug("kprobes reader terminated") + return + } + r.eventC <- *ev + } +} + +func (r kProbesReader) nextEvent(done <-chan struct{}) *Event { + for { + select { + case <-done: + return nil + + case event := <-r.watcher.EventChannel(): + if event.Path == "" || r.config.IsExcludedPath(event.Path) || + !r.config.IsIncludedPath(event.Path) { + continue + } + r.log.Debugw("Received kprobes event", + "file_path", event.Path, + "event_flags", event.Op) + + abs, err := filepath.Abs(event.Path) + if err != nil { + r.log.Errorw("Failed to obtain absolute path", + "file_path", event.Path, + "error", err, + ) + event.Path = filepath.Clean(event.Path) + } else { + event.Path = abs + } + + start := time.Now() + e := NewEvent(event.Path, kProbeTypeToAction(event.Op), SourceKProbes, + r.config.MaxFileSizeBytes, r.config.HashTypes, r.parsers) + e.rtt = time.Since(start) + + return &e + + case err := <-r.watcher.ErrorChannel(): + if err != nil { + r.log.Errorw("kprobes watcher error", "error", err) + } + } + } +} + +func kProbeTypeToAction(op uint32) Action { + switch op { + case unix.IN_CREATE, unix.IN_MOVED_TO: + return Created + case unix.IN_MODIFY: + return Updated + case unix.IN_DELETE: + return Deleted + case unix.IN_MOVED_FROM: + return Moved + case unix.IN_ATTRIB: + return AttributesModified + default: + return 0 + } +} diff --git a/auditbeat/module/file_integrity/eventreader_linux.go b/auditbeat/module/file_integrity/eventreader_linux.go new file mode 100644 index 00000000000..ac9ce7de60d --- /dev/null +++ b/auditbeat/module/file_integrity/eventreader_linux.go @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package file_integrity + +import ( + "errors" + + "github.com/elastic/elastic-agent-libs/logp" +) + +func NewEventReader(c Config, logger *logp.Logger) (EventProducer, error) { + if c.Backend == BackendAuto || c.Backend == BackendFSNotify || c.Backend == "" { + // Auto and unset defaults to fsnotify + l := logger.Named("fsnotify") + l.Info("selected backend: fsnotify") + return &fsNotifyReader{ + config: c, + log: l, + parsers: FileParsers(c), + }, nil + } + + if c.Backend == BackendEBPF { + l := logger.Named("ebpf") + l.Info("selected backend: ebpf") + + paths := make(map[string]struct{}) + for _, p := range c.Paths { + paths[p] = struct{}{} + } + + return &ebpfReader{ + config: c, + log: l, + parsers: FileParsers(c), + paths: paths, + eventC: make(chan Event), + }, nil + } + + if c.Backend == BackendKprobes { + l := logger.Named("kprobes") + l.Info("selected backend: kprobes") + return &kProbesReader{ + config: c, + log: l, + parsers: FileParsers(c), + }, nil + } + + // unimplemented + return nil, errors.ErrUnsupported +} diff --git a/auditbeat/module/file_integrity/eventreader_other.go b/auditbeat/module/file_integrity/eventreader_other.go new file mode 100644 index 00000000000..e9027a8b47d --- /dev/null +++ b/auditbeat/module/file_integrity/eventreader_other.go @@ -0,0 +1,32 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build freebsd || openbsd || netbsd || windows + +package file_integrity + +import ( + "github.com/elastic/elastic-agent-libs/logp" +) + +func NewEventReader(c Config, logger *logp.Logger) (EventProducer, error) { + return &fsNotifyReader{ + config: c, + log: logger.Named("fsnotify"), + parsers: FileParsers(c), + }, nil +} diff --git a/auditbeat/module/file_integrity/eventreader_test.go b/auditbeat/module/file_integrity/eventreader_test.go index 5ed273b76b4..d34f59f08c2 100644 --- a/auditbeat/module/file_integrity/eventreader_test.go +++ b/auditbeat/module/file_integrity/eventreader_test.go @@ -31,6 +31,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/logp" ) func init() { @@ -49,7 +51,7 @@ func TestEventReader(t *testing.T) { // Create a new EventProducer. config := defaultConfig config.Paths = []string{dir} - r, err := NewEventReader(config) + r, err := NewEventReader(config, logp.NewLogger("")) if err != nil { t.Fatal(err) } @@ -251,7 +253,7 @@ func TestRaces(t *testing.T) { config := defaultConfig config.Paths = dirs config.Recursive = true - r, err := NewEventReader(config) + r, err := NewEventReader(config, logp.NewLogger("")) if err != nil { t.Fatal(err) } diff --git a/auditbeat/module/file_integrity/eventreader_unsupported.go b/auditbeat/module/file_integrity/eventreader_unsupported.go index 79ab1f4245e..d039cc1bfe0 100644 --- a/auditbeat/module/file_integrity/eventreader_unsupported.go +++ b/auditbeat/module/file_integrity/eventreader_unsupported.go @@ -19,8 +19,12 @@ package file_integrity -import "errors" +import ( + "errors" -func NewEventReader(c Config) (EventProducer, error) { + "github.com/elastic/elastic-agent-libs/logp" +) + +func NewEventReader(c Config, logger *logp.Logger) (EventProducer, error) { return errors.New("file auditing metricset is not implemented on this system") } diff --git a/auditbeat/module/file_integrity/fileinfo_posix.go b/auditbeat/module/file_integrity/fileinfo_posix.go index f70a638bc65..d87c8fc4e20 100644 --- a/auditbeat/module/file_integrity/fileinfo_posix.go +++ b/auditbeat/module/file_integrity/fileinfo_posix.go @@ -69,18 +69,7 @@ func NewMetadata(path string, info os.FileInfo) (*Metadata, error) { fileInfo.Owner = owner.Username } - var selinux []byte - getExtendedAttributes(path, map[string]*[]byte{ - "security.selinux": &selinux, - "system.posix_acl_access": &fileInfo.POSIXACLAccess, - }) - // The selinux attr may be null terminated. It would be cheaper - // to use strings.TrimRight, but absent documentation saying - // that there is only ever a final null terminator, take the - // guaranteed correct path of terminating at the first found - // null byte. - selinux, _, _ = bytes.Cut(selinux, []byte{0}) - fileInfo.SELinux = string(selinux) + fillExtendedAttributes(fileInfo, path) group, err := user.LookupGroupId(strconv.Itoa(int(fileInfo.GID))) if err != nil { @@ -91,9 +80,25 @@ func NewMetadata(path string, info os.FileInfo) (*Metadata, error) { if fileInfo.Origin, err = GetFileOrigin(path); err != nil { errs = append(errs, err) } + return fileInfo, errs.Err() } +func fillExtendedAttributes(md *Metadata, path string) { + var selinux []byte + getExtendedAttributes(path, map[string]*[]byte{ + "security.selinux": &selinux, + "system.posix_acl_access": &md.POSIXACLAccess, + }) + // The selinux attr may be null terminated. It would be cheaper + // to use strings.TrimRight, but absent documentation saying + // that there is only ever a final null terminator, take the + // guaranteed correct path of terminating at the first found + // null byte. + selinux, _, _ = bytes.Cut(selinux, []byte{0}) + md.SELinux = string(selinux) +} + func getExtendedAttributes(path string, dst map[string]*[]byte) { f, err := os.Open(path) if err != nil { diff --git a/auditbeat/module/file_integrity/flatbuffers.go b/auditbeat/module/file_integrity/flatbuffers.go index 837d39cf226..f380e42252c 100644 --- a/auditbeat/module/file_integrity/flatbuffers.go +++ b/auditbeat/module/file_integrity/flatbuffers.go @@ -164,6 +164,14 @@ func fbWriteMetadata(b *flatbuffers.Builder, m *Metadata) flatbuffers.UOffsetT { schema.MetadataAddType(b, schema.TypeDir) case SymlinkType: schema.MetadataAddType(b, schema.TypeSymlink) + case CharDeviceType: + schema.MetadataAddType(b, schema.TypeCharDevice) + case BlockDeviceType: + schema.MetadataAddType(b, schema.TypeBlockDevice) + case FIFOType: + schema.MetadataAddType(b, schema.TypeFIFO) + case SocketType: + schema.MetadataAddType(b, schema.TypeSocket) } if selinuxOffset > 0 { schema.MetadataAddSelinux(b, selinuxOffset) @@ -191,10 +199,12 @@ func fbWriteEvent(b *flatbuffers.Builder, e *Event) flatbuffers.UOffsetT { schema.EventAddTimestampNs(b, e.Timestamp.UnixNano()) switch e.Source { - case SourceFSNotify: - schema.EventAddSource(b, schema.SourceFSNotify) case SourceScan: schema.EventAddSource(b, schema.SourceScan) + case SourceFSNotify: + schema.EventAddSource(b, schema.SourceFSNotify) + case SourceEBPF: + schema.EventAddSource(b, schema.SourceEBPF) } if targetPathOffset > 0 { @@ -235,6 +245,8 @@ func fbDecodeEvent(path string, buf []byte) *Event { rtn.Source = SourceScan case schema.SourceFSNotify: rtn.Source = SourceFSNotify + case schema.SourceEBPF: + rtn.Source = SourceEBPF } action := e.Action() @@ -285,6 +297,14 @@ func fbDecodeMetadata(e *schema.Event) *Metadata { rtn.Type = DirType case schema.TypeSymlink: rtn.Type = SymlinkType + case schema.TypeCharDevice: + rtn.Type = CharDeviceType + case schema.TypeBlockDevice: + rtn.Type = BlockDeviceType + case schema.TypeFIFO: + rtn.Type = FIFOType + case schema.TypeSocket: + rtn.Type = SocketType default: rtn.Type = UnknownType } diff --git a/auditbeat/module/file_integrity/kprobes/embed/3.10.0-1062.1.1.el7.x86_64.btf b/auditbeat/module/file_integrity/kprobes/embed/3.10.0-1062.1.1.el7.x86_64.btf new file mode 100644 index 00000000000..5aa17730ece Binary files /dev/null and b/auditbeat/module/file_integrity/kprobes/embed/3.10.0-1062.1.1.el7.x86_64.btf differ diff --git a/auditbeat/module/file_integrity/kprobes/embed/4.14.101-91.76.amzn2.aarch64.btf b/auditbeat/module/file_integrity/kprobes/embed/4.14.101-91.76.amzn2.aarch64.btf new file mode 100644 index 00000000000..94fd094090c Binary files /dev/null and b/auditbeat/module/file_integrity/kprobes/embed/4.14.101-91.76.amzn2.aarch64.btf differ diff --git a/auditbeat/module/file_integrity/kprobes/embed/4.18.0-193.1.2.el7.aarch64.btf b/auditbeat/module/file_integrity/kprobes/embed/4.18.0-193.1.2.el7.aarch64.btf new file mode 100644 index 00000000000..98d0968b692 Binary files /dev/null and b/auditbeat/module/file_integrity/kprobes/embed/4.18.0-193.1.2.el7.aarch64.btf differ diff --git a/auditbeat/module/file_integrity/kprobes/embed/5.10.0-0.deb10.17-rt-arm64.btf b/auditbeat/module/file_integrity/kprobes/embed/5.10.0-0.deb10.17-rt-arm64.btf new file mode 100644 index 00000000000..8189466782b Binary files /dev/null and b/auditbeat/module/file_integrity/kprobes/embed/5.10.0-0.deb10.17-rt-arm64.btf differ diff --git a/auditbeat/module/file_integrity/kprobes/embed/5.10.109-200.el7.aarch64.btf b/auditbeat/module/file_integrity/kprobes/embed/5.10.109-200.el7.aarch64.btf new file mode 100644 index 00000000000..6a035907819 Binary files /dev/null and b/auditbeat/module/file_integrity/kprobes/embed/5.10.109-200.el7.aarch64.btf differ diff --git a/auditbeat/module/file_integrity/kprobes/embed/5.8.0-1035-aws.btf b/auditbeat/module/file_integrity/kprobes/embed/5.8.0-1035-aws.btf new file mode 100644 index 00000000000..3a9f73b1e35 Binary files /dev/null and b/auditbeat/module/file_integrity/kprobes/embed/5.8.0-1035-aws.btf differ diff --git a/auditbeat/module/file_integrity/kprobes/errors.go b/auditbeat/module/file_integrity/kprobes/errors.go new file mode 100644 index 00000000000..f3da8878b1b --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/errors.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import "errors" + +var ( + ErrVerifyOverlappingEvents = errors.New("probe overlapping events") + ErrVerifyMissingEvents = errors.New("probe missing events") + ErrVerifyUnexpectedEvent = errors.New("received an event that was not expected") + ErrVerifyNoEventsToExpect = errors.New("no probe events to expect") + ErrSymbolNotFound = errors.New("symbol not found") + ErrAckTimeout = errors.New("timeout") +) diff --git a/auditbeat/module/file_integrity/kprobes/events.go b/auditbeat/module/file_integrity/kprobes/events.go new file mode 100644 index 00000000000..2ab2b3e1bdb --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events.go @@ -0,0 +1,142 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "sync" + + "github.com/elastic/beats/v7/auditbeat/tracing" +) + +var probeEventPool = sync.Pool{ + New: func() interface{} { + return &ProbeEvent{} + }, +} + +// ProbeEvent represents a kprobe event. +// Different Mask* fields represent different kind of events. +// For MaskMonitor, the fields that are filled are: +// +// ParentIno, ParentDevMajor, ParentDevMinor, FileIno, FileDevMajor, FileDevMinor, FileName +// +// For MaskCreate, the fields that are filled are: +// +// ParentIno, ParentDevMajor, ParentDevMinor, FileIno, FileDevMajor, FileDevMinor, FileName +// +// For MaskDelete, the fields that are filled are: +// +// ParentIno, ParentDevMajor, ParentDevMinor, FileName +// +// For MaskModify, the fields that are filled are: +// +// FileIno, FileDevMajor, FileDevMinor +// +// For MaskAttrib, the fields that are filled are: +// +// FileIno, FileDevMajor, FileDevMinor +// +// For MaskMoveTo, the fields that are filled are: +// +// ParentIno, ParentDevMajor, ParentDevMinor, FileName +// +// For MaskMoveFrom, the fields that are filled are: +// +// ParentIno, ParentDevMajor, ParentDevMinor, FileName +// +// The reason that we opted for one Type (aka ProbeEvent struct) to capture different events as +// inner fields is to utilise the same sync.Pool. As events are eventually generated by any +// process on the system, a storm of events can easily occur, avoiding constant allocations +// should benefit the performance of garbage collector. +type ProbeEvent struct { + Meta tracing.Metadata `kprobe:"metadata"` + MaskMonitor uint32 + MaskCreate uint32 `kprobe:"mc,allowundefined"` + MaskDelete uint32 `kprobe:"md,allowundefined"` + MaskAttrib uint32 `kprobe:"ma,allowundefined"` + MaskModify uint32 `kprobe:"mm,allowundefined"` + MaskDir uint32 `kprobe:"mid,allowundefined"` + MaskMoveTo uint32 `kprobe:"mmt,allowundefined"` + MaskMoveFrom uint32 `kprobe:"mmf,allowundefined"` + ParentIno uint64 `kprobe:"pi"` + ParentDevMajor uint32 `kprobe:"pdmj"` + ParentDevMinor uint32 `kprobe:"pdmn"` + FileIno uint64 `kprobe:"fi"` + FileDevMajor uint32 `kprobe:"fdmj"` + FileDevMinor uint32 `kprobe:"fdmn"` + FileName string `kprobe:"fn"` +} + +// allocProbeEvent gets a ProbeEvent from the sync.Pool and zero it out. Note that depending on the +// pool state an allocation might happen. +func allocProbeEvent() any { + probeEvent := probeEventPool.Get().(*ProbeEvent) + // zero out all Mask related fields + probeEvent.MaskMonitor = 0 + probeEvent.MaskCreate = 0 + probeEvent.MaskDelete = 0 + probeEvent.MaskAttrib = 0 + probeEvent.MaskModify = 0 + probeEvent.MaskDir = 0 + probeEvent.MaskMoveTo = 0 + probeEvent.MaskMoveFrom = 0 + return probeEvent +} + +// allocDeleteProbeEvent gets a ProbeEvent from the sync.Pool and zero it out except for MaskDelete. +// Note that depending on the pool state an allocation might happen. +func allocDeleteProbeEvent() any { + probeEvent := probeEventPool.Get().(*ProbeEvent) + // zero out all Mask related fields except MaskDelete + probeEvent.MaskMonitor = 0 + probeEvent.MaskCreate = 0 + probeEvent.MaskDelete = 1 + probeEvent.MaskAttrib = 0 + probeEvent.MaskModify = 0 + probeEvent.MaskDir = 0 + probeEvent.MaskMoveTo = 0 + probeEvent.MaskMoveFrom = 0 + return probeEvent +} + +// allocMonitorProbeEvent gets a ProbeEvent from the sync.Pool and zero it out except for MaskMonitor. +// Note that depending on the pool state an allocation might happen. +func allocMonitorProbeEvent() any { + probeEvent := probeEventPool.Get().(*ProbeEvent) + // zero out all Mask related fields except MaskMonitor + probeEvent.MaskMonitor = 1 + probeEvent.MaskCreate = 0 + probeEvent.MaskDelete = 0 + probeEvent.MaskAttrib = 0 + probeEvent.MaskModify = 0 + probeEvent.MaskDir = 0 + probeEvent.MaskMoveTo = 0 + probeEvent.MaskMoveFrom = 0 + return probeEvent +} + +// releaseProbeEvent returns a ProbeEvent to the pool. +func releaseProbeEvent(c *ProbeEvent) { + if c == nil { + return + } + + probeEventPool.Put(c) +} diff --git a/auditbeat/module/file_integrity/kprobes/events_cache.go b/auditbeat/module/file_integrity/kprobes/events_cache.go new file mode 100644 index 00000000000..d8559535ad1 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events_cache.go @@ -0,0 +1,161 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "path/filepath" +) + +type ( + dEntriesIndex map[dKey]*dEntry + dEntriesMoveIndex map[uint64]*dEntry +) + +// dEntryCache is a cache of directory entries (dEntries) that exposes appropriate methods to add, get, remove and +// handle move operations. Note that dEntryCache is designed to be utilised by a single goroutine at a time and thus +// is not thread safe. +type dEntryCache struct { + index dEntriesIndex + moveCache dEntriesMoveIndex +} + +func newDirEntryCache() *dEntryCache { + return &dEntryCache{ + index: make(map[dKey]*dEntry), + moveCache: make(map[uint64]*dEntry), + } +} + +// Get returns the dEntry associated with the given key. +func (d *dEntryCache) Get(key dKey) *dEntry { + entry, exists := d.index[key] + if !exists { + return nil + } + + return entry +} + +// removeRecursively removes the given entry and all its children from the dEntryCache. Note that it is +// the responsibility of the caller to release the resources associated with the entry by calling Release. +func removeRecursively(d *dEntryCache, entry *dEntry) { + for _, child := range entry.Children { + removeRecursively(d, child) + } + + delete(d.index, dKey{ + Ino: entry.Ino, + DevMajor: entry.DevMajor, + DevMinor: entry.DevMinor, + }) +} + +// Remove removes the given entry and all its children from the dEntryCache. Note that it is +// the responsibility of the caller to release the resources associated with the entry by calling +// Release on the dEntry. +func (d *dEntryCache) Remove(entry *dEntry) *dEntry { + if entry == nil { + return nil + } + + entry.Parent.RemoveChild(entry.Name) + entry.Parent = nil + + removeRecursively(d, entry) + return entry +} + +// Add adds the given dEntry to the dEntryCache. +func (d *dEntryCache) Add(entry *dEntry, parent *dEntry) { + if entry == nil { + return + } + + _ = addRecursive(d, entry, parent, parent.Path(), nil) +} + +// addRecursive recursively adds entries to the dEntryCache and calls a function on each entry's path (if specified). +// addRecursive satisfies the needs of Add and MoveTo. For the latter the caller would like to traverse all new dEntries +// added to the dEntryCache and this is done efficiently by providing a callback function. +func addRecursive(d *dEntryCache, entry *dEntry, parent *dEntry, rootPath string, cb func(path string) error) error { + var path string + if cb != nil { + path = filepath.Join(rootPath, entry.Name) + if err := cb(path); err != nil { + return err + } + } + + parent.AddChild(entry) + + d.index[dKey{ + Ino: entry.Ino, + DevMajor: entry.DevMajor, + DevMinor: entry.DevMinor, + }] = entry + + for _, child := range entry.Children { + if err := addRecursive(d, child, entry, path, cb); err != nil { + return err + } + } + + return nil +} + +// MoveFrom removes the given entry from the dEntryCache, adds it in the intermediate moveCache associating it +// with the caller process TID and returns it. It returns nil if the entry was not found in the dEntryCache. +// Note, that such as association between the entry and the caller process TID is mandatory as Move{To,From} events +// for older Linux kernel provide only the Filename of the moved file and only parent info is available. +func (d *dEntryCache) MoveFrom(tid uint64, entry *dEntry) { + if entry == nil { + return + } + + d.Remove(entry) + + d.moveCache[tid] = entry +} + +// MoveTo gets the entry associated with the given TID from the moveCache and moves it to the under the new parent +// entry. Also, supplying a callback function allows the caller to traverse all new dEntries added to the dEntryCache. +// It returns true if the entry was found in the moveCache and false otherwise. +func (d *dEntryCache) MoveTo(tid uint64, newParent *dEntry, newFileName string, cb func(path string) error) (bool, error) { + entry, exists := d.moveCache[tid] + if !exists { + return false, nil + } + + delete(d.moveCache, tid) + entry.Name = newFileName + + return true, addRecursive(d, entry, newParent, newParent.Path(), cb) +} + +// MoveClear removes the entry associated with the given TID from the moveCache. +func (d *dEntryCache) MoveClear(tid uint64) { + entry, exists := d.moveCache[tid] + if !exists { + return + } + + delete(d.moveCache, tid) + entry.Release() +} diff --git a/auditbeat/module/file_integrity/kprobes/events_cache_entry.go b/auditbeat/module/file_integrity/kprobes/events_cache_entry.go new file mode 100644 index 00000000000..b44b4fe5c41 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events_cache_entry.go @@ -0,0 +1,133 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import "strings" + +type dKey struct { + Ino uint64 + DevMajor uint32 + DevMinor uint32 +} + +type dEntryChildren map[string]*dEntry + +type dEntry struct { + Parent *dEntry + Depth uint32 + Children dEntryChildren + Name string + Ino uint64 + DevMajor uint32 + DevMinor uint32 +} + +func (d *dEntry) GetParent() *dEntry { + if d == nil { + return nil + } + + return d.Parent +} + +func pathRecursive(d *dEntry, buffer *strings.Builder, size int) { + nameLen := len(d.Name) + + if d.Parent == nil { + size += nameLen + buffer.Grow(size) + buffer.WriteString(d.Name) + return + } + + size += nameLen + 1 + pathRecursive(d.Parent, buffer, size) + buffer.WriteByte('/') + buffer.WriteString(d.Name) +} + +func (d *dEntry) Path() string { + if d == nil { + return "" + } + + var buffer strings.Builder + pathRecursive(d, &buffer, 0) + defer buffer.Reset() + return buffer.String() +} + +// releaseRecursive recursive func to satisfy the needs of Release. +func releaseRecursive(val *dEntry) { + for _, child := range val.Children { + releaseRecursive(child) + delete(val.Children, child.Name) + } + + val.Children = nil + val.Parent = nil +} + +// Release releases the resources associated with the given dEntry and all its children. +func (d *dEntry) Release() { + if d == nil { + return + } + + releaseRecursive(d) +} + +func (d *dEntry) RemoveChild(name string) { + if d == nil || d.Children == nil { + return + } + + delete(d.Children, name) +} + +// AddChild adds a child entry to the dEntry. +func (d *dEntry) AddChild(child *dEntry) { + if d == nil || child == nil { + return + } + + if d.Children == nil { + d.Children = make(map[string]*dEntry) + } + + child.Parent = d + child.Depth = d.Depth + 1 + + d.Children[child.Name] = child +} + +// GetChild returns the child entry with the given name, if it exists. Otherwise, nil is returned. +func (d *dEntry) GetChild(name string) *dEntry { + if d == nil || d.Children == nil { + return nil + } + + child, exists := d.Children[name] + if !exists { + return nil + } + + return child +} diff --git a/auditbeat/module/file_integrity/kprobes/events_cache_test.go b/auditbeat/module/file_integrity/kprobes/events_cache_test.go new file mode 100644 index 00000000000..ca1a39d4bcb --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events_cache_test.go @@ -0,0 +1,624 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func (d *dEntryCache) Dump(path string) error { + fileDump, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return err + } + + defer fileDump.Close() + + for _, entry := range d.index { + if _, err = fileDump.WriteString(entry.Path() + "\n"); err != nil { + return err + } + } + + return nil +} + +func Test_DirEntryCache_Add(t *testing.T) { + cases := []struct { + name string + parent *dEntry + children map[string]*dEntry + }{ + { + "dentry_no_children", + &dEntry{ + Depth: 0, + Name: "test", + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + nil, + }, + { + "dentry_with_children", + &dEntry{ + Depth: 1, + Name: "test", + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + map[string]*dEntry{ + "child1": { + Depth: 2, + Name: "child1", + Ino: 2, + DevMajor: 1, + DevMinor: 1, + }, + "child2": { + Depth: 2, + Name: "child2", + Ino: 3, + DevMajor: 1, + DevMinor: 1, + }, + }, + }, + { + // we shouldn't add nil dentries + "check_nil_dentry_add", + nil, + nil, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cache := newDirEntryCache() + + expectedLen := 0 + if c.parent != nil { + expectedLen++ + if c.children != nil { + for _, child := range c.children { + c.parent.AddChild(child) + expectedLen++ + } + } + } + + cache.Add(c.parent, nil) + + require.Len(t, cache.index, expectedLen) + if c.parent != nil { + require.Equal(t, c.parent, cache.index[dKey{ + Ino: c.parent.Ino, + DevMajor: c.parent.DevMajor, + DevMinor: c.parent.DevMinor, + }]) + } + + if c.children != nil { + for _, child := range c.children { + require.Equal(t, child, cache.index[dKey{ + Ino: child.Ino, + DevMajor: child.DevMajor, + DevMinor: child.DevMinor, + }]) + } + } + }) + } +} + +func Test_DirEntryCache_Get(t *testing.T) { + cases := []struct { + name string + key dKey + entry *dEntry + }{ + { + "dentry_exists", + dKey{ + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + &dEntry{ + Depth: 1, + Parent: nil, + Children: nil, + Name: "test", + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + }, + { + "dentry_non_existent", + dKey{ + Ino: 10000, + DevMajor: 2, + DevMinor: 3, + }, + nil, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cache := newDirEntryCache() + cache.Add(c.entry, nil) + + cacheEntry := cache.Get(c.key) + require.Equal(t, c.entry, cacheEntry) + }) + } +} + +func Test_DirEntryCache_Remove(t *testing.T) { + cases := []struct { + name string + parent *dEntry + children dEntryChildren + childrenChildren dEntryChildren + }{ + { + "dentry_no_children", + &dEntry{ + Parent: nil, + Name: "test", + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + nil, + nil, + }, + { + "dentry_with_children", + &dEntry{ + Parent: nil, + Name: "test", + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + dEntryChildren{ + "child1": { + Parent: nil, + Name: "child1", + Ino: 4, + DevMajor: 1, + DevMinor: 1, + }, + "child2": { + Parent: nil, + Name: "child2", + Ino: 7, + DevMajor: 1, + DevMinor: 1, + }, + }, + nil, + }, + { + "dentry_with_children_children", + &dEntry{ + Parent: nil, + Name: "test", + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + dEntryChildren{ + "child1": { + Parent: nil, + Name: "child1", + Ino: 4, + DevMajor: 1, + DevMinor: 1, + }, + "child2": { + Parent: nil, + Name: "child2", + Ino: 7, + DevMajor: 1, + DevMinor: 1, + }, + }, + dEntryChildren{ + "child_child1": { + Parent: nil, + Name: "child_child1", + Ino: 10, + DevMajor: 1, + DevMinor: 1, + }, + }, + }, + { + "dentry_nil", + nil, + nil, + nil, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cache := newDirEntryCache() + cache.Add(c.parent, nil) + + if c.parent != nil { + if c.children != nil { + for _, child := range c.children { + cache.Add(child, c.parent) + } + } + + if len(c.children) > 0 && c.childrenChildren != nil { + for _, childrenChildrenParent := range c.children { + for _, child := range c.childrenChildren { + cache.Add(child, childrenChildrenParent) + } + break + } + } + } + + removedEntry := cache.Remove(c.parent) + require.Len(t, cache.index, 0) + require.Equal(t, c.parent, removedEntry) + + removedEntry.Release() + if removedEntry != nil { + require.Nil(t, removedEntry.Children) + } + }) + } +} + +func Test_DirEntryCache_MoveFrom(t *testing.T) { + cases := []struct { + name string + tid uint64 + parent *dEntry + children dEntryChildren + }{ + { + "dentry_move", + 1, + &dEntry{ + Name: "test", + Depth: 0, + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + dEntryChildren{ + "child1": { + Name: "child1", + Ino: 4, + DevMajor: 1, + DevMinor: 1, + }, + "child2": { + Name: "child2", + Ino: 7, + DevMajor: 1, + DevMinor: 1, + }, + }, + }, + { + "dentry_nil", + 1, + nil, + nil, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cache := newDirEntryCache() + cache.Add(c.parent, nil) + + if c.parent != nil { + if c.children != nil { + for _, child := range c.children { + cache.Add(child, c.parent) + } + } + } + + cache.MoveFrom(c.tid, c.parent) + + require.Empty(t, cache.index) + + if c.parent == nil { + require.Len(t, cache.moveCache, 0) + return + } + + require.Len(t, cache.moveCache, 1) + + moveEntry, exists := cache.moveCache[c.tid] + require.True(t, exists) + require.Equal(t, c.parent, moveEntry) + if c.children != nil { + require.NotNil(t, c.parent.Children) + for _, child := range moveEntry.Children { + require.Equal(t, c.parent.Depth+1, child.Depth) + } + } else { + require.Nil(t, c.parent.Children) + } + }) + } +} + +func Test_DirEntryCache_MoveTo(t *testing.T) { + cases := []struct { + name string + srcTid uint64 + dstTid uint64 + entry *dEntry + children dEntryChildren + targetParent *dEntry + newFileName string + pathsToSee []string + err error + }{ + { + "dentry_move", + 1, + 1, + &dEntry{ + Name: "test", + Depth: 0, + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + dEntryChildren{ + "child1": { + Name: "child1", + Ino: 4, + DevMajor: 1, + DevMinor: 1, + }, + "child2": { + Name: "child2", + Ino: 7, + DevMajor: 1, + DevMinor: 1, + }, + }, + &dEntry{ + Name: "test2", + Depth: 0, + Ino: 10, + DevMajor: 1, + DevMinor: 1, + }, + "test3", + []string{ + "test2/test3", + "test2/test3/child1", + "test2/test3/child2", + }, + nil, + }, + { + "dentry_not_found", + 1, + 2, + &dEntry{ + Name: "test", + Depth: 0, + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + nil, + nil, + "", + nil, + nil, + }, + { + "callback_err", + 1, + 1, + &dEntry{ + Name: "test", + Depth: 0, + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + nil, + nil, + "", + nil, + errors.New("error"), + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var movedPaths []string + + cache := newDirEntryCache() + if c.entry != nil { + if c.children != nil { + for _, child := range c.children { + c.entry.AddChild(child) + } + } + cache.moveCache[c.srcTid] = c.entry + } + + movedEntry, err := cache.MoveTo(c.dstTid, c.targetParent, c.newFileName, func(path string) error { + if c.err != nil { + return c.err + } + + movedPaths = append(movedPaths, path) + return nil + }) + if c.err == nil { + require.Nil(t, err) + } else { + require.ErrorIs(t, err, c.err) + } + + if c.srcTid == c.dstTid { + require.True(t, movedEntry) + require.Empty(t, cache.moveCache) + } else { + require.False(t, movedEntry) + require.NotEmpty(t, cache.moveCache) + } + require.ElementsMatch(t, c.pathsToSee, movedPaths) + }) + } +} + +func Test_DirEntryCache_MoveClear(t *testing.T) { + cases := []struct { + name string + srcTid uint64 + dstTid uint64 + entry *dEntry + }{ + { + "dentry_move", + 1, + 1, + &dEntry{ + Name: "test", + Depth: 0, + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + }, + { + "dentry_not_found", + 1, + 2, + &dEntry{ + Name: "test", + Depth: 0, + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cache := newDirEntryCache() + if c.entry != nil { + cache.moveCache[c.srcTid] = c.entry + } + + cache.MoveClear(c.dstTid) + + if c.srcTid == c.dstTid { + require.Empty(t, cache.moveCache) + } else { + require.NotEmpty(t, cache.moveCache) + } + }) + } +} + +func Test_DirEntryCache_GetChild(t *testing.T) { + cases := []struct { + name string + entry *dEntry + children dEntryChildren + childName string + }{ + { + "dentry_with_children", + &dEntry{ + Name: "test", + Depth: 0, + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + dEntryChildren{ + "child1": { + Name: "child1", + Ino: 4, + DevMajor: 1, + DevMinor: 1, + }, + "child2": { + Name: "child2", + Ino: 7, + DevMajor: 1, + DevMinor: 1, + }, + }, + "child1", + }, + { + "dentry_no_children", + &dEntry{ + Name: "test", + Depth: 0, + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, + nil, + "child1", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + for _, child := range c.children { + c.entry.AddChild(child) + } + + childEntry := c.entry.GetChild(c.childName) + + if c.children == nil { + require.Nil(t, childEntry) + } else { + require.NotNil(t, childEntry) + } + }) + } +} diff --git a/auditbeat/module/file_integrity/kprobes/events_process.go b/auditbeat/module/file_integrity/kprobes/events_process.go new file mode 100644 index 00000000000..23e8a110f58 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events_process.go @@ -0,0 +1,240 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "errors" + "path/filepath" + + "golang.org/x/sys/unix" +) + +type Emitter interface { + Emit(ePath string, pid uint32, op uint32) error +} + +type eProcessor struct { + p pathTraverser + e Emitter + d *dEntryCache + isRecursive bool +} + +func newEventProcessor(p pathTraverser, e Emitter, isRecursive bool) *eProcessor { + return &eProcessor{ + p: p, + e: e, + d: newDirEntryCache(), + isRecursive: isRecursive, + } +} + +func (e *eProcessor) process(_ context.Context, pe *ProbeEvent) error { + // after processing return the probe event to the pool + defer releaseProbeEvent(pe) + + switch { + case pe.MaskMonitor == 1: + // Monitor events are only generated by our own pathTraverser.AddPathToMonitor or + // pathTraverser.WalkAsync + + monitorPath, match := e.p.GetMonitorPath(pe.FileIno, pe.FileDevMajor, pe.FileDevMinor, pe.FileName) + if !match { + return nil + } + + entry := e.d.Get(dKey{ + Ino: pe.FileIno, + DevMajor: pe.FileDevMajor, + DevMinor: pe.FileDevMinor, + }) + + parentEntry := e.d.Get(dKey{ + Ino: pe.ParentIno, + DevMajor: pe.ParentDevMajor, + DevMinor: pe.ParentDevMinor, + }) + + if parentEntry == nil { + entry = &dEntry{ + Name: monitorPath.fullPath, + Ino: pe.FileIno, + Depth: monitorPath.depth, + DevMajor: pe.FileDevMajor, + DevMinor: pe.FileDevMinor, + } + } else { + if entry == nil { + entry = &dEntry{ + Name: pe.FileName, + Ino: pe.FileIno, + Depth: parentEntry.Depth + 1, + DevMajor: pe.FileDevMajor, + DevMinor: pe.FileDevMinor, + } + } + } + + e.d.Add(entry, parentEntry) + + if !monitorPath.isFromMove { + return nil + } + + return e.e.Emit(entry.Path(), monitorPath.tid, unix.IN_MOVED_TO) + + case pe.MaskCreate == 1: + parentEntry := e.d.Get(dKey{ + Ino: pe.ParentIno, + DevMajor: pe.ParentDevMajor, + DevMinor: pe.ParentDevMinor, + }) + + if parentEntry == nil || parentEntry.Depth >= 1 && !e.isRecursive { + return nil + } + + entry := &dEntry{ + Children: nil, + Name: pe.FileName, + Ino: pe.FileIno, + DevMajor: pe.FileDevMajor, + DevMinor: pe.FileDevMinor, + } + + e.d.Add(entry, parentEntry) + + return e.e.Emit(entry.Path(), pe.Meta.TID, unix.IN_CREATE) + + case pe.MaskModify == 1: + entry := e.d.Get(dKey{ + Ino: pe.FileIno, + DevMajor: pe.FileDevMajor, + DevMinor: pe.FileDevMinor, + }) + + if entry == nil { + return nil + } + + return e.e.Emit(entry.Path(), pe.Meta.TID, unix.IN_MODIFY) + + case pe.MaskAttrib == 1: + entry := e.d.Get(dKey{ + Ino: pe.FileIno, + DevMajor: pe.FileDevMajor, + DevMinor: pe.FileDevMinor, + }) + + if entry == nil { + return nil + } + + return e.e.Emit(entry.Path(), pe.Meta.TID, unix.IN_ATTRIB) + + case pe.MaskMoveFrom == 1: + parentEntry := e.d.Get(dKey{ + Ino: pe.ParentIno, + DevMajor: pe.ParentDevMajor, + DevMinor: pe.ParentDevMinor, + }) + + if parentEntry == nil || parentEntry.Depth >= 1 && !e.isRecursive { + e.d.MoveClear(uint64(pe.Meta.TID)) + return nil + } + + entry := parentEntry.GetChild(pe.FileName) + if entry == nil { + return nil + } + + entryPath := entry.Path() + + e.d.MoveFrom(uint64(pe.Meta.TID), entry) + + return e.e.Emit(entryPath, pe.Meta.TID, unix.IN_MOVED_FROM) + + case pe.MaskMoveTo == 1: + parentEntry := e.d.Get(dKey{ + Ino: pe.ParentIno, + DevMajor: pe.ParentDevMajor, + DevMinor: pe.ParentDevMinor, + }) + + if parentEntry == nil || parentEntry.Depth >= 1 && !e.isRecursive { + // if parentEntry is nil then this move event is not + // for a directory we monitor + e.d.MoveClear(uint64(pe.Meta.TID)) + return nil + } + + if existingChild := parentEntry.GetChild(pe.FileName); existingChild != nil { + e.d.Remove(existingChild) + existingChild.Release() + } + + moved, err := e.d.MoveTo(uint64(pe.Meta.TID), parentEntry, pe.FileName, func(path string) error { + return e.e.Emit(path, pe.Meta.TID, unix.IN_MOVED_TO) + }) + if err != nil { + return err + } + if moved { + return nil + } + + newEntryPath := filepath.Join(parentEntry.Path(), pe.FileName) + e.p.WalkAsync(newEntryPath, parentEntry.Depth+1, pe.Meta.TID) + + return nil + + case pe.MaskDelete == 1: + parentEntry := e.d.Get(dKey{ + Ino: pe.ParentIno, + DevMajor: pe.ParentDevMajor, + DevMinor: pe.ParentDevMinor, + }) + + if parentEntry == nil || parentEntry.Depth >= 1 && !e.isRecursive { + return nil + } + + entry := parentEntry.GetChild(pe.FileName) + if entry == nil { + return nil + } + + entryPath := entry.Path() + + e.d.Remove(entry) + + if err := e.e.Emit(entryPath, pe.Meta.TID, unix.IN_DELETE); err != nil { + return err + } + + entry.Release() + + return nil + default: + return errors.New("unknown event type") + } +} diff --git a/auditbeat/module/file_integrity/kprobes/events_process_test.go b/auditbeat/module/file_integrity/kprobes/events_process_test.go new file mode 100644 index 00000000000..1d0b44b2622 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events_process_test.go @@ -0,0 +1,678 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "testing" + + "github.com/elastic/beats/v7/auditbeat/tracing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +type EmitterMock struct { + mock.Mock +} + +func (e *EmitterMock) Emit(ePath string, pid uint32, op uint32) error { + args := e.Called(ePath, pid, op) + return args.Error(0) +} + +func Test_EventProcessor_process(t *testing.T) { + type emitted struct { + path string + pid uint32 + op uint32 + } + + cases := []struct { + name string + statMatches []statMatch + events []*ProbeEvent + emits []emitted + isRecursive bool + }{ + { + "recursive_processor", + []statMatch{ + { + ino: 1, + major: 1, + minor: 1, + depth: 0, + fileName: "root", + isFromMove: false, + tid: 0, + fullPath: "/root/test", + }, + { + ino: 10, + major: 1, + minor: 1, + depth: 0, + fileName: "root2", + isFromMove: false, + tid: 0, + fullPath: "/root2/test", + }, + }, + []*ProbeEvent{ + { + // shouldn't add to cache + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "root", + FileIno: 1, + FileDevMajor: 100, + FileDevMinor: 100, + }, + { + // shouldn't add to cache + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "root", + FileIno: 1, + FileDevMajor: 200, + FileDevMinor: 200, + }, + { + // should add to cache but no event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "root", + FileIno: 1, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should add to cache but no event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "root2", + FileIno: 10, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit create event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskCreate: 1, + ParentDevMinor: 1, + ParentIno: 1, + ParentDevMajor: 1, + FileName: "test_create", + FileIno: 2, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should not emit create event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskCreate: 1, + ParentDevMinor: 1, + ParentIno: 3, + ParentDevMajor: 1, + FileName: "test_create", + FileIno: 2, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should not emit modify event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskModify: 1, + FileIno: 3, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit modify event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskModify: 1, + FileIno: 2, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should not emit attrib event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskAttrib: 1, + FileIno: 3, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit attrib event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskAttrib: 1, + FileIno: 2, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit delete event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskDelete: 1, + ParentDevMinor: 1, + ParentIno: 1, + ParentDevMajor: 1, + FileName: "test_create", + }, + { + // should not emit delete event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskDelete: 1, + ParentDevMinor: 1, + ParentIno: 3, + ParentDevMajor: 1, + FileName: "test_create", + }, + { + // should emit create event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskCreate: 1, + ParentDevMinor: 1, + ParentIno: 10, + ParentDevMajor: 1, + FileName: "test_create2", + FileIno: 11, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit create event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskCreate: 1, + ParentDevMinor: 1, + ParentIno: 11, + ParentDevMajor: 1, + FileName: "test_child", + FileIno: 12, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit move_from event + Meta: tracing.Metadata{ + PID: 2, + TID: 2, + }, + MaskMoveFrom: 1, + ParentDevMinor: 1, + ParentIno: 10, + ParentDevMajor: 1, + FileName: "test_create2", + }, + { + // should emit two move_to events + Meta: tracing.Metadata{ + PID: 2, + TID: 2, + }, + MaskMoveTo: 1, + ParentDevMinor: 1, + ParentIno: 1, + ParentDevMajor: 1, + FileName: "test_create_moved2", + }, + { + // should emit two move_to events + Meta: tracing.Metadata{ + PID: 3, + TID: 3, + }, + MaskMoveTo: 1, + ParentDevMinor: 1, + ParentIno: 1, + ParentDevMajor: 1, + FileName: "test_create_moved_outside", + }, + }, + []emitted{ + { + path: "/root/test/test_create", + pid: 1, + op: unix.IN_CREATE, + }, + { + path: "/root/test/test_create", + pid: 1, + op: unix.IN_MODIFY, + }, + { + path: "/root/test/test_create", + pid: 1, + op: unix.IN_ATTRIB, + }, + { + path: "/root/test/test_create", + pid: 1, + op: unix.IN_DELETE, + }, + { + path: "/root2/test/test_create2", + pid: 1, + op: unix.IN_CREATE, + }, + { + path: "/root2/test/test_create2/test_child", + pid: 1, + op: unix.IN_CREATE, + }, + { + path: "/root2/test/test_create2", + pid: 2, + op: unix.IN_MOVED_FROM, + }, + { + path: "/root/test/test_create_moved2", + pid: 2, + op: unix.IN_MOVED_TO, + }, + { + path: "/root/test/test_create_moved2/test_child", + pid: 2, + op: unix.IN_MOVED_TO, + }, + { + path: "/root/test/test_create_moved_outside", + pid: 3, + op: unix.IN_MOVED_TO, + }, + }, + true, + }, + { + "nonrecursive_processor", + []statMatch{ + { + ino: 10, + major: 1, + minor: 1, + depth: 0, + fileName: "target_dir", + isFromMove: false, + tid: 0, + fullPath: "/target_dir", + }, + { + ino: 11, + major: 1, + minor: 1, + depth: 1, + fileName: "track_me", + isFromMove: false, + tid: 0, + fullPath: "/target_dir/track_me", + }, + { + ino: 100, + major: 1, + minor: 1, + depth: 1, + fileName: "nested", + isFromMove: false, + tid: 0, + fullPath: "/target_dir/nested", + }, + { + ino: 1000, + major: 1, + minor: 1, + depth: 2, + fileName: "deeper", + isFromMove: false, + tid: 0, + fullPath: "/target_dir/nested/deeper", + }, + }, + []*ProbeEvent{ + { + // shouldn't add to cache + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "target_dir", + FileIno: 1, + FileDevMajor: 100, + FileDevMinor: 100, + }, + { + // should add to cache but no event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "target_dir", + FileIno: 10, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should add to cache but no event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "track_me", + FileIno: 11, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should add to cache but no event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "nested", + FileIno: 100, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // shouldn't add to cache and no event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskMonitor: 1, + FileName: "deeper", + FileIno: 1000, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit create event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskCreate: 1, + ParentDevMinor: 1, + ParentIno: 10, + ParentDevMajor: 1, + FileName: "test_create", + FileIno: 12, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should not emit create event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskCreate: 1, + ParentDevMinor: 1, + ParentIno: 100, + ParentDevMajor: 1, + FileName: "test_create", + FileIno: 101, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should not emit modify event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskModify: 1, + FileIno: 101, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit modify event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskModify: 1, + FileIno: 12, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit modify event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskModify: 1, + FileIno: 11, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should not emit attrib event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskAttrib: 1, + FileIno: 101, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit attrib event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskAttrib: 1, + FileIno: 11, + FileDevMajor: 1, + FileDevMinor: 1, + }, + { + // should emit delete event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskDelete: 1, + ParentDevMinor: 1, + ParentIno: 10, + ParentDevMajor: 1, + FileName: "test_create", + }, + { + // should not emit delete event + Meta: tracing.Metadata{ + PID: 1, + TID: 1, + }, + MaskDelete: 1, + ParentDevMinor: 1, + ParentIno: 100, + ParentDevMajor: 1, + FileName: "test_create", + }, + }, + []emitted{ + { + path: "/target_dir/test_create", + pid: 1, + op: unix.IN_CREATE, + }, + { + path: "/target_dir/test_create", + pid: 1, + op: unix.IN_MODIFY, + }, + { + path: "/target_dir/track_me", + pid: 1, + op: unix.IN_MODIFY, + }, + { + path: "/target_dir/track_me", + pid: 1, + op: unix.IN_ATTRIB, + }, + { + path: "/target_dir/test_create", + pid: 1, + op: unix.IN_DELETE, + }, + }, + false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var emittedEvents []emitted + + mockEmitter := &EmitterMock{} + mockEmitterCall := mockEmitter.On("Emit", mock.Anything, mock.Anything, mock.Anything) + mockEmitterCall.Run(func(args mock.Arguments) { + emittedEvents = append(emittedEvents, emitted{ + path: args.Get(0).(string), + pid: args.Get(1).(uint32), + op: args.Get(2).(uint32), + }) + mockEmitterCall.ReturnArguments = []any{nil} + }) + + mockPathTraverser := &pathTraverserMock{} + mockPathTraverserCall := mockPathTraverser.On("GetMonitorPath", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + mockPathTraverserCall.Run(func(args mock.Arguments) { + ino := args.Get(0).(uint64) + major := args.Get(1).(uint32) + minor := args.Get(2).(uint32) + name := args.Get(3).(string) + if len(c.statMatches) == 0 { + mockPathTraverserCall.ReturnArguments = []any{MonitorPath{}, false} + return + } + + if c.statMatches[0].ino != ino || + c.statMatches[0].major != major || + c.statMatches[0].minor != minor || + c.statMatches[0].fileName != name { + mockPathTraverserCall.ReturnArguments = []any{MonitorPath{}, false} + return + } + + mockPathTraverserCall.ReturnArguments = []any{MonitorPath{ + fullPath: c.statMatches[0].fullPath, + depth: c.statMatches[0].depth, + isFromMove: c.statMatches[0].isFromMove, + tid: c.statMatches[0].tid, + }, true} + + c.statMatches = c.statMatches[1:] + }) + + mockPathTraverser.On("WalkAsync", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + pid := args.Get(2).(uint32) + + c.statMatches = append(c.statMatches, statMatch{ + fullPath: args.Get(0).(string), + depth: args.Get(1).(uint32), + ino: 20, + major: 1, + minor: 1, + isFromMove: true, + fileName: "test_create_moved_outside", + tid: pid, + }) + + c.events = append(c.events, []*ProbeEvent{ + { + Meta: tracing.Metadata{PID: 1, TID: 1}, + MaskMonitor: 1, + ParentDevMinor: 1, + ParentIno: 1, + ParentDevMajor: 1, + FileName: "test_create_moved_outside", + FileIno: 20, + FileDevMajor: 1, + FileDevMinor: 1, + }, + }...) + }) + + eProc := newEventProcessor(mockPathTraverser, mockEmitter, c.isRecursive) + for len(c.events) > 0 { + err := eProc.process(context.TODO(), c.events[0]) + require.NoError(t, err) + c.events = c.events[1:] + } + + require.Equal(t, c.emits, emittedEvents) + }) + } +} diff --git a/auditbeat/module/file_integrity/kprobes/events_test.go b/auditbeat/module/file_integrity/kprobes/events_test.go new file mode 100644 index 00000000000..b39e0621ab5 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events_test.go @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_allocProbeEvents(t *testing.T) { + p := allocProbeEvent() + require.IsType(t, &ProbeEvent{}, p) + + releaseProbeEvent(nil) + + pE := p.(*ProbeEvent) + require.Zero(t, pE.MaskMonitor) + require.Zero(t, pE.MaskCreate) + require.Zero(t, pE.MaskDelete) + require.Zero(t, pE.MaskAttrib) + require.Zero(t, pE.MaskModify) + require.Zero(t, pE.MaskDir) + require.Zero(t, pE.MaskMoveTo) + require.Zero(t, pE.MaskMoveFrom) + releaseProbeEvent(pE) + + p = allocDeleteProbeEvent() + require.IsType(t, &ProbeEvent{}, p) + + pE = p.(*ProbeEvent) + require.Zero(t, pE.MaskMonitor) + require.Zero(t, pE.MaskCreate) + require.Equal(t, pE.MaskDelete, uint32(1)) + require.Zero(t, pE.MaskAttrib) + require.Zero(t, pE.MaskModify) + require.Zero(t, pE.MaskDir) + require.Zero(t, pE.MaskMoveTo) + require.Zero(t, pE.MaskMoveFrom) + releaseProbeEvent(pE) + + p = allocMonitorProbeEvent() + require.IsType(t, &ProbeEvent{}, p) + + pE = p.(*ProbeEvent) + require.Equal(t, pE.MaskMonitor, uint32(1)) + require.Zero(t, pE.MaskCreate) + require.Zero(t, pE.MaskDelete) + require.Zero(t, pE.MaskAttrib) + require.Zero(t, pE.MaskModify) + require.Zero(t, pE.MaskDir) + require.Zero(t, pE.MaskMoveTo) + require.Zero(t, pE.MaskMoveFrom) + releaseProbeEvent(pE) +} + +func BenchmarkEventAllocation(b *testing.B) { + var p *ProbeEvent + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < 10000; j++ { + p = &ProbeEvent{} + _ = p + p = &ProbeEvent{MaskMonitor: 1} + _ = p + p = &ProbeEvent{MaskDelete: 1} + _ = p + } + } + _ = p +} + +func BenchmarkEventPool(b *testing.B) { + var p any + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < 10000; j++ { + p = allocProbeEvent() + _ = p + releaseProbeEvent(p.(*ProbeEvent)) + p = allocMonitorProbeEvent() + _ = p + releaseProbeEvent(p.(*ProbeEvent)) + p = allocDeleteProbeEvent() + _ = p + releaseProbeEvent(p.(*ProbeEvent)) + } + } + _ = p +} diff --git a/auditbeat/module/file_integrity/kprobes/events_verifier.go b/auditbeat/module/file_integrity/kprobes/events_verifier.go new file mode 100644 index 00000000000..2a5f902b363 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events_verifier.go @@ -0,0 +1,280 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "os" + "path/filepath" + "sync" + "time" + + "golang.org/x/sys/unix" +) + +type eventID struct { + path string + op uint32 +} + +var eventGenerators = []func(*eventsVerifier, string, string) error{ + // create file - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + file, err := os.OpenFile(targetFilePath, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return err + } + defer file.Close() + e.addEventToExpect(targetFilePath, unix.IN_CREATE) + return nil + }, + // truncate file - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Truncate(targetFilePath, 0); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_MODIFY) + return nil + }, + // write to file - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + file, err := os.OpenFile(targetFilePath, os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer file.Close() + if _, err := file.WriteString("test"); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_MODIFY) + return nil + }, + // change owner of file - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Chown(targetFilePath, os.Getuid(), os.Getgid()); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_ATTRIB) + return nil + }, + // change mode of file - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Chmod(targetFilePath, 0o700); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_ATTRIB) + return nil + }, + // change times of file - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := unix.Utimes(targetFilePath, []unix.Timeval{ + unix.NsecToTimeval(time.Now().UnixNano()), + unix.NsecToTimeval(time.Now().UnixNano()), + }); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_ATTRIB) + return nil + }, + // add attribute to file - generates 1 event + // Note that this may fail if the filesystem doesn't support extended attributes + // This is allVerified we just skip adding the respective event to verify + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + attrName := "user.myattr" + attrValue := []byte("Hello, xattr!") + if err := unix.Setxattr(targetFilePath, attrName, attrValue, 0); err != nil { + if !errors.Is(err, unix.EOPNOTSUPP) { + return err + } + } else { + e.addEventToExpect(targetFilePath, unix.IN_ATTRIB) + } + return nil + }, + // move file - generates 2 events + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Rename(targetFilePath, targetMovedFilePath); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_MOVED_FROM) + e.addEventToExpect(targetMovedFilePath, unix.IN_MOVED_TO) + return nil + }, + // remove file - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Remove(targetMovedFilePath); err != nil { + return err + } + e.addEventToExpect(targetMovedFilePath, unix.IN_DELETE) + return nil + }, + // create a directory - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Mkdir(targetFilePath, 0o600); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_CREATE) + return nil + }, + // change mode of directory - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Chmod(targetFilePath, 0o644); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_ATTRIB) + return nil + }, + // change owner of directory - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Chown(targetFilePath, os.Getuid(), os.Getgid()); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_ATTRIB) + return nil + }, + // add attribute to directory - generates 1 event + // Note that this may fail if the filesystem doesn't support extended attributes + // This is allVerified we just skip adding the respective event to verify + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + attrName := "user.myattr" + attrValue := []byte("Hello, xattr!") + if err := unix.Setxattr(targetFilePath, attrName, attrValue, 0); err != nil { + if !errors.Is(err, unix.EOPNOTSUPP) { + return err + } + } else { + e.addEventToExpect(targetFilePath, unix.IN_ATTRIB) + } + return nil + }, + // change times of directory - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := unix.Utimes(targetFilePath, []unix.Timeval{ + unix.NsecToTimeval(time.Now().UnixNano()), + unix.NsecToTimeval(time.Now().UnixNano()), + }); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_ATTRIB) + return nil + }, + // move directory - generates 2 events + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Rename(targetFilePath, targetMovedFilePath); err != nil { + return err + } + e.addEventToExpect(targetFilePath, unix.IN_MOVED_FROM) + e.addEventToExpect(targetMovedFilePath, unix.IN_MOVED_TO) + return nil + }, + // remove the directory - generates 1 event + func(e *eventsVerifier, targetFilePath string, targetMovedFilePath string) error { + if err := os.Remove(targetMovedFilePath); err != nil { + return err + } + e.addEventToExpect(targetMovedFilePath, unix.IN_DELETE) + return nil + }, +} + +type eventsVerifier struct { + sync.Mutex + basePath string + eventsToExpect map[eventID]int + eventsToExpectNr int +} + +func newEventsVerifier(basePath string) (*eventsVerifier, error) { + return &eventsVerifier{ + basePath: basePath, + eventsToExpect: make(map[eventID]int), + }, nil +} + +func (e *eventsVerifier) validateEvent(path string, _ uint32, op uint32) error { + e.Lock() + defer e.Unlock() + + eID := eventID{ + path: path, + op: op, + } + _, exists := e.eventsToExpect[eID] + + if !exists { + return ErrVerifyUnexpectedEvent + } + + e.eventsToExpect[eID]-- + return nil +} + +// addEventToExpect adds an event to the eventsVerifier's list of expected events. +func (e *eventsVerifier) addEventToExpect(path string, op uint32) { + e.eventsToExpectNr++ + + eID := eventID{ + path: path, + op: op, + } + _, exists := e.eventsToExpect[eID] + + if !exists { + e.eventsToExpect[eID] = 1 + return + } + + e.eventsToExpect[eID]++ +} + +func (e *eventsVerifier) GenerateEvents() error { + targetFilePath := filepath.Join(e.basePath, "validate_file") + targetMovedFilePath := targetFilePath + "_moved" + + for _, genFunc := range eventGenerators { + e.Lock() + if err := genFunc(e, targetFilePath, targetMovedFilePath); err != nil { + e.Unlock() + return err + } + e.Unlock() + } + + return nil +} + +// Verified checks that all expected events filled during GenerateEvents() are present without any missing +// or duplicated. +func (e *eventsVerifier) Verified() error { + if e.eventsToExpectNr == 0 { + return ErrVerifyNoEventsToExpect + } + + for _, status := range e.eventsToExpect { + switch { + case status < 0: + return ErrVerifyOverlappingEvents + case status > 0: + return ErrVerifyMissingEvents + } + } + + return nil +} diff --git a/auditbeat/module/file_integrity/kprobes/events_verifier_test.go b/auditbeat/module/file_integrity/kprobes/events_verifier_test.go new file mode 100644 index 00000000000..c630f8a2e69 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/events_verifier_test.go @@ -0,0 +1,192 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func Test_EventsVerifier(t *testing.T) { + type verifierEvents struct { + path string + op uint32 + } + + cases := []struct { + name string + emitErr error + verifyErr error + expectedEvents []verifierEvents + emittedEvents []verifierEvents + }{ + { + "no_error", + nil, + nil, + []verifierEvents{ + { + path: "test", + op: unix.IN_ATTRIB, + }, + { + path: "test", + op: unix.IN_MOVED_FROM, + }, + { + path: "test", + op: unix.IN_MOVED_TO, + }, + { + path: "test", + op: unix.IN_MODIFY, + }, + { + path: "test", + op: unix.IN_CREATE, + }, + { + path: "test", + op: unix.IN_DELETE, + }, + }, + []verifierEvents{ + { + path: "test", + op: unix.IN_ATTRIB, + }, + { + path: "test", + op: unix.IN_MOVED_FROM, + }, + { + path: "test", + op: unix.IN_MOVED_TO, + }, + { + path: "test", + op: unix.IN_MODIFY, + }, + { + path: "test", + op: unix.IN_CREATE, + }, + { + path: "test", + op: unix.IN_DELETE, + }, + }, + }, { + "overlapping_events", + nil, + ErrVerifyOverlappingEvents, + []verifierEvents{ + { + path: "test", + op: unix.IN_ATTRIB, + }, + }, + []verifierEvents{ + { + path: "test", + op: unix.IN_ATTRIB, + }, + { + path: "test", + op: unix.IN_ATTRIB, + }, + }, + }, { + "missing_events", + nil, + ErrVerifyMissingEvents, + []verifierEvents{ + { + path: "test", + op: unix.IN_ATTRIB, + }, + }, + nil, + }, { + "unexpected_events", + ErrVerifyUnexpectedEvent, + nil, + []verifierEvents{ + { + path: "test", + op: unix.IN_ATTRIB, + }, + }, + []verifierEvents{ + { + path: "test", + op: unix.IN_DELETE, + }, + }, + }, { + "no_events_to_expect", + nil, + ErrVerifyNoEventsToExpect, + nil, + nil, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + e, err := newEventsVerifier("") + require.NoError(t, err) + + for _, ev := range c.expectedEvents { + e.addEventToExpect(ev.path, ev.op) + } + + for _, ev := range c.emittedEvents { + require.ErrorIs(t, e.validateEvent(ev.path, 0, ev.op), c.emitErr) + if c.emitErr != nil { + return + } + } + + require.ErrorIs(t, e.Verified(), c.verifyErr) + }) + } +} + +func Test_EventsVerifier_GenerateEvents(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + require.NoError(t, err) + + defer func() { + rmErr := os.RemoveAll(tmpDir) + require.NoError(t, rmErr) + }() + + e, err := newEventsVerifier(tmpDir) + require.NoError(t, err) + + err = e.GenerateEvents() + require.NoError(t, err) + + require.NotEmpty(t, e.eventsToExpect) +} diff --git a/auditbeat/module/file_integrity/kprobes/executor.go b/auditbeat/module/file_integrity/kprobes/executor.go new file mode 100644 index 00000000000..2f0485dde3c --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/executor.go @@ -0,0 +1,127 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "runtime" + + "golang.org/x/sys/unix" +) + +type executor interface { + Run(f func() error) error + GetTID() int +} + +// fixedExecutor runs tasks on a fixed OS thread (see runtime.LockOSThread). +type fixedExecutor struct { + ctx context.Context + cancelFn context.CancelFunc + // tid is the OS identifier for the thread where it is running. + tid int + runC chan func() error + retC chan error +} + +// Run submits new tasks to run on the executor and waits for them to finish returning any error. +func (ex *fixedExecutor) Run(f func() error) error { + if ctxErr := ex.ctx.Err(); ctxErr != nil { + return ctxErr + } + + select { + case ex.runC <- f: + case <-ex.ctx.Done(): + return ex.ctx.Err() + } + + select { + case <-ex.ctx.Done(): + return ex.ctx.Err() + case err := <-ex.retC: + return err + } +} + +// GetTID returns the OS identifier for the thread where executor goroutine is locked against. +func (ex *fixedExecutor) GetTID() int { + return ex.tid +} + +// Close terminates the executor. Pending tasks will still be run. +func (ex *fixedExecutor) Close() { + ex.cancelFn() + close(ex.runC) +} + +// newFixedThreadExecutor returns a new fixedExecutor. +func newFixedThreadExecutor(ctx context.Context) *fixedExecutor { + mCtx, cancelFn := context.WithCancel(ctx) + + ex := &fixedExecutor{ + ctx: mCtx, + cancelFn: cancelFn, + runC: make(chan func() error, 1), + retC: make(chan error, 1), + } + + tidC := make(chan int) + + go func() { + defer close(ex.retC) + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + select { + case <-ctx.Done(): + return + case tidC <- unix.Gettid(): + close(tidC) + } + + for { + select { + case runF, ok := <-ex.runC: + if !ok { + // channel closed + return + } + + select { + case ex.retC <- runF(): + case <-ex.ctx.Done(): + return + } + + case <-ex.ctx.Done(): + return + } + } + }() + + select { + case ex.tid = <-tidC: + case <-ctx.Done(): + return nil + } + + return ex +} diff --git a/auditbeat/module/file_integrity/kprobes/executor_test.go b/auditbeat/module/file_integrity/kprobes/executor_test.go new file mode 100644 index 00000000000..c016bbf9766 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/executor_test.go @@ -0,0 +1,132 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_executor(t *testing.T) { + // parent context is cancelled at creation + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + exec := newFixedThreadExecutor(ctx) + require.Nil(t, exec) + + // parent context is cancelled + ctx, cancel = context.WithCancel(context.Background()) + exec = newFixedThreadExecutor(ctx) + require.NotNil(t, exec) + + err := exec.Run(func() error { + cancel() + time.Sleep(10 * time.Second) + return nil + }) + require.ErrorIs(t, err, ctx.Err()) + require.ErrorIs(t, exec.Run(func() error { + return nil + }), ctx.Err()) + + // executor is closed while running cancelled + exec = newFixedThreadExecutor(context.Background()) + require.NotNil(t, exec) + + err = exec.Run(func() error { + exec.Close() + time.Sleep(10 * time.Second) + return nil + }) + require.ErrorIs(t, err, exec.ctx.Err()) + + // normal exec no error + exec = newFixedThreadExecutor(context.Background()) + require.NotNil(t, exec) + + err = exec.Run(func() error { + time.Sleep(1 * time.Second) + return nil + }) + require.NoError(t, err) + exec.Close() + + // exec with error + exec = newFixedThreadExecutor(context.Background()) + require.NotNil(t, exec) + retErr := errors.New("test error") + + err = exec.Run(func() error { + return retErr + }) + require.ErrorIs(t, err, retErr) + exec.Close() + + // check that runs are indeed sequential + // as pathTraverser depends on it + err = nil + atomicInt := uint32(0) + atomicCheck := func() error { + swapped := atomic.CompareAndSwapUint32(&atomicInt, 0, 1) + if !swapped { + return errors.New("parallel runs") + } + time.Sleep(1 * time.Second) + swapped = atomic.CompareAndSwapUint32(&atomicInt, 1, 0) + if !swapped { + return errors.New("parallel runs") + } + return nil + } + exec = newFixedThreadExecutor(context.Background()) + require.NotNil(t, exec) + errChannel := make(chan error, 1) + wg := sync.WaitGroup{} + start := make(chan struct{}) + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + if runErr := exec.Run(atomicCheck); runErr != nil { + select { + case errChannel <- runErr: + default: + } + } + }() + } + time.Sleep(1 * time.Second) + close(start) + wg.Wait() + select { + case err = <-errChannel: + default: + + } + require.Nil(t, err) +} diff --git a/auditbeat/module/file_integrity/kprobes/kallsyms.go b/auditbeat/module/file_integrity/kprobes/kallsyms.go new file mode 100644 index 00000000000..88ea033f884 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/kallsyms.go @@ -0,0 +1,97 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "bufio" + "fmt" + "io" + "os" + "regexp" + "strings" +) + +const kAllSymsPath = "/proc/kallsyms" + +type runtimeSymbolInfo struct { + symbolName string + isOptimised bool + optimisedSymbolName string +} + +// getSymbolInfoRuntime returns the runtime symbol information for the given symbolName +// from the /proc/kallsyms file. +func getSymbolInfoRuntime(symbolName string) (runtimeSymbolInfo, error) { + kAllSymsFile, err := os.Open(kAllSymsPath) + if err != nil { + return runtimeSymbolInfo{}, err + } + + defer kAllSymsFile.Close() + + return getSymbolInfoFromReader(kAllSymsFile, symbolName) +} + +// getSymbolInfoFromReader retrieves symbol information from a reader that is expected to +// provide content in the same format as /proc/kallsyms +func getSymbolInfoFromReader(reader io.Reader, symbolName string) (runtimeSymbolInfo, error) { + fileScanner := bufio.NewScanner(reader) + fileScanner.Split(bufio.ScanLines) + + symReg, err := regexp.Compile(fmt.Sprintf("(?m)^([a-fA-F0-9]+).*?(%s(|.*?)?)(\\s+.*$|$)", symbolName)) + if err != nil { + return runtimeSymbolInfo{}, err + } + + // optimised symbols start with the unoptimised symbol name + // followed by ".{optimisation_type}..." + optimisedSymbolName := symbolName + "." + + for fileScanner.Scan() { + matches := symReg.FindAllSubmatch(fileScanner.Bytes(), -1) + if len(matches) == 0 { + continue + } + + for _, match := range matches { + matchSymbolName := string(match[2]) + switch { + case strings.HasPrefix(matchSymbolName, optimisedSymbolName): + return runtimeSymbolInfo{ + symbolName: symbolName, + isOptimised: true, + optimisedSymbolName: matchSymbolName, + }, nil + case strings.EqualFold(matchSymbolName, symbolName): + return runtimeSymbolInfo{ + symbolName: symbolName, + isOptimised: false, + optimisedSymbolName: "", + }, nil + } + } + } + + if fileScanner.Err() != nil { + return runtimeSymbolInfo{}, err + } + + return runtimeSymbolInfo{}, ErrSymbolNotFound +} diff --git a/auditbeat/module/file_integrity/kprobes/kallsyms_test.go b/auditbeat/module/file_integrity/kprobes/kallsyms_test.go new file mode 100644 index 00000000000..beeb4693cf4 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/kallsyms_test.go @@ -0,0 +1,75 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_getSymbolInfoFromReader(t *testing.T) { + content := `0000000000000000 t fsnotify_move +0000000000000000 T fsnotify +0000000000000000 T fsnotifyy +0000000000000000 t fsnotify_file.isra.0 [btrfs] +0000000000000000 t chmod_common.isra.0` + + cases := []struct { + tName string + symbolName string + isOptimised bool + optimisedSymbolName string + err error + }{ + { + tName: "symbol_exists", + symbolName: "fsnotify", + isOptimised: false, + optimisedSymbolName: "", + err: nil, + }, + { + tName: "symbol_exists_optimised", + symbolName: "chmod_common", + isOptimised: true, + optimisedSymbolName: "chmod_common.isra.0", + err: nil, + }, + { + tName: "symbol_exists_optimised_with_space_at_end", + symbolName: "fsnotify_file", + isOptimised: true, + optimisedSymbolName: "fsnotify_file.isra.0", + err: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.tName, func(t *testing.T) { + symInfo, err := getSymbolInfoFromReader(strings.NewReader(content), tc.symbolName) + require.IsType(t, err, tc.err) + require.Equal(t, tc.symbolName, symInfo.symbolName) + require.Equal(t, tc.isOptimised, symInfo.isOptimised) + require.Equal(t, tc.optimisedSymbolName, symInfo.optimisedSymbolName) + }) + } +} diff --git a/auditbeat/module/file_integrity/kprobes/monitor.go b/auditbeat/module/file_integrity/kprobes/monitor.go new file mode 100644 index 00000000000..1b3cef35aed --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/monitor.go @@ -0,0 +1,225 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "time" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/go-perf" +) + +type MonitorEvent struct { + Path string + PID uint32 + Op uint32 +} + +type monitorEmitter struct { + ctx context.Context + eventC chan<- MonitorEvent +} + +func newMonitorEmitter(ctx context.Context, eventC chan MonitorEvent) *monitorEmitter { + return &monitorEmitter{ + ctx: ctx, + eventC: eventC, + } +} + +func (m *monitorEmitter) Emit(ePath string, pid uint32, op uint32) error { + select { + case <-m.ctx.Done(): + return m.ctx.Err() + + case m.eventC <- MonitorEvent{ + Path: ePath, + PID: pid, + Op: op, + }: + return nil + } +} + +type Monitor struct { + eventC chan MonitorEvent + pathMonitor *pTraverser + perfChannel perfChannel + errC chan error + eProc *eProcessor + log *logp.Logger + ctx context.Context + cancelFn context.CancelFunc + running uint32 + isRecursive bool + closeErr error +} + +func New(isRecursive bool) (*Monitor, error) { + ctx := context.TODO() + + validatedProbes, exec, err := getVerifiedProbes(ctx, 5*time.Second) + if err != nil { + return nil, err + } + + pChannel, err := newPerfChannel(validatedProbes, 10, 4096, perf.AllThreads) + if err != nil { + return nil, err + } + + return newMonitor(ctx, isRecursive, pChannel, exec) +} + +func newMonitor(ctx context.Context, isRecursive bool, pChannel perfChannel, exec executor) (*Monitor, error) { + mCtx, cancelFunc := context.WithCancel(ctx) + + p, err := newPathMonitor(mCtx, exec, 0, isRecursive) + if err != nil { + cancelFunc() + return nil, err + } + + eventChannel := make(chan MonitorEvent, 512) + eProc := newEventProcessor(p, newMonitorEmitter(mCtx, eventChannel), isRecursive) + + return &Monitor{ + eventC: eventChannel, + pathMonitor: p, + perfChannel: pChannel, + errC: make(chan error, 1), + eProc: eProc, + log: logp.NewLogger("file_integrity"), + ctx: mCtx, + cancelFn: cancelFunc, + isRecursive: isRecursive, + closeErr: nil, + }, nil +} + +func (w *Monitor) Add(path string) error { + switch atomic.LoadUint32(&w.running) { + case 0: + return errors.New("monitor not started") + case 2: + return errors.New("monitor is closed") + } + + return w.pathMonitor.AddPathToMonitor(w.ctx, path) +} + +func (w *Monitor) Close() error { + if !atomic.CompareAndSwapUint32(&w.running, 1, 2) { + switch atomic.LoadUint32(&w.running) { + case 0: + // monitor hasn't started yet + atomic.StoreUint32(&w.running, 2) + default: + return nil + } + } + + w.cancelFn() + var allErr error + allErr = errors.Join(allErr, w.pathMonitor.Close()) + allErr = errors.Join(allErr, w.perfChannel.Close()) + + return allErr +} + +func (w *Monitor) EventChannel() <-chan MonitorEvent { + return w.eventC +} + +func (w *Monitor) ErrorChannel() <-chan error { + return w.errC +} + +func (w *Monitor) writeErr(err error) { + select { + case w.errC <- err: + case <-w.ctx.Done(): + } +} + +func (w *Monitor) Start() error { + if !atomic.CompareAndSwapUint32(&w.running, 0, 1) { + return errors.New("monitor already started") + } + + if err := w.perfChannel.Run(); err != nil { + if closeErr := w.Close(); closeErr != nil { + w.log.Warnf("error at closing watcher: %v", closeErr) + } + return err + } + + go func() { + defer func() { + close(w.eventC) + if closeErr := w.Close(); closeErr != nil { + w.log.Warnf("error at closing watcher: %v", closeErr) + } + }() + + for { + select { + case <-w.ctx.Done(): + return + + case e, ok := <-w.perfChannel.C(): + if !ok { + w.writeErr(fmt.Errorf("read invalid event from perf channel")) + return + } + + switch eWithType := e.(type) { + case *ProbeEvent: + if err := w.eProc.process(w.ctx, eWithType); err != nil { + w.writeErr(err) + return + } + continue + default: + w.writeErr(errors.New("unexpected event type")) + return + } + + case err := <-w.perfChannel.ErrC(): + w.writeErr(err) + return + + case lost := <-w.perfChannel.LostC(): + w.writeErr(fmt.Errorf("events lost %d", lost)) + return + + case err := <-w.pathMonitor.ErrC(): + w.writeErr(err) + return + } + } + }() + + return nil +} diff --git a/auditbeat/module/file_integrity/kprobes/monitor_test.go b/auditbeat/module/file_integrity/kprobes/monitor_test.go new file mode 100644 index 00000000000..da1021d5bbf --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/monitor_test.go @@ -0,0 +1,681 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "errors" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/elastic/beats/v7/auditbeat/tracing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/sys/unix" +) + +type monitorTestSuite struct { + suite.Suite +} + +func Test_Monitor(t *testing.T) { + suite.Run(t, new(monitorTestSuite)) +} + +func (p *monitorTestSuite) TestDoubleClose() { + ctx := context.Background() + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Close").Return(nil) + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + err = m.Close() + p.Require().NoError(err) + err = m.Close() + p.Require().NoError(err) +} + +func (p *monitorTestSuite) TestPerfChannelClose() { + ctx := context.Background() + mockPerfChannel := &perfChannelMock{} + closeErr := errors.New("error closing perf channel") + mockPerfChannel.On("Close").Return(closeErr) + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + err = m.Close() + p.Require().ErrorIs(err, closeErr) +} + +func (p *monitorTestSuite) TestPerfChannelRunErr() { + ctx := context.Background() + mockPerfChannel := &perfChannelMock{} + runErr := errors.New("perf channel run err") + mockPerfChannel.On("Run").Return(runErr) + mockPerfChannel.On("Close").Return(nil) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + + err = m.Start() + p.Require().Error(err, runErr) + + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestRunPerfChannelLost() { + ctx := context.Background() + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + + err = m.Start() + p.Require().NoError(err) + + select { + case perfLost <- 10: + case <-time.After(5 * time.Second): + p.Fail("timeout at writing perf lost") + } + + select { + case err = <-m.ErrorChannel(): + p.Require().Error(err) + case <-time.After(5 * time.Second): + p.Fail("no error received") + } + + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestRunPerfChannelErr() { + ctx := context.Background() + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + + err = m.Start() + p.Require().NoError(err) + + runErr := errors.New("perf channel run err") + select { + case perfErr <- runErr: + case <-time.After(5 * time.Second): + p.Fail("timeout at writing perf err") + } + + select { + case err = <-m.ErrorChannel(): + p.Require().ErrorIs(err, runErr) + case <-time.After(5 * time.Second): + p.Fail("no error received") + } + + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestRunPathErr() { + ctx := context.Background() + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + + err = m.Start() + p.Require().NoError(err) + + runErr := errors.New("path channel run err") + select { + case m.pathMonitor.errC <- runErr: + case <-time.After(5 * time.Second): + p.Fail("timeout at writing path err") + } + + select { + case err = <-m.ErrorChannel(): + p.Require().ErrorIs(err, runErr) + case <-time.After(5 * time.Second): + p.Fail("no error received") + } + + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestRunUnknownEventType() { + ctx := context.Background() + + type Unknown struct{} + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + + err = m.Start() + p.Require().NoError(err) + + select { + case perfEvent <- &Unknown{}: + case <-time.After(5 * time.Second): + p.Fail("timeout at writing perf event") + } + + select { + case err = <-m.ErrorChannel(): + p.Require().Error(err) + case <-time.After(5 * time.Second): + p.Fail("no error received") + } + + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestRunPerfCloseEventChan() { + ctx := context.Background() + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + + err = m.Start() + p.Require().NoError(err) + + close(perfEvent) + + select { + case err = <-m.ErrorChannel(): + p.Require().Error(err) + case <-time.After(5 * time.Second): + p.Fail("no error received") + } + + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestDoubleStart() { + ctx := context.Background() + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + err = m.Start() + p.Require().NoError(err) + err = m.Start() + p.Require().Error(err) + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestAddPathNotStarted() { + ctx := context.Background() + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Close").Return(nil) + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + err = m.Add("not-exist") + p.Require().Error(err) + + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestAddPathNotClosed() { + ctx := context.Background() + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + err = m.Start() + p.Require().NoError(err) + + p.Require().NoError(m.Close()) + + p.Require().Error(m.Add("not-exist")) +} + +func (p *monitorTestSuite) TestRunNoError() { + ctx := context.Background() + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + m.eProc.d.Add(&dEntry{ + Parent: nil, + Depth: 0, + Children: nil, + Name: "/test/test", + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, nil) + + err = m.Start() + p.Require().NoError(err) + + probeEvent := &ProbeEvent{ + Meta: tracing.Metadata{ + TID: 1, + PID: 1, + }, + MaskModify: 1, + FileIno: 1, + FileDevMajor: 1, + FileDevMinor: 1, + FileName: "test", + } + + select { + case perfEvent <- probeEvent: + case <-time.After(5 * time.Second): + p.Fail("timeout on writing event to perf channel") + } + + select { + case emittedEvent := <-m.EventChannel(): + p.Require().Equal(uint32(unix.IN_MODIFY), emittedEvent.Op) + p.Require().Equal("/test/test", emittedEvent.Path) + p.Require().Equal(uint32(1), emittedEvent.PID) + case <-time.After(5 * time.Second): + p.Fail("timeout on waiting event from monitor") + } + + p.Require().NoError(m.Close()) +} + +type emitterMock struct { + mock.Mock +} + +func (e *emitterMock) Emit(ePath string, pid uint32, op uint32) error { + args := e.Called(ePath, pid, op) + return args.Error(0) +} + +func (p *monitorTestSuite) TestRunEmitError() { + ctx := context.Background() + + perfLost := make(chan uint64) + perfEvent := make(chan interface{}) + perfErr := make(chan error) + + mockPerfChannel := &perfChannelMock{} + mockPerfChannel.On("Run").Return(nil) + mockPerfChannel.On("Close").Return(nil) + mockPerfChannel.On("C").Return(perfEvent) + mockPerfChannel.On("ErrC").Return(perfErr) + mockPerfChannel.On("LostC").Return(perfLost) + + emitErr := errors.New("emit error") + mockEmitter := &emitterMock{} + mockEmitter.On("Emit", mock.Anything, mock.Anything, mock.Anything).Return(emitErr) + + exec := newFixedThreadExecutor(ctx) + m, err := newMonitor(ctx, true, mockPerfChannel, exec) + p.Require().NoError(err) + + m.eProc.e = mockEmitter + m.eProc.d.Add(&dEntry{ + Parent: nil, + Depth: 0, + Children: nil, + Name: "/test/test", + Ino: 1, + DevMajor: 1, + DevMinor: 1, + }, nil) + + err = m.Start() + p.Require().NoError(err) + + probeEvent := &ProbeEvent{ + Meta: tracing.Metadata{ + TID: 1, + PID: 1, + }, + MaskModify: 1, + FileIno: 1, + FileDevMajor: 1, + FileDevMinor: 1, + FileName: "test", + } + + select { + case perfEvent <- probeEvent: + case <-time.After(5 * time.Second): + p.Fail("timeout on writing event to perf channel") + } + + select { + case err = <-m.ErrorChannel(): + p.Require().ErrorIs(err, emitErr) + case <-time.After(5 * time.Second): + p.Fail("timeout on waiting err from monitor") + } + + p.Require().NoError(m.Close()) +} + +func (p *monitorTestSuite) TestNew() { + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + p.T().Skip("skipping on non-amd64/arm64") + return + } + + if os.Getuid() != 0 { + p.T().Skip("skipping as non-root") + return + } + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + m, err := New(true) + p.Require().NoError(err) + + tmpDir, err := os.MkdirTemp("", "kprobe_bench_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + errChan := make(chan error) + cancelChan := make(chan struct{}) + + targetFile := filepath.Join(tmpDir, "file_kprobes.txt") + tid := uint32(unix.Gettid()) + + expectedEvents := []MonitorEvent{ + { + Op: uint32(unix.IN_CREATE), + Path: targetFile, + PID: tid, + }, + { + Op: uint32(unix.IN_MODIFY), + Path: targetFile, + PID: tid, + }, + { + Op: uint32(unix.IN_ATTRIB), + Path: targetFile, + PID: tid, + }, + { + Op: uint32(unix.IN_MODIFY), + Path: targetFile, + PID: tid, + }, + { + Op: uint32(unix.IN_MODIFY), + Path: targetFile, + PID: tid, + }, + { + Op: uint32(unix.IN_MODIFY), + Path: targetFile, + PID: tid, + }, + } + + var seenEvents []MonitorEvent + go func() { + defer close(errChan) + for { + select { + case mErr := <-m.ErrorChannel(): + select { + case errChan <- mErr: + case <-cancelChan: + return + } + case e, ok := <-m.EventChannel(): + if !ok { + select { + case errChan <- errors.New("closed event channel"): + case <-cancelChan: + return + } + } + seenEvents = append(seenEvents, e) + continue + case <-cancelChan: + return + } + } + }() + + p.Require().NoError(m.Start()) + p.Require().NoError(m.Add(tmpDir)) + + p.Require().NoError(os.WriteFile(targetFile, []byte("hello world!"), 0o644)) + p.Require().NoError(os.Chmod(targetFile, 0o777)) + p.Require().NoError(os.WriteFile(targetFile, []byte("data"), 0o644)) + p.Require().NoError(os.Truncate(targetFile, 0)) + + time.Sleep(5 * time.Second) + close(cancelChan) + err = <-errChan + if err != nil { + p.Require().Fail(err.Error()) + } + + p.Require().Equal(expectedEvents, seenEvents) +} + +const kernelURL string = "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.6.7.tar.xz" + +func downloadKernel(filepath string) error { + // Create the file + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + // Get the data + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, kernelURL, nil) + if err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + return err +} + +func BenchmarkMonitor(b *testing.B) { + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + b.Skip("skipping on non-amd64/arm64") + return + } + + if os.Getuid() != 0 { + b.Skip("skipping as non-root") + return + } + + tmpDir, err := os.MkdirTemp("", "kprobe_bench_test") + require.NoError(b, err) + defer os.RemoveAll(tmpDir) + + tarFilePath := filepath.Join(tmpDir, "linux-6.6.7.tar.xz") + + m, err := New(true) + require.NoError(b, err) + + errChan := make(chan error) + cancelChan := make(chan struct{}) + + seenEvents := uint64(0) + go func() { + defer close(errChan) + for { + select { + case mErr := <-m.ErrorChannel(): + select { + case errChan <- mErr: + case <-cancelChan: + return + } + case <-m.EventChannel(): + seenEvents += 1 + continue + case <-cancelChan: + return + } + } + }() + + require.NoError(b, m.Start()) + require.NoError(b, m.Add(tmpDir)) + + err = downloadKernel(tarFilePath) + + // decompress + require.NoError(b, err) + cmd := exec.Command("tar", "-xvf", "./linux-6.6.7.tar.xz") + cmd.Dir = tmpDir + err = cmd.Run() + require.NoError(b, err) + + // re-decompress; causes deletions of previous files + cmd = exec.Command("tar", "-xvf", "./linux-6.6.7.tar.xz") + cmd.Dir = tmpDir + err = cmd.Run() + require.NoError(b, err) + + time.Sleep(2 * time.Second) + close(cancelChan) + err = <-errChan + if err != nil { + require.Fail(b, err.Error()) + } + + require.NoError(b, m.Close()) + + // decompressing linux-6.6.7.tar.xz created 87082 files (includes created folder); measured with decompressing and + // running "find . | wc -l" + // so the dcache entry should contain 1 (tmpDir) + 1 (linux-6.6.7.tar.xz archive) + // + 87082 (folder + archive contents) dentries + require.Len(b, m.eProc.d.index, 87082+2) + + b.Logf("processed %d events", seenEvents) +} diff --git a/auditbeat/module/file_integrity/kprobes/path.go b/auditbeat/module/file_integrity/kprobes/path.go new file mode 100644 index 00000000000..982c44ec240 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/path.go @@ -0,0 +1,321 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "time" +) + +type MonitorPath struct { + fullPath string + depth uint32 + isFromMove bool + tid uint32 +} + +type pathTraverser interface { + AddPathToMonitor(ctx context.Context, path string) error + GetMonitorPath(ino uint64, major uint32, minor uint32, name string) (MonitorPath, bool) + WalkAsync(path string, depth uint32, tid uint32) + ErrC() <-chan error + Close() error +} + +type statMatch struct { + ino uint64 + major uint32 + minor uint32 + depth uint32 + fileName string + isFromMove bool + tid uint32 + fullPath string +} + +type pTraverser struct { + mtx sync.RWMutex + errC chan error + ctx context.Context + cancelFn context.CancelFunc + e executor + w inotifyWatcher + isRecursive bool + waitQueueChan chan struct{} + sMatchTimeout time.Duration + statQueue []statMatch +} + +var lstat = os.Lstat // for testing + +func newPathMonitor(ctx context.Context, exec executor, timeOut time.Duration, isRecursive bool) (*pTraverser, error) { + mWatcher, err := newInotifyWatcher() + if err != nil { + return nil, err + } + + if timeOut == 0 { + timeOut = 5 * time.Second + } + + mCtx, cancelFn := context.WithCancel(ctx) + + return &pTraverser{ + mtx: sync.RWMutex{}, + ctx: mCtx, + errC: make(chan error), + cancelFn: cancelFn, + e: exec, + w: mWatcher, + isRecursive: isRecursive, + sMatchTimeout: timeOut, + }, nil +} + +func (r *pTraverser) Close() error { + r.cancelFn() + return r.w.Close() +} + +func (r *pTraverser) GetMonitorPath(ino uint64, major uint32, minor uint32, name string) (MonitorPath, bool) { + if r.ctx.Err() != nil { + return MonitorPath{}, false + } + + r.mtx.Lock() + defer r.mtx.Unlock() + + if len(r.statQueue) == 0 { + return MonitorPath{}, false + } + + monitorPath := r.statQueue[0] + if monitorPath.ino != ino || + monitorPath.major != major || + monitorPath.minor != minor || + monitorPath.fileName != name { + return MonitorPath{}, false + } + + r.statQueue = r.statQueue[1:] + + if len(r.statQueue) == 0 && r.waitQueueChan != nil { + close(r.waitQueueChan) + r.waitQueueChan = nil + } + + return MonitorPath{ + fullPath: monitorPath.fullPath, + depth: monitorPath.depth, + isFromMove: monitorPath.isFromMove, + tid: monitorPath.tid, + }, true +} + +func readDirNames(dirName string) ([]string, error) { + f, err := os.Open(dirName) + if err != nil { + return nil, err + } + names, err := f.Readdirnames(-1) + _ = f.Close() + if err != nil { + return nil, err + } + sort.Strings(names) + return names, nil +} + +func (r *pTraverser) ErrC() <-chan error { + return r.errC +} + +func (r *pTraverser) WalkAsync(path string, depth uint32, tid uint32) { + if r.ctx.Err() != nil { + return + } + + go func() { + walkErr := r.e.Run(func() error { + return r.walk(r.ctx, path, depth, true, tid) + }) + + if walkErr == nil { + return + } + + select { + case r.errC <- walkErr: + case <-r.ctx.Done(): + } + }() +} + +func (r *pTraverser) walkRecursive(ctx context.Context, path string, mounts mountPoints, depth uint32, isFromMove bool, tid uint32) error { + if ctx.Err() != nil { + return ctx.Err() + } + + if r.ctx.Err() != nil { + return r.ctx.Err() + } + + if !r.isRecursive && depth > 1 { + return nil + } + + // get the mountpoint associated to this path + mnt := mounts.getMountByPath(path) + if mnt == nil { + return fmt.Errorf("could not find mount for %s", path) + } + + // add the inotify watcher if it does not exist + if _, err := r.w.Add(mnt.DeviceMajor, mnt.DeviceMinor, path); err != nil { + return err + } + + r.mtx.Lock() + info, err := lstat(path) + if err != nil { + // maybe this path got deleted/moved in the meantime + // return nil + r.mtx.Unlock() + //lint:ignore nilerr no errors returned for lstat from walkRecursive + return nil + } + + // if we are about to stat the root of the mountpoint, and the subtree has a different base + // from the base of the path (e.g. /watch [path] -> /etc/test [subtree]) + // the filename reported in the kprobe event will be "test" instead of "watch". Thus, we need to + // construct the filename based on the base name of the subtree. + mntPath := strings.Replace(path, mnt.Path, "", 1) + if !strings.HasPrefix(mntPath, mnt.Subtree) { + mntPath = filepath.Join(mnt.Subtree, mntPath) + } + + matchFileName := filepath.Base(mntPath) + + r.statQueue = append(r.statQueue, statMatch{ + ino: info.Sys().(*syscall.Stat_t).Ino, + major: mnt.DeviceMajor, + minor: mnt.DeviceMinor, + depth: depth, + fileName: matchFileName, + isFromMove: isFromMove, + tid: tid, + fullPath: path, + }) + r.mtx.Unlock() + + if !info.IsDir() { + return nil + } + + names, err := readDirNames(path) + if err != nil { + // maybe this dir got deleted/moved in the meantime + // return nil + //lint:ignore nilerr no errors returned for readDirNames from walkRecursive + return nil + } + + for _, name := range names { + filename := filepath.Join(path, name) + if err = r.walkRecursive(ctx, filename, mounts, depth+1, isFromMove, tid); err != nil { + //lint:ignore nilerr no errors returned for readDirNames from walkRecursive + return nil + } + } + return nil +} + +func (r *pTraverser) waitForWalk(ctx context.Context) error { + r.mtx.Lock() + + // statQueue is already empty, return + if len(r.statQueue) == 0 { + r.mtx.Unlock() + return nil + } + + r.waitQueueChan = make(chan struct{}) + r.mtx.Unlock() + + select { + // ctx of pTraverser is done + case <-r.ctx.Done(): + return r.ctx.Err() + // ctx of walk is done + case <-ctx.Done(): + return ctx.Err() + // statQueue is empty + case <-r.waitQueueChan: + return nil + // timeout + case <-time.After(r.sMatchTimeout): + return ErrAckTimeout + } +} + +func (r *pTraverser) walk(ctx context.Context, path string, depth uint32, isFromMove bool, tid uint32) error { + // get a snapshot of all mountpoints + mounts, err := getAllMountPoints() + if err != nil { + return err + } + + // start walking the given path + if err := r.walkRecursive(ctx, path, mounts, depth, isFromMove, tid); err != nil { + return err + } + + // wait for the monitor queue to get empty + return r.waitForWalk(ctx) +} + +func (r *pTraverser) AddPathToMonitor(ctx context.Context, path string) error { + if r.ctx.Err() != nil { + return r.ctx.Err() + } + + if ctx.Err() != nil { + return ctx.Err() + } + + // we care about the existence of the path only in AddPathToMonitor + // walk masks out all file existence errors + _, err := lstat(path) + if err != nil { + return err + } + + // paths from AddPathToMonitor are always starting with a depth of 0 + return r.e.Run(func() error { + return r.walk(ctx, path, 0, false, 0) + }) +} diff --git a/auditbeat/module/file_integrity/kprobes/path_inotify.go b/auditbeat/module/file_integrity/kprobes/path_inotify.go new file mode 100644 index 00000000000..05d483837f4 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/path_inotify.go @@ -0,0 +1,133 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "sync" + + "golang.org/x/sys/unix" +) + +type mountID struct { + major uint32 + minor uint32 +} + +type inotifyWatcher interface { + Add(devMajor uint32, devMinor uint32, mountPath string) (bool, error) + Close() error +} + +type iWatcher struct { + inotifyFD int + mounts map[mountID]struct{} + uniqueFDs map[uint32]struct{} + closed bool + mtx sync.Mutex +} + +var inotifyAddWatch = unix.InotifyAddWatch + +// newInotifyWatcher creates a new inotifyWatcher object and initializes the inotify file descriptor. +// +// It returns a pointer to the newly created inotifyWatcher object and an error if there was any. +// +// Note: Having such a inotifyWatcher is necessary for Linux kernels v5.15+ (commit +// https://lore.kernel.org/all/20210810151220.285179-5-amir73il@gmail.com/). Essentially this commit adds +// a proactive check in the inline fsnotify helpers to avoid calling fsnotify() and __fsnotify_parent() (our +// kprobes) in case there are no marks of any type (inode/sb/mount) for an inode's super block. To bypass this check, +// and always make sure that our kprobes are triggered, we use the inotifyWatcher to add an inotify watch on the +// mountpoints that we are interested in (inotify IN_MOUNT doesn't interfere with our probes). Also, it keeps track +// of the mountpoints (referenced by devmajor and devminor) that have already had an inotify watch added and does not +// add them again. +func newInotifyWatcher() (*iWatcher, error) { + fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) + if fd == -1 { + return nil, errno + } + + return &iWatcher{ + inotifyFD: fd, + mounts: make(map[mountID]struct{}), + uniqueFDs: make(map[uint32]struct{}), + }, nil +} + +// Add adds a mount to the inotifyWatcher. +// +// It takes in the device major number, device minor number, and mount as parameters. +// It returns false if the mount with the same device major number and minor number already +// has an inotify watch added. Also, it returns an error if there was any error. +func (w *iWatcher) Add(devMajor uint32, devMinor uint32, mountPath string) (bool, error) { + w.mtx.Lock() + defer w.mtx.Unlock() + + if w.closed { + return false, errors.New("inotify watcher already closed") + } + + id := mountID{ + major: devMajor, + minor: devMinor, + } + + if _, exists := w.mounts[id]; exists { + return false, nil + } + + wd, err := inotifyAddWatch(w.inotifyFD, mountPath, unix.IN_UNMOUNT) + if err != nil { + return false, err + } + + _, fdExists := w.uniqueFDs[uint32(wd)] + if fdExists { + return false, nil + } + + w.uniqueFDs[uint32(wd)] = struct{}{} + w.mounts[id] = struct{}{} + return true, nil +} + +// Close closes the inotifyWatcher and releases any associated resources. +// +// It removes all inotify watches added. If any error occurs +// during the removal of watches, it will be accumulated and returned as a single +// error value. After removing all watches, it closes the inotify file descriptor. +func (w *iWatcher) Close() error { + w.mtx.Lock() + defer w.mtx.Unlock() + + var allErr error + for fd := range w.uniqueFDs { + if _, err := unix.InotifyRmWatch(w.inotifyFD, fd); err != nil { + allErr = errors.Join(allErr, err) + } + } + w.uniqueFDs = nil + + allErr = errors.Join(allErr, unix.Close(w.inotifyFD)) + + w.mounts = nil + + return allErr +} diff --git a/auditbeat/module/file_integrity/kprobes/path_inotify_test.go b/auditbeat/module/file_integrity/kprobes/path_inotify_test.go new file mode 100644 index 00000000000..2cc0f37e51a --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/path_inotify_test.go @@ -0,0 +1,98 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func Test_InotifyWatcher(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + watcher, err := newInotifyWatcher() + require.NoError(t, err) + + added, err := watcher.Add(1, 1, tmpDir) + require.NoError(t, err) + require.True(t, added) + + added, err = watcher.Add(1, 1, filepath.Join(tmpDir, "test")) + require.NoError(t, err) + require.False(t, added) + + added, err = watcher.Add(2, 2, tmpDir) + require.NoError(t, err) + require.False(t, added) + + tmpDir2, err := os.MkdirTemp("", "kprobe_unit_test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir2) + + added, err = watcher.Add(2, 2, tmpDir2) + require.NoError(t, err) + require.True(t, added) + + require.NoError(t, watcher.Close()) + + _, err = watcher.Add(1, 1, tmpDir) + require.Error(t, err) +} + +func Test_InotifyWatcher_Add_Err(t *testing.T) { + watcher, err := newInotifyWatcher() + require.NoError(t, err) + + inotifyAddWatch = func(fd int, pathname string, mask uint32) (int, error) { + return -1, os.ErrInvalid + } + defer func() { + inotifyAddWatch = unix.InotifyAddWatch + }() + + _, err = watcher.Add(1, 1, "non_existent") + require.ErrorIs(t, err, os.ErrInvalid) + + require.NoError(t, watcher.Close()) +} + +func Test_InotifyWatcher_Close_Err(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + watcher, err := newInotifyWatcher() + require.NoError(t, err) + + added, err := watcher.Add(1, 1, tmpDir) + require.NoError(t, err) + require.True(t, added) + + err = os.RemoveAll(tmpDir) + require.NoError(t, err) + + require.Error(t, watcher.Close()) +} diff --git a/auditbeat/module/file_integrity/kprobes/path_mountpoint.go b/auditbeat/module/file_integrity/kprobes/path_mountpoint.go new file mode 100644 index 00000000000..9b4f66309f0 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/path_mountpoint.go @@ -0,0 +1,234 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "bufio" + "fmt" + "io" + "os" + "sort" + "strconv" + "strings" + "sync" +) + +// Used to make the mount functions thread safe +var mountMutex sync.Mutex + +// mount contains information for a specific mounted filesystem. +// +// Path - Absolute path where the directory is mounted +// FilesystemType - Type of the mounted filesystem, e.g. "ext4" +// Device - Device for filesystem (empty string if we cannot find one) +// DeviceMajor - Device major number of the filesystem. This is set even if +// Device isn't, since all filesystems have a device +// number assigned by the kernel, even pseudo-filesystems. +// DeviceMinor - Device minor number of the filesystem. This is set even if +// Device isn't, since all filesystems have a device +// number assigned by the kernel, even pseudo-filesystems. +// Subtree - The mounted subtree of the filesystem. This is usually +// "/", meaning that the entire filesystem is mounted, but +// it can differ for bind mounts. +// ReadOnly - True if this is a read-only mount +type mount struct { + Path string + FilesystemType string + DeviceMajor uint32 + DeviceMinor uint32 + Subtree string + ReadOnly bool +} + +// mountPoints allows mounts to be sorted by Path length. +type mountPoints []*mount + +func (p mountPoints) Len() int { return len(p) } +func (p mountPoints) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p mountPoints) Less(i, j int) bool { + if len(p[i].Path) == len(p[j].Path) { + return p[i].Path > p[j].Path + } + + return len(p[i].Path) > len(p[j].Path) +} + +// getMountByPath returns the mount point that matches the given path. +// +// The path parameter specifies the path to search for a matching mount point. +// It should not be empty. +// +// The function returns a pointer to a mount struct if a matching mount point is found, +// otherwise it returns nil. +func (p mountPoints) getMountByPath(path string) *mount { + if path == "" { + return nil + } + + // Remove trailing slash if it not root / + if len(path) > 1 && path[len(path)-1] == '/' { + path = path[:len(path)-1] + } + + for _, mount := range p { + mountPath := mount.Path + if strings.HasPrefix(path, mountPath) { + return mount + } + } + + return nil +} + +// Unescape octal-encoded escape sequences in a string from the mountinfo file. +// The kernel encodes the ' ', '\t', '\n', and '\\' bytes this way. This +// function exactly inverts what the kernel does, including by preserving +// invalid UTF-8. +func unescapeString(str string) string { + var sb strings.Builder + for i := 0; i < len(str); i++ { + b := str[i] + if b == '\\' && i+3 < len(str) { + if parsed, err := strconv.ParseInt(str[i+1:i+4], 8, 8); err == nil { + b = uint8(parsed) + i += 3 + } + } + sb.WriteByte(b) + } + return sb.String() +} + +// Parse one line of /proc/self/mountinfo. +// +// The line contains the following space-separated fields: +// +// [0] mount ID +// [1] parent ID +// [2] major:minor +// [3] root +// [4] mount point +// [5] mount options +// [6...n-1] optional field(s) +// [n] separator +// [n+1] filesystem type +// [n+2] mount source +// [n+3] super options +// +// For more details, see https://www.kernel.org/doc/Documentation/filesystems/proc.txt +func parseMountInfoLine(line string) (*mount, error) { + fields := strings.Split(line, " ") + if len(fields) < 10 { + return nil, nil + } + + // Count the optional fields. In case new fields are appended later, + // don't simply assume that n == len(fields) - 4. + n := 6 + for fields[n] != "-" { + n++ + if n >= len(fields) { + return nil, nil + } + } + if n+3 >= len(fields) { + return nil, nil + } + + mnt := &mount{} + var err error + mnt.DeviceMajor, mnt.DeviceMinor, err = newDeviceMajorMinorFromString(fields[2]) + if err != nil { + return nil, err + } + mnt.Subtree = unescapeString(fields[3]) + mnt.Path = unescapeString(fields[4]) + for _, opt := range strings.Split(fields[5], ",") { + if opt == "ro" { + mnt.ReadOnly = true + } + } + mnt.FilesystemType = unescapeString(fields[n+1]) + return mnt, nil +} + +// readMountInfo reads mount information from the given input reader and returns +// a list of mount points and an error. Each mount point is represented by a mount +// struct containing information about the mount. +func readMountInfo(r io.Reader) (mountPoints, error) { + seenMountsByPath := make(map[string]*mount) + var mPoints mountPoints //nolint:prealloc //can't be preallocated as the number of lines is unknown before scan + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + mnt, err := parseMountInfoLine(line) + if err != nil { + return nil, err + } + + if mnt == nil { + continue + } + + _, exists := seenMountsByPath[mnt.Path] + if exists { + // duplicate mountpoint entries have been observed for + // /proc/sys/fs/binfmt_misc + continue + } + + mPoints = append(mPoints, mnt) + // Note this overrides the info if we have seen the mountpoint + // earlier in the file. This is correct behavior because the + // mountpoints are listed in mount order. + seenMountsByPath[mnt.Path] = mnt + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + sort.Sort(mPoints) + + return mPoints, nil +} + +// getAllMountPoints populates the mount mappings by parsing /proc/self/mountinfo. +func getAllMountPoints() (mountPoints, error) { + mountMutex.Lock() + defer mountMutex.Unlock() + + file, err := os.Open("/proc/self/mountinfo") + if err != nil { + return nil, err + } + defer file.Close() + return readMountInfo(file) +} + +// newDeviceMajorMinorFromString generates a new device major and minor numbers from a given string. +func newDeviceMajorMinorFromString(str string) (uint32, uint32, error) { + var major, minor uint32 + if count, _ := fmt.Sscanf(str, "%d:%d", &major, &minor); count != 2 { + return 0, 0, fmt.Errorf("invalid device number string %q", str) + } + return major, minor, nil +} diff --git a/auditbeat/module/file_integrity/kprobes/path_mountpoint_test.go b/auditbeat/module/file_integrity/kprobes/path_mountpoint_test.go new file mode 100644 index 00000000000..99389d07576 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/path_mountpoint_test.go @@ -0,0 +1,108 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_readMountInfo(t *testing.T) { + procContents := `19 42 0:19 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw +42 0 252:1 / /etc/test/test rw,noatime shared:1 - xfs /dev/vda1 rw,attr2,inode64,noquota +20 42 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw +23 21 0:20 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw +25 42 0:22 / /run rw,nosuid,nodev shared:23 - tmpfs tmpfs rw,mode=755 +26 19 0:23 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:8 - tmpfs tmpfs ro,mode=755 +42 0 252:1 / / rw,noatime shared:1 - xfs /dev/vda1 rw,attr2,inode64,noquota +45 19 0:8 / /sys/kernel/debug rw,relatime shared:26 - debugfs debugfs rw +46 20 0:39 / /proc/sys/fs/binfmt_misc rw,relatime shared:27 - autofs systemd-1 rw,fd=34,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13706 +47 42 259:0 / /boot/efi rw,noatime shared:28 - vfat /dev/vda128 rw,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro +42 0 252:1 / /etc/test rw,noatime shared:1 - xfs /dev/vda1 rw,attr2,inode64,noquota +` + + sortedPaths := []string{ + "/proc/sys/fs/binfmt_misc", + "/sys/kernel/debug", + "/sys/fs/cgroup", + "/etc/test/test", + "/etc/test", + "/boot/efi", + "/dev/shm", + "/proc", + "/sys", + "/run", + "/", + } + + reader := strings.NewReader(procContents) + + mounts, err := readMountInfo(reader) + require.NoError(t, err) + require.Len(t, mounts, 11) + + for i, path := range sortedPaths { + require.Equal(t, path, mounts[i].Path) + } + + require.Equal(t, mounts[10], &mount{ + Path: "/", + FilesystemType: "xfs", + DeviceMajor: 252, + DeviceMinor: 1, + Subtree: "/", + ReadOnly: false, + }) + + require.Equal(t, mounts[2], &mount{ + Path: "/sys/fs/cgroup", + FilesystemType: "tmpfs", + DeviceMajor: 0, + DeviceMinor: 23, + Subtree: "/", + ReadOnly: true, + }) + + require.Equal(t, mounts[0], &mount{ + Path: "/proc/sys/fs/binfmt_misc", + FilesystemType: "autofs", + DeviceMajor: 0, + DeviceMinor: 39, + Subtree: "/", + ReadOnly: false, + }) + + pathMountPoint := mounts.getMountByPath("/etc/test/") + + require.Equal(t, pathMountPoint, &mount{ + Path: "/etc/test", + FilesystemType: "xfs", + DeviceMajor: 252, + DeviceMinor: 1, + Subtree: "/", + ReadOnly: false, + }) + + pathMountPoint = mounts.getMountByPath("unknown") + + require.Nil(t, pathMountPoint) +} diff --git a/auditbeat/module/file_integrity/kprobes/path_test.go b/auditbeat/module/file_integrity/kprobes/path_test.go new file mode 100644 index 00000000000..1689bf5a633 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/path_test.go @@ -0,0 +1,685 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "os" + "path/filepath" + "sync" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func Test_PathTraverser_newPathMonitor(t *testing.T) { + ctx := context.Background() + + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + require.NoError(t, err) + require.Equal(t, pTrav.sMatchTimeout, 5*time.Second) + require.NoError(t, pTrav.Close()) + + pTrav, err = newPathMonitor(ctx, newFixedThreadExecutor(ctx), 2*time.Second, true) + require.NoError(t, err) + require.Equal(t, pTrav.sMatchTimeout, 2*time.Second) + require.NoError(t, pTrav.Close()) +} + +type pathTestSuite struct { + suite.Suite +} + +func Test_PathTraverser(t *testing.T) { + suite.Run(t, new(pathTestSuite)) +} + +func (p *pathTestSuite) TestContextCancelBeforeAdd() { + // cancelled parent context + ctx, cancelFn := context.WithCancel(context.Background()) + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + p.Require().NoError(err) + cancelFn() + err = pTrav.AddPathToMonitor(ctx, "not-existing-path") + p.Require().ErrorIs(err, ctx.Err()) + p.Require().NoError(pTrav.Close()) + + // cancelled traverser context + ctx, cancelFn = context.WithCancel(context.Background()) + pTrav, err = newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + p.Require().NoError(err) + pTrav.cancelFn() + err = pTrav.AddPathToMonitor(ctx, "not-existing-path") + p.Require().ErrorIs(err, pTrav.ctx.Err()) + p.Require().NoError(pTrav.Close()) + cancelFn() +} + +func (p *pathTestSuite) TestAddParentContextDone() { + ctx, cancelFn := context.WithCancel(context.Background()) + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + p.Require().NoError(err) + cancelFn() + err = pTrav.AddPathToMonitor(ctx, "not-existing-path") + p.Require().ErrorIs(err, ctx.Err()) + p.Require().NoError(pTrav.Close()) +} + +func (p *pathTestSuite) TestRecursiveWalkAsync() { + var createdPathsOrder []string + createdPathsWithDepth := make(map[string]uint32) + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + createdPathsWithDepth[tmpDir] = 1 + createdPathsOrder = append(createdPathsOrder, tmpDir) + + testDir := filepath.Join(tmpDir, "test_dir") + err = os.Mkdir(testDir, 0o744) + p.Require().NoError(err) + createdPathsWithDepth[testDir] = 2 + createdPathsOrder = append(createdPathsOrder, testDir) + + testDirTestFile := filepath.Join(tmpDir, "test_dir", "test_file") + f, err := os.Create(testDirTestFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + createdPathsWithDepth[testDirTestFile] = 3 + createdPathsOrder = append(createdPathsOrder, testDirTestFile) + + testFile := filepath.Join(tmpDir, "test_file") + f, err = os.Create(testFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + createdPathsWithDepth[testFile] = 2 + createdPathsOrder = append(createdPathsOrder, testFile) + + mounts, err := getAllMountPoints() + p.Require().NoError(err) + + p.Require().Equal(len(createdPathsOrder), len(createdPathsWithDepth)) + + expectedStatQueue := make([]statMatch, 0, len(createdPathsOrder)) + for _, path := range createdPathsOrder { + + depth, exists := createdPathsWithDepth[path] + p.Require().True(exists) + + info, err := os.Lstat(path) + p.Require().NoError(err) + mnt := mounts.getMountByPath(path) + p.Require().NotNil(mnt) + expectedStatQueue = append(expectedStatQueue, statMatch{ + ino: info.Sys().(*syscall.Stat_t).Ino, + major: mnt.DeviceMajor, + minor: mnt.DeviceMinor, + depth: depth, + fileName: info.Name(), + isFromMove: true, + tid: 2, + fullPath: path, + }) + } + + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + p.Require().NoError(err) + defer func() { + p.Require().NoError(pTrav.Close()) + }() + + pTrav.WalkAsync(tmpDir, 1, 2) + + tries := 0 + for idx := 0; idx < len(expectedStatQueue); { + mPath, match := pTrav.GetMonitorPath( + expectedStatQueue[idx].ino, + expectedStatQueue[idx].major, + expectedStatQueue[idx].minor, + expectedStatQueue[idx].fileName, + ) + + if match { + p.Require().Equal(expectedStatQueue[idx].fullPath, mPath.fullPath) + p.Require().Equal(expectedStatQueue[idx].isFromMove, mPath.isFromMove) + p.Require().Equal(expectedStatQueue[idx].tid, mPath.tid) + p.Require().Equal(expectedStatQueue[idx].depth, mPath.depth) + + tries = 0 + idx++ + continue + } + + if tries >= 3 { + p.Require().Fail("no match found") + return + } + + time.Sleep(300 * time.Millisecond) + tries++ + } + + select { + case err = <-pTrav.errC: + default: + } + + p.Require().NoError(err) + p.Require().Empty(pTrav.statQueue) +} + +func (p *pathTestSuite) TestWalkAsyncTimeoutErr() { + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + p.Require().NoError(err) + defer func() { + p.Require().NoError(pTrav.Close()) + }() + + pTrav.WalkAsync(tmpDir, 1, 2) + + select { + case err = <-pTrav.errC: + case <-time.After(10 * time.Second): + p.Require().Fail("no timeout error received") + } + + p.Require().ErrorIs(err, ErrAckTimeout) +} + +func (p *pathTestSuite) TestNonRecursiveWalkAsync() { + var createdPathsOrder []string + createdPathsWithDepth := make(map[string]uint32) + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + createdPathsWithDepth[tmpDir] = 1 + createdPathsOrder = append(createdPathsOrder, tmpDir) + + testDir := filepath.Join(tmpDir, "test_dir") + err = os.Mkdir(testDir, 0o744) + p.Require().NoError(err) + + testDirTestFile := filepath.Join(tmpDir, "test_dir", "test_file") + f, err := os.Create(testDirTestFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + + testFile := filepath.Join(tmpDir, "test_file") + f, err = os.Create(testFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + + mounts, err := getAllMountPoints() + p.Require().NoError(err) + + p.Require().Equal(len(createdPathsOrder), len(createdPathsWithDepth)) + + expectedStatQueue := make([]statMatch, 0, len(createdPathsOrder)) + for _, path := range createdPathsOrder { + + depth, exists := createdPathsWithDepth[path] + p.Require().True(exists) + + info, err := os.Lstat(path) + p.Require().NoError(err) + mnt := mounts.getMountByPath(path) + p.Require().NotNil(mnt) + expectedStatQueue = append(expectedStatQueue, statMatch{ + ino: info.Sys().(*syscall.Stat_t).Ino, + major: mnt.DeviceMajor, + minor: mnt.DeviceMinor, + depth: depth, + fileName: info.Name(), + isFromMove: true, + tid: 2, + fullPath: path, + }) + } + + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, false) + p.Require().NoError(err) + defer func() { + p.Require().NoError(pTrav.Close()) + }() + + pTrav.WalkAsync(tmpDir, 1, 2) + + tries := 0 + for idx := 0; idx < len(expectedStatQueue); { + mPath, match := pTrav.GetMonitorPath( + expectedStatQueue[idx].ino, + expectedStatQueue[idx].major, + expectedStatQueue[idx].minor, + expectedStatQueue[idx].fileName, + ) + + if match { + p.Require().Equal(expectedStatQueue[idx].fullPath, mPath.fullPath) + p.Require().Equal(expectedStatQueue[idx].isFromMove, mPath.isFromMove) + p.Require().Equal(expectedStatQueue[idx].tid, mPath.tid) + p.Require().Equal(expectedStatQueue[idx].depth, mPath.depth) + + tries = 0 + idx++ + continue + } + + if tries >= 3 { + p.Require().Fail("no match found") + return + } + + time.Sleep(300 * time.Millisecond) + tries++ + } + + select { + case err = <-pTrav.errC: + default: + } + + p.Require().NoError(err) + p.Require().Empty(pTrav.statQueue) +} + +func (p *pathTestSuite) TestAddTraverserContextCancel() { + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 10*time.Second, true) + p.Require().NoError(err) + defer func() { + p.Require().NoError(pTrav.Close()) + }() + + errChan := make(chan error) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + errPath := pTrav.AddPathToMonitor(ctx, tmpDir) + if errPath != nil { + errChan <- errPath + } + close(errChan) + }() + + tries := 0 + for { + if tries >= 4 { + p.Require().Fail("no path was added in 5 tries") + } + if len(pTrav.statQueue) == 0 { + tries++ + time.Sleep(1 * time.Second) + continue + } + break + } + pTrav.cancelFn() + + err = <-errChan + p.Require().ErrorIs(err, pTrav.ctx.Err()) +} + +func (p *pathTestSuite) TestAddTimeout() { + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 5*time.Second, true) + p.Require().NoError(err) + defer func() { + p.Require().NoError(pTrav.Close()) + }() + + errChan := make(chan error) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + errPath := pTrav.AddPathToMonitor(ctx, tmpDir) + if errPath != nil { + errChan <- errPath + } + close(errChan) + }() + + select { + case err = <-errChan: + case <-time.After(10 * time.Second): + p.Require().Fail("no path was added in 10 seconds") + } + p.Require().ErrorIs(err, ErrAckTimeout) +} + +func (p *pathTestSuite) TestRecursiveAdd() { + var createdPathsOrder []string + createdPathsWithDepth := make(map[string]uint32) + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + createdPathsWithDepth[tmpDir] = 0 + createdPathsOrder = append(createdPathsOrder, tmpDir) + + testDir := filepath.Join(tmpDir, "test_dir") + err = os.Mkdir(testDir, 0o744) + p.Require().NoError(err) + createdPathsWithDepth[testDir] = 1 + createdPathsOrder = append(createdPathsOrder, testDir) + + testDirTestFile := filepath.Join(tmpDir, "test_dir", "test_file") + f, err := os.Create(testDirTestFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + createdPathsWithDepth[testDirTestFile] = 2 + createdPathsOrder = append(createdPathsOrder, testDirTestFile) + + testFile := filepath.Join(tmpDir, "test_file") + f, err = os.Create(testFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + createdPathsWithDepth[testFile] = 1 + createdPathsOrder = append(createdPathsOrder, testFile) + + mounts, err := getAllMountPoints() + p.Require().NoError(err) + + p.Require().Equal(len(createdPathsOrder), len(createdPathsWithDepth)) + + expectedStatQueue := make([]statMatch, 0, len(createdPathsOrder)) + for _, path := range createdPathsOrder { + + depth, exists := createdPathsWithDepth[path] + p.Require().True(exists) + + info, err := os.Lstat(path) + p.Require().NoError(err) + mnt := mounts.getMountByPath(path) + p.Require().NotNil(mnt) + expectedStatQueue = append(expectedStatQueue, statMatch{ + ino: info.Sys().(*syscall.Stat_t).Ino, + major: mnt.DeviceMajor, + minor: mnt.DeviceMinor, + depth: depth, + fileName: info.Name(), + isFromMove: false, + tid: 0, + fullPath: path, + }) + } + + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + p.Require().NoError(err) + defer func() { + p.Require().NoError(pTrav.Close()) + }() + + errChan := make(chan error) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + errPath := pTrav.AddPathToMonitor(ctx, tmpDir) + if errPath != nil { + errChan <- errPath + } + close(errChan) + }() + + tries := 0 + for idx := 0; idx < len(expectedStatQueue); { + mPath, match := pTrav.GetMonitorPath( + expectedStatQueue[idx].ino, + expectedStatQueue[idx].major, + expectedStatQueue[idx].minor, + expectedStatQueue[idx].fileName, + ) + + if match { + p.Require().Equal(expectedStatQueue[idx].fullPath, mPath.fullPath) + p.Require().Equal(expectedStatQueue[idx].isFromMove, mPath.isFromMove) + p.Require().Equal(expectedStatQueue[idx].tid, mPath.tid) + p.Require().Equal(expectedStatQueue[idx].depth, mPath.depth) + + tries = 0 + idx++ + continue + } + + if tries >= 3 { + p.Require().Fail("no match found") + } + + time.Sleep(100 * time.Millisecond) + tries++ + } + + err = <-errChan + p.Require().NoError(err) + p.Require().Empty(pTrav.statQueue) +} + +func (p *pathTestSuite) TestNonRecursiveAdd() { + var createdPathsOrder []string + createdPathsWithDepth := make(map[string]uint32) + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + createdPathsWithDepth[tmpDir] = 0 + createdPathsOrder = append(createdPathsOrder, tmpDir) + + testDir := filepath.Join(tmpDir, "test_dir") + err = os.Mkdir(testDir, 0o744) + p.Require().NoError(err) + createdPathsWithDepth[testDir] = 1 + createdPathsOrder = append(createdPathsOrder, testDir) + + testDirTestFile := filepath.Join(tmpDir, "test_dir", "test_file") + f, err := os.Create(testDirTestFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + + testFile := filepath.Join(tmpDir, "test_file") + f, err = os.Create(testFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + createdPathsWithDepth[testFile] = 1 + createdPathsOrder = append(createdPathsOrder, testFile) + + mounts, err := getAllMountPoints() + p.Require().NoError(err) + + p.Require().Equal(len(createdPathsOrder), len(createdPathsWithDepth)) + + expectedStatQueue := make([]statMatch, 0, len(createdPathsOrder)) + for _, path := range createdPathsOrder { + + depth, exists := createdPathsWithDepth[path] + p.Require().True(exists) + + info, err := os.Lstat(path) + p.Require().NoError(err) + mnt := mounts.getMountByPath(path) + p.Require().NotNil(mnt) + expectedStatQueue = append(expectedStatQueue, statMatch{ + ino: info.Sys().(*syscall.Stat_t).Ino, + major: mnt.DeviceMajor, + minor: mnt.DeviceMinor, + depth: depth, + fileName: info.Name(), + isFromMove: false, + tid: 0, + fullPath: path, + }) + } + + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, false) + p.Require().NoError(err) + defer func() { + p.Require().NoError(pTrav.Close()) + }() + + errChan := make(chan error) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + errPath := pTrav.AddPathToMonitor(ctx, tmpDir) + if errPath != nil { + errChan <- errPath + } + close(errChan) + }() + + tries := 0 + for idx := 0; idx < len(expectedStatQueue); { + mPath, match := pTrav.GetMonitorPath( + expectedStatQueue[idx].ino, + expectedStatQueue[idx].major, + expectedStatQueue[idx].minor, + expectedStatQueue[idx].fileName, + ) + + if match { + p.Require().Equal(expectedStatQueue[idx].fullPath, mPath.fullPath) + p.Require().Equal(expectedStatQueue[idx].isFromMove, mPath.isFromMove) + p.Require().Equal(expectedStatQueue[idx].tid, mPath.tid) + p.Require().Equal(expectedStatQueue[idx].depth, mPath.depth) + + tries = 0 + idx++ + continue + } + + if tries >= 3 { + p.Require().Fail("no match found") + } + + time.Sleep(100 * time.Millisecond) + tries++ + } + + err = <-errChan + p.Require().NoError(err) + p.Require().Empty(pTrav.statQueue) +} + +func (p *pathTestSuite) TestStatErrAtRootAdd() { + defer func() { + lstat = os.Lstat + }() + // lstat error at root path to monitor + lstat = func(path string) (os.FileInfo, error) { + return nil, os.ErrNotExist + } + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + p.Require().NoError(err) + err = pTrav.AddPathToMonitor(ctx, "not-existing-path") + p.Require().ErrorIs(err, os.ErrNotExist) + p.Require().NoError(pTrav.Close()) +} + +func (p *pathTestSuite) TestStatErrAtWalk() { + defer func() { + lstat = os.Lstat + }() + + tmpDir, err := os.MkdirTemp("", "kprobe_unit_test") + p.Require().NoError(err) + defer os.RemoveAll(tmpDir) + + testDir := filepath.Join(tmpDir, "test_dir") + err = os.Mkdir(testDir, 0o744) + p.Require().NoError(err) + + testDirTestFile := filepath.Join(tmpDir, "test_dir", "test_file") + f, err := os.Create(testDirTestFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + + testFile := filepath.Join(tmpDir, "test_file") + f, err = os.Create(testFile) + p.Require().NoError(err) + p.Require().NoError(f.Close()) + + // lstat error at root path to monitor + lstat = func(path string) (os.FileInfo, error) { + info, err := os.Lstat(path) + lstat = func(name string) (os.FileInfo, error) { + return nil, os.ErrNotExist + } + + return info, err + } + ctx := context.Background() + pTrav, err := newPathMonitor(ctx, newFixedThreadExecutor(ctx), 0, true) + p.Require().NoError(err) + err = pTrav.AddPathToMonitor(ctx, tmpDir) + p.Require().NoError(err) + p.Require().NoError(pTrav.Close()) +} + +type pathTraverserMock struct { + mock.Mock +} + +func (p *pathTraverserMock) AddPathToMonitor(ctx context.Context, path string) error { + args := p.Called(ctx, path) + return args.Error(0) +} + +func (p *pathTraverserMock) GetMonitorPath(ino uint64, major uint32, minor uint32, name string) (MonitorPath, bool) { + args := p.Called(ino, major, minor, name) + return args.Get(0).(MonitorPath), args.Bool(1) +} + +func (p *pathTraverserMock) WalkAsync(path string, depth uint32, tid uint32) { + p.Called(path, depth, tid) +} + +func (p *pathTraverserMock) ErrC() <-chan error { + args := p.Called() + return args.Get(0).(<-chan error) +} + +func (p *pathTraverserMock) Close() error { + args := p.Called() + return args.Error(0) +} diff --git a/auditbeat/module/file_integrity/kprobes/perf_channel.go b/auditbeat/module/file_integrity/kprobes/perf_channel.go new file mode 100644 index 00000000000..a76fe3f2a63 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/perf_channel.go @@ -0,0 +1,77 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "time" + + "github.com/elastic/beats/v7/auditbeat/tracing" +) + +type perfChannel interface { + C() <-chan interface{} + ErrC() <-chan error + LostC() <-chan uint64 + Run() error + Close() error +} + +func newPerfChannel(probes map[tracing.Probe]tracing.AllocateFn, ringSizeExponent int, bufferSize int, pid int) (*tracing.PerfChannel, error) { + tfs, err := tracing.NewTraceFS() + if err != nil { + return nil, err + } + + pChannel, err := tracing.NewPerfChannel( + tracing.WithTimestamp(), + tracing.WithRingSizeExponent(ringSizeExponent), + tracing.WithBufferSize(bufferSize), + tracing.WithTID(pid), + tracing.WithPollTimeout(200*time.Millisecond), + tracing.WithWakeUpEvents(500), + ) + if err != nil { + return nil, err + } + + for probe, allocFn := range probes { + _ = tfs.RemoveKProbe(probe) + + err := tfs.AddKProbe(probe) + if err != nil { + return nil, err + } + desc, err := tfs.LoadProbeFormat(probe) + if err != nil { + return nil, err + } + + decoder, err := tracing.NewStructDecoder(desc, allocFn) + if err != nil { + return nil, err + } + + if err := pChannel.MonitorProbe(desc, decoder); err != nil { + return nil, err + } + } + + return pChannel, nil +} diff --git a/libbeat/publisher/queue/memqueue/batchbuf.go b/auditbeat/module/file_integrity/kprobes/perf_channel_test.go similarity index 55% rename from libbeat/publisher/queue/memqueue/batchbuf.go rename to auditbeat/module/file_integrity/kprobes/perf_channel_test.go index 87c3a1052f3..810bd59bcff 100644 --- a/libbeat/publisher/queue/memqueue/batchbuf.go +++ b/auditbeat/module/file_integrity/kprobes/perf_channel_test.go @@ -15,39 +15,37 @@ // specific language governing permissions and limitations // under the License. -package memqueue +//go:build linux -type batchBuffer struct { - next *batchBuffer - flushed bool - entries []queueEntry +package kprobes + +import "github.com/stretchr/testify/mock" + +type perfChannelMock struct { + mock.Mock +} + +func (p *perfChannelMock) C() <-chan interface{} { + args := p.Called() + return args.Get(0).(chan interface{}) } -func newBatchBuffer(sz int) *batchBuffer { - b := &batchBuffer{} - b.entries = make([]queueEntry, 0, sz) - return b +func (p *perfChannelMock) ErrC() <-chan error { + args := p.Called() + return args.Get(0).(chan error) } -func (b *batchBuffer) add(entry queueEntry) { - b.entries = append(b.entries, entry) +func (p *perfChannelMock) LostC() <-chan uint64 { + args := p.Called() + return args.Get(0).(chan uint64) } -func (b *batchBuffer) length() int { - return len(b.entries) +func (p *perfChannelMock) Run() error { + args := p.Called() + return args.Error(0) } -func (b *batchBuffer) cancel(producer *ackProducer) int { - entries := b.entries[:0] - - removedCount := 0 - for _, entry := range b.entries { - if entry.producer == producer { - removedCount++ - continue - } - entries = append(entries, entry) - } - b.entries = entries - return removedCount +func (p *perfChannelMock) Close() error { + args := p.Called() + return args.Error(0) } diff --git a/auditbeat/module/file_integrity/kprobes/probes.go b/auditbeat/module/file_integrity/kprobes/probes.go new file mode 100644 index 00000000000..836dff04cdf --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes.go @@ -0,0 +1,139 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "github.com/elastic/beats/v7/auditbeat/tracing" + + tkbtf "github.com/elastic/tk-btf" + + "golang.org/x/sys/unix" +) + +const ( + fsEventModify = uint32(unix.IN_MODIFY) + fsEventAttrib = uint32(unix.IN_ATTRIB) + fsEventMovedFrom = uint32(unix.IN_MOVED_FROM) + fsEventMovedTo = uint32(unix.IN_MOVED_TO) + fsEventCreate = uint32(unix.IN_CREATE) + fsEventDelete = uint32(unix.IN_DELETE) + fsEventIsDir = uint32(unix.IN_ISDIR) +) + +const ( + devMajor = uint32(0xFFF00000) + devMinor = uint32(0x3FF) +) + +type probeWithAllocFunc struct { + probe *tkbtf.Probe + allocateFn func() any +} + +type shouldBuildCheck func(spec *tkbtf.Spec) bool + +type symbol interface { + buildProbes(spec *tkbtf.Spec) ([]*probeWithAllocFunc, error) + + onErr(err error) bool +} + +type probeManager struct { + symbols []symbol + buildChecks []shouldBuildCheck + getSymbolInfoRuntime func(symbolName string) (runtimeSymbolInfo, error) +} + +func newProbeManager(e executor) (*probeManager, error) { + fs := &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: getSymbolInfoRuntime, + } + + if err := loadFsNotifySymbol(fs); err != nil { + return nil, err + } + + if err := loadFsNotifyParentSymbol(fs); err != nil { + return nil, err + } + + if err := loadFsNotifyNameRemoveSymbol(fs); err != nil { + return nil, err + } + + if err := loadVFSGetAttrSymbol(fs, e); err != nil { + return nil, err + } + + return fs, nil +} + +func (fs *probeManager) shouldBuild(spec *tkbtf.Spec) bool { + for _, check := range fs.buildChecks { + if !check(spec) { + return false + } + } + + return true +} + +func (fs *probeManager) build(spec *tkbtf.Spec) (map[tracing.Probe]tracing.AllocateFn, error) { + trProbesMap := make(map[tracing.Probe]tracing.AllocateFn) + + for _, s := range fs.symbols { + probesWithAlloc, err := s.buildProbes(spec) + if err != nil { + return nil, err + } + + for _, p := range probesWithAlloc { + trProbe := tracing.Probe{ + Group: "auditbeat_fim", + Name: p.probe.GetID(), + Address: p.probe.GetSymbolName(), + Fetchargs: p.probe.GetTracingEventProbe(), + Filter: p.probe.GetTracingEventFilter(), + } + switch p.probe.GetType() { + case tkbtf.ProbeTypeKRetProbe: + trProbe.Type = tracing.TypeKRetProbe + default: + trProbe.Type = tracing.TypeKProbe + } + trProbesMap[trProbe] = p.allocateFn + } + } + + return trProbesMap, nil +} + +func (fs *probeManager) onErr(err error) bool { + repeat := false + for _, s := range fs.symbols { + if s.onErr(err) { + repeat = true + } + } + + return repeat +} diff --git a/auditbeat/module/file_integrity/kprobes/probes_fsnotify.go b/auditbeat/module/file_integrity/kprobes/probes_fsnotify.go new file mode 100644 index 00000000000..39f944377ea --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes_fsnotify.go @@ -0,0 +1,195 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "fmt" + + tkbtf "github.com/elastic/tk-btf" +) + +type fsNotifySymbol struct { + symbolName string + inodeProbeFilter string + dentryProbeFilter string + pathProbeFilter string + lastOnErr error + seenSpecs map[*tkbtf.Spec]struct{} +} + +func loadFsNotifySymbol(s *probeManager) error { + symbolInfo, err := s.getSymbolInfoRuntime("fsnotify") + if err != nil { + return err + } + + if symbolInfo.isOptimised { + return fmt.Errorf("symbol %s is optimised", symbolInfo.symbolName) + } + + s.buildChecks = append(s.buildChecks, func(spec *tkbtf.Spec) bool { + return spec.ContainsSymbol(symbolInfo.symbolName) + }) + + // default filters for all three fsnotify probes enable mask_create, mask_delete, mask_attrib, mask_modify, + // mask_moved_to, and mask_moved_from events. + s.symbols = append(s.symbols, &fsNotifySymbol{ + symbolName: symbolInfo.symbolName, + }) + + return nil +} + +func (f *fsNotifySymbol) buildProbes(spec *tkbtf.Spec) ([]*probeWithAllocFunc, error) { + allocFunc := allocProbeEvent + + _, seen := f.seenSpecs[spec] + if !seen { + + if f.seenSpecs == nil { + f.seenSpecs = make(map[*tkbtf.Spec]struct{}) + } + + f.lastOnErr = nil + // reset probe filters for each new spec + // this probes shouldn't cause any ErrVerifyOverlappingEvents or ErrVerifyMissingEvents + // for linux kernel versions linux 5.17+, thus we start from here. To see how we handle all + // linux kernels filter variation check OnErr() method. + f.seenSpecs[spec] = struct{}{} + f.pathProbeFilter = "(mc==1 || md==1 || ma==1 || mm==1 || mmt==1 || mmf==1) && dt==1" + f.inodeProbeFilter = "(mc==1 || md==1 || ma==1 || mm==1 || mmt==1 || mmf==1) && dt==2 && nptr!=0" + f.dentryProbeFilter = "(mc==1 || md==1 || ma==1 || mm==1 || mmt==1 || mmf==1) && dt==3" + } + + pathProbe := tkbtf.NewKProbe().SetRef("fsnotify_path").AddFetchArgs( + tkbtf.NewFetchArg("pi", "u64").FuncParamWithCustomType("data", tkbtf.WrapPointer, "path", "dentry", "d_parent", "d_inode", "i_ino"), + tkbtf.NewFetchArg("mc", tkbtf.BitFieldTypeMask(fsEventCreate)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("md", tkbtf.BitFieldTypeMask(fsEventDelete)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("ma", tkbtf.BitFieldTypeMask(fsEventAttrib)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mm", tkbtf.BitFieldTypeMask(fsEventModify)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mid", tkbtf.BitFieldTypeMask(fsEventIsDir)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mmt", tkbtf.BitFieldTypeMask(fsEventMovedTo)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mmf", tkbtf.BitFieldTypeMask(fsEventMovedFrom)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("fi", "u64").FuncParamWithCustomType("data", tkbtf.WrapPointer, "path", "dentry", "d_inode", "i_ino"), + tkbtf.NewFetchArg("dt", "s32").FuncParamWithName("data_type").FuncParamWithName("data_is"), + tkbtf.NewFetchArg("fdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "path", "dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "path", "dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "path", "dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "path", "dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fn", "string").FuncParamWithCustomType("data", tkbtf.WrapPointer, "path", "dentry", "d_name", "name"), + ).SetFilter(f.pathProbeFilter) + + inodeProbe := tkbtf.NewKProbe().SetRef("fsnotify_inode").AddFetchArgs( + tkbtf.NewFetchArg("pi", "u64").FuncParamWithName("dir", "i_ino").FuncParamWithName("to_tell", "i_ino"), + tkbtf.NewFetchArg("mc", tkbtf.BitFieldTypeMask(fsEventCreate)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("md", tkbtf.BitFieldTypeMask(fsEventDelete)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("ma", tkbtf.BitFieldTypeMask(fsEventAttrib)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mm", tkbtf.BitFieldTypeMask(fsEventModify)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mid", tkbtf.BitFieldTypeMask(fsEventIsDir)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mmt", tkbtf.BitFieldTypeMask(fsEventMovedTo)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mmf", tkbtf.BitFieldTypeMask(fsEventMovedFrom)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("nptr", "u64").FuncParamWithName("file_name"), + tkbtf.NewFetchArg("fi", "u64").FuncParamWithCustomType("data", tkbtf.WrapPointer, "inode", "i_ino"), + tkbtf.NewFetchArg("dt", "s32").FuncParamWithName("data_type").FuncParamWithName("data_is"), + tkbtf.NewFetchArg("fdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithName("dir", "i_sb", "s_dev").FuncParamWithName("to_tell", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithName("dir", "i_sb", "s_dev").FuncParamWithName("to_tell", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fn", "string").FuncParamWithName("file_name", "name").FuncParamWithName("file_name"), + ).SetFilter(f.inodeProbeFilter) + + dentryProbe := tkbtf.NewKProbe().SetRef("fsnotify_dentry").AddFetchArgs( + tkbtf.NewFetchArg("mc", tkbtf.BitFieldTypeMask(fsEventCreate)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("md", tkbtf.BitFieldTypeMask(fsEventDelete)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("ma", tkbtf.BitFieldTypeMask(fsEventAttrib)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mm", tkbtf.BitFieldTypeMask(fsEventModify)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mid", tkbtf.BitFieldTypeMask(fsEventIsDir)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mmt", tkbtf.BitFieldTypeMask(fsEventMovedTo)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mmf", tkbtf.BitFieldTypeMask(fsEventMovedFrom)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("fi", "u64").FuncParamWithCustomType("data", tkbtf.WrapPointer, "dentry", "d_inode", "i_ino"), + tkbtf.NewFetchArg("dt", "s32").FuncParamWithName("data_type").FuncParamWithName("data_is"), + tkbtf.NewFetchArg("fdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pi", "u64").FuncParamWithCustomType("data", tkbtf.WrapPointer, "dentry", "d_parent", "d_inode", "i_ino"), + tkbtf.NewFetchArg("pdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithCustomType("data", tkbtf.WrapPointer, "dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fn", "string").FuncParamWithCustomType("data", tkbtf.WrapPointer, "dentry", "d_name", "name"), + ).SetFilter(f.dentryProbeFilter) + + btfSymbol := tkbtf.NewSymbol(f.symbolName).AddProbes( + inodeProbe, + dentryProbe, + pathProbe, + ) + + if err := spec.BuildSymbol(btfSymbol); err != nil { + return nil, err + } + + return []*probeWithAllocFunc{ + { + probe: inodeProbe, + allocateFn: allocFunc, + }, + { + probe: dentryProbe, + allocateFn: allocFunc, + }, + { + probe: pathProbe, + allocateFn: allocFunc, + }, + }, nil +} + +func (f *fsNotifySymbol) onErr(err error) bool { + if f.lastOnErr != nil && errors.Is(err, f.lastOnErr) { + return false + } + + f.lastOnErr = err + + switch { + case errors.Is(err, ErrVerifyOverlappingEvents): + + // on ErrVerifyOverlappingEvents for linux kernel versions < 5.7 the __fsnotify_parent + // probe is capturing and sending the modify events as well, thus disable them for + // fsnotify and return true to signal a retry. + f.inodeProbeFilter = "(mc==1 || md==1 || ma==1 || mmt==1 || mmf==1) && dt==2 && nptr!=0" + f.dentryProbeFilter = "(mc==1 || md==1 || ma==1 || mmt==1 || mmf==1) && dt==3" + f.pathProbeFilter = "(mc==1 || md==1 || ma==1 || mmt==1 || mmf==1) && dt==1" + + return true + case errors.Is(err, ErrVerifyMissingEvents): + + // on ErrVerifyMissingEvents for linux kernel versions 5.10 - 5.16 the __fsnotify_parent + // probe is not capturing and sending the modify attributes events for directories, thus + // we adjust the filters to allow them flowing through fsnotify and return true to signal + // a retry. + f.pathProbeFilter = "(mc==1 || md==1 || ma==1 || mm==1 || mmt==1 || mmf==1) && dt==1" + f.inodeProbeFilter = "(mc==1 || md==1 || ma==1 || mm==1 || mmt==1 || mmf==1) && dt==2 && (nptr!=0 || (mid==1 && ma==1))" + f.dentryProbeFilter = "(mc==1 || md==1 || ma==1 || mm==1 || mmt==1 || mmf==1) && dt==3" + + return true + default: + return false + } +} diff --git a/auditbeat/module/file_integrity/kprobes/probes_fsnotify_nameremove.go b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_nameremove.go new file mode 100644 index 00000000000..ecabb94c7d2 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_nameremove.go @@ -0,0 +1,90 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "fmt" + + tkbtf "github.com/elastic/tk-btf" +) + +type fsNotifyNameRemoveSymbol struct { + symbolName string +} + +func loadFsNotifyNameRemoveSymbol(s *probeManager) error { + symbolInfo, err := s.getSymbolInfoRuntime("fsnotify_nameremove") + if err != nil { + if errors.Is(err, ErrSymbolNotFound) { + s.buildChecks = append(s.buildChecks, func(spec *tkbtf.Spec) bool { + return !spec.ContainsSymbol(symbolInfo.symbolName) + }) + return nil + } + return err + } + + if symbolInfo.isOptimised { + return fmt.Errorf("symbol %s is optimised", symbolInfo.symbolName) + } + + s.buildChecks = append(s.buildChecks, func(spec *tkbtf.Spec) bool { + return spec.ContainsSymbol(symbolInfo.symbolName) + }) + + s.symbols = append(s.symbols, &fsNotifyNameRemoveSymbol{ + symbolName: symbolInfo.symbolName, + }) + + return nil +} + +func (f *fsNotifyNameRemoveSymbol) buildProbes(spec *tkbtf.Spec) ([]*probeWithAllocFunc, error) { + allocFunc := allocDeleteProbeEvent + + probe := tkbtf.NewKProbe().AddFetchArgs( + tkbtf.NewFetchArg("mid", "u32").FuncParamWithName("isdir"), + tkbtf.NewFetchArg("fi", "u64").FuncParamWithName("dentry", "d_inode", "i_ino"), + tkbtf.NewFetchArg("fdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithName("dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithName("dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pi", "u64").FuncParamWithName("dentry", "d_parent", "d_inode", "i_ino"), + tkbtf.NewFetchArg("pdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithName("dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithName("dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fn", "string").FuncParamWithName("dentry", "d_name", "name"), + ) + + btfSymbol := tkbtf.NewSymbol(f.symbolName).AddProbes(probe) + + if err := spec.BuildSymbol(btfSymbol); err != nil { + return nil, err + } + + return []*probeWithAllocFunc{ + { + probe: probe, + allocateFn: allocFunc, + }, + }, nil +} + +func (f *fsNotifyNameRemoveSymbol) onErr(_ error) bool { + return false +} diff --git a/auditbeat/module/file_integrity/kprobes/probes_fsnotify_nameremove_test.go b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_nameremove_test.go new file mode 100644 index 00000000000..90169b697b3 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_nameremove_test.go @@ -0,0 +1,107 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_fsNotifyNameRemoveSymbol_buildProbes(t *testing.T) { + specs, err := loadEmbeddedSpecs() + require.NoError(t, err) + require.NotEmpty(t, specs) + + s := &fsNotifyNameRemoveSymbol{} + + for _, spec := range specs { + switch { + case spec.ContainsSymbol("fsnotify_nameremove"): + s.symbolName = "fsnotify_nameremove" + default: + continue + } + + _, err := s.buildProbes(spec) + require.NoError(t, err) + } +} + +func Test_fsNotifyNameRemoveSymbol_load(t *testing.T) { + prbMgr := &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.NoError(t, loadFsNotifyNameRemoveSymbol(prbMgr)) + require.Equal(t, len(prbMgr.symbols), 0) + require.Equal(t, len(prbMgr.buildChecks), 1) + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + if symbolName != "fsnotify_nameremove" { + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + + return runtimeSymbolInfo{ + symbolName: "fsnotify_nameremove", + isOptimised: true, + optimisedSymbolName: "fsnotify_nameremove.isra.0", + }, nil + } + require.Error(t, loadFsNotifyNameRemoveSymbol(prbMgr)) + + unknownErr := errors.New("unknown error") + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{}, unknownErr + } + require.Error(t, loadFsNotifyNameRemoveSymbol(prbMgr)) + + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{ + symbolName: "fsnotify_nameremove", + isOptimised: false, + optimisedSymbolName: "", + }, nil + } + + require.NoError(t, loadFsNotifyNameRemoveSymbol(prbMgr)) + require.Equal(t, len(prbMgr.symbols), 1) + require.Equal(t, len(prbMgr.buildChecks), 1) +} + +func Test_fsNotifyNameRemoveSymbol_onErr(t *testing.T) { + s := &fsNotifyNameRemoveSymbol{} + + testErr := fmt.Errorf("test: %w", ErrVerifyOverlappingEvents) + repeat := s.onErr(testErr) + require.False(t, repeat) +} diff --git a/auditbeat/module/file_integrity/kprobes/probes_fsnotify_parent.go b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_parent.go new file mode 100644 index 00000000000..5b128273675 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_parent.go @@ -0,0 +1,99 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "fmt" + + tkbtf "github.com/elastic/tk-btf" +) + +type fsNotifyParentSymbol struct { + symbolName string + filter string +} + +func loadFsNotifyParentSymbol(s *probeManager) error { + symbolInfo, err := s.getSymbolInfoRuntime("__fsnotify_parent") + if err != nil { + if !errors.Is(err, ErrSymbolNotFound) { + return err + } + + symbolInfo, err = s.getSymbolInfoRuntime("fsnotify_parent") + if err != nil { + return err + } + } + + if symbolInfo.isOptimised { + return fmt.Errorf("symbol %s is optimised", symbolInfo.symbolName) + } + + s.buildChecks = append(s.buildChecks, func(spec *tkbtf.Spec) bool { + return spec.ContainsSymbol(symbolInfo.symbolName) + }) + + s.symbols = append(s.symbols, &fsNotifyParentSymbol{ + symbolName: symbolInfo.symbolName, + filter: "(mc==1 || md==1 || ma==1 || mm==1)", + }) + + return nil +} + +func (f *fsNotifyParentSymbol) buildProbes(spec *tkbtf.Spec) ([]*probeWithAllocFunc, error) { + allocFunc := allocProbeEvent + + probe := tkbtf.NewKProbe().AddFetchArgs( + tkbtf.NewFetchArg("mc", tkbtf.BitFieldTypeMask(fsEventCreate)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("md", tkbtf.BitFieldTypeMask(fsEventDelete)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("ma", tkbtf.BitFieldTypeMask(fsEventAttrib)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mm", tkbtf.BitFieldTypeMask(fsEventModify)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mid", tkbtf.BitFieldTypeMask(fsEventIsDir)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mmt", tkbtf.BitFieldTypeMask(fsEventMovedTo)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("mmf", tkbtf.BitFieldTypeMask(fsEventMovedFrom)).FuncParamWithName("mask"), + tkbtf.NewFetchArg("fi", "u64").FuncParamWithName("dentry", "d_inode", "i_ino"), + tkbtf.NewFetchArg("fdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithName("dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithName("dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pi", "u64").FuncParamWithName("dentry", "d_parent", "d_inode", "i_ino"), + tkbtf.NewFetchArg("pdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithName("dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithName("dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fn", "string").FuncParamWithName("dentry", "d_name", "name"), + ).SetFilter(f.filter) + + btfSymbol := tkbtf.NewSymbol(f.symbolName).AddProbes(probe) + + if err := spec.BuildSymbol(btfSymbol); err != nil { + return nil, err + } + + return []*probeWithAllocFunc{ + { + probe: probe, + allocateFn: allocFunc, + }, + }, nil +} + +func (f *fsNotifyParentSymbol) onErr(_ error) bool { + return false +} diff --git a/auditbeat/module/file_integrity/kprobes/probes_fsnotify_parent_test.go b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_parent_test.go new file mode 100644 index 00000000000..4d4ea4bb47f --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_parent_test.go @@ -0,0 +1,141 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_fsNotifyParentSymbol_buildProbes(t *testing.T) { + specs, err := loadEmbeddedSpecs() + require.NoError(t, err) + require.NotEmpty(t, specs) + + s := &fsNotifyParentSymbol{} + + for _, spec := range specs { + switch { + case spec.ContainsSymbol("__fsnotify_parent"): + s.symbolName = "__fsnotify_parent" + case spec.ContainsSymbol("fsnotify_parent"): + s.symbolName = "fsnotify_parent" + default: + t.FailNow() + } + + _, err := s.buildProbes(spec) + require.NoError(t, err) + } +} + +func Test_fsNotifyParentSymbol_load(t *testing.T) { + prbMgr := &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.ErrorIs(t, loadFsNotifyParentSymbol(prbMgr), ErrSymbolNotFound) + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + if symbolName == "fsnotify_parent" { + return runtimeSymbolInfo{ + symbolName: "fsnotify_parent", + isOptimised: true, + optimisedSymbolName: "fsnotify_parent.isra.0", + }, nil + } + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.Error(t, loadFsNotifyParentSymbol(prbMgr)) + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + if symbolName == "fsnotify_parent" { + return runtimeSymbolInfo{ + symbolName: "fsnotify_parent", + isOptimised: false, + optimisedSymbolName: "", + }, nil + } + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.NoError(t, loadFsNotifyParentSymbol(prbMgr)) + require.NotEmpty(t, prbMgr.symbols) + require.NotEmpty(t, prbMgr.buildChecks) + require.IsType(t, &fsNotifyParentSymbol{}, prbMgr.symbols[0]) + require.Equal(t, prbMgr.symbols[0].(*fsNotifyParentSymbol).symbolName, "fsnotify_parent") + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + if symbolName == "__fsnotify_parent" { + return runtimeSymbolInfo{ + symbolName: "__fsnotify_parent", + isOptimised: false, + optimisedSymbolName: "", + }, nil + } + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.NoError(t, loadFsNotifyParentSymbol(prbMgr)) + require.NotEmpty(t, prbMgr.symbols) + require.NotEmpty(t, prbMgr.buildChecks) + require.IsType(t, &fsNotifyParentSymbol{}, prbMgr.symbols[0]) + require.Equal(t, prbMgr.symbols[0].(*fsNotifyParentSymbol).symbolName, "__fsnotify_parent") + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + unknownErr := errors.New("unknown error") + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{}, unknownErr + } + require.Error(t, loadFsNotifyParentSymbol(prbMgr)) +} + +func Test_fsNotifyParentSymbol_onErr(t *testing.T) { + s := &fsNotifyParentSymbol{} + + testErr := fmt.Errorf("test: %w", ErrVerifyOverlappingEvents) + repeat := s.onErr(testErr) + require.False(t, repeat) +} diff --git a/auditbeat/module/file_integrity/kprobes/probes_fsnotify_test.go b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_test.go new file mode 100644 index 00000000000..9392deffe72 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes_fsnotify_test.go @@ -0,0 +1,101 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_fsNotifySymbol_buildProbes(t *testing.T) { + specs, err := loadEmbeddedSpecs() + require.NoError(t, err) + require.NotEmpty(t, specs) + + s := &fsNotifySymbol{ + symbolName: "fsnotify", + lastOnErr: nil, + } + + for _, spec := range specs { + + if !spec.ContainsSymbol("fsnotify") { + t.FailNow() + } + + _, err := s.buildProbes(spec) + require.NoError(t, err) + } +} + +func Test_fsNotifySymbol_load(t *testing.T) { + prbMgr := &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.ErrorIs(t, loadFsNotifySymbol(prbMgr), ErrSymbolNotFound) + + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + if symbolName != "fsnotify" { + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + + return runtimeSymbolInfo{ + symbolName: "fsnotify", + isOptimised: true, + optimisedSymbolName: "fsnotify.isra.0", + }, nil + } + + require.Error(t, loadFsNotifySymbol(prbMgr)) + + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{ + symbolName: "fsnotify", + isOptimised: false, + optimisedSymbolName: "", + }, nil + } + + require.NoError(t, loadFsNotifySymbol(prbMgr)) + require.Equal(t, len(prbMgr.symbols), 1) + require.Equal(t, len(prbMgr.buildChecks), 1) +} + +func Test_fsNotifySymbol_onErr(t *testing.T) { + s := &fsNotifySymbol{ + symbolName: "fsnotify", + lastOnErr: nil, + } + + require.True(t, s.onErr(ErrVerifyOverlappingEvents)) + + require.True(t, s.onErr(ErrVerifyMissingEvents)) + + require.False(t, s.onErr(ErrVerifyMissingEvents)) + + require.False(t, s.onErr(ErrVerifyUnexpectedEvent)) +} diff --git a/auditbeat/module/file_integrity/kprobes/probes_vfs_getattr.go b/auditbeat/module/file_integrity/kprobes/probes_vfs_getattr.go new file mode 100644 index 00000000000..6bc0f4ab01f --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes_vfs_getattr.go @@ -0,0 +1,95 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "errors" + "fmt" + + tkbtf "github.com/elastic/tk-btf" +) + +type vfsGetAttrSymbol struct { + symbolName string + filter string +} + +func loadVFSGetAttrSymbol(s *probeManager, e executor) error { + // get the vfs_getattr_nosec symbol information + symbolInfo, err := s.getSymbolInfoRuntime("vfs_getattr_nosec") + if err != nil { + if !errors.Is(err, ErrSymbolNotFound) { + return err + } + + // for older kernel versions use the vfs_getattr symbol + symbolInfo, err = s.getSymbolInfoRuntime("vfs_getattr") + if err != nil { + return err + } + } + + // we do not support optimised symbols + if symbolInfo.isOptimised { + return fmt.Errorf("symbol %s is optimised", symbolInfo.symbolName) + } + + s.buildChecks = append(s.buildChecks, func(spec *tkbtf.Spec) bool { + return spec.ContainsSymbol(symbolInfo.symbolName) + }) + + s.symbols = append(s.symbols, &vfsGetAttrSymbol{ + symbolName: symbolInfo.symbolName, + filter: fmt.Sprintf("common_pid==%d", e.GetTID()), + }) + + return nil +} + +func (f *vfsGetAttrSymbol) buildProbes(spec *tkbtf.Spec) ([]*probeWithAllocFunc, error) { + allocFunc := allocMonitorProbeEvent + + probe := tkbtf.NewKProbe().AddFetchArgs( + tkbtf.NewFetchArg("pi", "u64").FuncParamWithName("path", "dentry", "d_parent", "d_inode", "i_ino"), + tkbtf.NewFetchArg("fi", "u64").FuncParamWithName("path", "dentry", "d_inode", "i_ino"), + tkbtf.NewFetchArg("fdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithName("path", "dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithName("path", "dentry", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmj", tkbtf.BitFieldTypeMask(devMajor)).FuncParamWithName("path", "dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("pdmn", tkbtf.BitFieldTypeMask(devMinor)).FuncParamWithName("path", "dentry", "d_parent", "d_inode", "i_sb", "s_dev"), + tkbtf.NewFetchArg("fn", "string").FuncParamWithName("path", "dentry", "d_name", "name"), + ).SetFilter(f.filter) + + btfSymbol := tkbtf.NewSymbol(f.symbolName).AddProbes(probe) + + if err := spec.BuildSymbol(btfSymbol); err != nil { + return nil, err + } + + return []*probeWithAllocFunc{ + { + probe: probe, + allocateFn: allocFunc, + }, + }, nil +} + +func (f *vfsGetAttrSymbol) onErr(_ error) bool { + return false +} diff --git a/auditbeat/module/file_integrity/kprobes/probes_vfs_getattr_test.go b/auditbeat/module/file_integrity/kprobes/probes_vfs_getattr_test.go new file mode 100644 index 00000000000..114f026e179 --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/probes_vfs_getattr_test.go @@ -0,0 +1,148 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_vfsGetAttr_buildProbes(t *testing.T) { + specs, err := loadEmbeddedSpecs() + require.NoError(t, err) + require.NotEmpty(t, specs) + + s := &vfsGetAttrSymbol{} + + for _, spec := range specs { + switch { + case spec.ContainsSymbol("vfs_getattr_nosec"): + s.symbolName = "vfs_getattr_nosec" + case spec.ContainsSymbol("vfs_getattr"): + s.symbolName = "vfs_getattr" + default: + t.FailNow() + } + + _, err := s.buildProbes(spec) + require.NoError(t, err) + + if err != nil { + t.FailNow() + } + } +} + +func Test_vfsGetAttr_load(t *testing.T) { + exec := newFixedThreadExecutor(context.TODO()) + + prbMgr := &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.ErrorIs(t, loadVFSGetAttrSymbol(prbMgr, exec), ErrSymbolNotFound) + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + if symbolName == "vfs_getattr_nosec" { + return runtimeSymbolInfo{ + symbolName: "vfs_getattr_nosec", + isOptimised: true, + optimisedSymbolName: "vfs_getattr_nosec.isra.0", + }, nil + } + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.Error(t, loadVFSGetAttrSymbol(prbMgr, exec)) + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + if symbolName == "vfs_getattr" { + return runtimeSymbolInfo{ + symbolName: "vfs_getattr", + isOptimised: false, + optimisedSymbolName: "", + }, nil + } + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.NoError(t, loadVFSGetAttrSymbol(prbMgr, exec)) + require.NotEmpty(t, prbMgr.symbols) + require.NotEmpty(t, prbMgr.buildChecks) + require.IsType(t, &vfsGetAttrSymbol{}, prbMgr.symbols[0]) + require.Equal(t, prbMgr.symbols[0].(*vfsGetAttrSymbol).symbolName, "vfs_getattr") + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + if symbolName == "vfs_getattr_nosec" { + return runtimeSymbolInfo{ + symbolName: "vfs_getattr_nosec", + isOptimised: false, + optimisedSymbolName: "", + }, nil + } + return runtimeSymbolInfo{}, ErrSymbolNotFound + } + require.NoError(t, loadVFSGetAttrSymbol(prbMgr, exec)) + require.NotEmpty(t, prbMgr.symbols) + require.NotEmpty(t, prbMgr.buildChecks) + require.IsType(t, &vfsGetAttrSymbol{}, prbMgr.symbols[0]) + require.Equal(t, prbMgr.symbols[0].(*vfsGetAttrSymbol).symbolName, "vfs_getattr_nosec") + + prbMgr = &probeManager{ + symbols: nil, + buildChecks: nil, + getSymbolInfoRuntime: nil, + } + unknownErr := errors.New("unknown error") + prbMgr.getSymbolInfoRuntime = func(symbolName string) (runtimeSymbolInfo, error) { + return runtimeSymbolInfo{}, unknownErr + } + require.Error(t, loadVFSGetAttrSymbol(prbMgr, exec)) +} + +func Test_vfsGetAttr_onErr(t *testing.T) { + s := &vfsGetAttrSymbol{} + + testErr := fmt.Errorf("test: %w", ErrVerifyOverlappingEvents) + repeat := s.onErr(testErr) + require.False(t, repeat) +} diff --git a/auditbeat/module/file_integrity/kprobes/verifier.go b/auditbeat/module/file_integrity/kprobes/verifier.go new file mode 100644 index 00000000000..0ea1cf57f1e --- /dev/null +++ b/auditbeat/module/file_integrity/kprobes/verifier.go @@ -0,0 +1,226 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package kprobes + +import ( + "bytes" + "context" + "embed" + "errors" + "io/fs" + "os" + "strings" + "time" + + "github.com/elastic/beats/v7/auditbeat/tracing" + + tkbtf "github.com/elastic/tk-btf" +) + +//go:embed embed +var embedBTFFolder embed.FS + +func getVerifiedProbes(ctx context.Context, timeout time.Duration) (map[tracing.Probe]tracing.AllocateFn, executor, error) { + fExec := newFixedThreadExecutor(ctx) + + probeMgr, err := newProbeManager(fExec) + if err != nil { + return nil, nil, err + } + + specs, err := loadAllSpecs() + if err != nil { + return nil, nil, err + } + + var allErr error + for len(specs) > 0 { + + s := specs[0] + if !probeMgr.shouldBuild(s) { + specs = specs[1:] + continue + } + + probes, err := probeMgr.build(s) + if err != nil { + allErr = errors.Join(allErr, err) + specs = specs[1:] + continue + } + + if err := verify(ctx, fExec, probes, timeout); err != nil { + if probeMgr.onErr(err) { + continue + } + allErr = errors.Join(allErr, err) + specs = specs[1:] + continue + } + + return probes, fExec, nil + } + + fExec.Close() + return nil, nil, errors.Join(allErr, errors.New("could not validate probes")) +} + +func loadAllSpecs() ([]*tkbtf.Spec, error) { + var specs []*tkbtf.Spec + + spec, err := tkbtf.NewSpecFromKernel() + if err != nil { + if !errors.Is(err, tkbtf.ErrSpecKernelNotSupported) { + return nil, err + } + } else { + specs = append(specs, spec) + } + + embeddedSpecs, err := loadEmbeddedSpecs() + if err != nil { + return nil, err + } + specs = append(specs, embeddedSpecs...) + return specs, nil +} + +func loadEmbeddedSpecs() ([]*tkbtf.Spec, error) { + var specs []*tkbtf.Spec + err := fs.WalkDir(embedBTFFolder, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if !strings.HasSuffix(path, ".btf") { + return nil + } + + embedFileBytes, err := embedBTFFolder.ReadFile(path) + if err != nil { + return err + } + + embedSpec, err := tkbtf.NewSpecFromReader(bytes.NewReader(embedFileBytes), nil) + if err != nil { + return err + } + + specs = append(specs, embedSpec) + return nil + }) + if err != nil { + return nil, err + } + + return specs, nil +} + +func verify(ctx context.Context, exec executor, probes map[tracing.Probe]tracing.AllocateFn, timeout time.Duration) error { + basePath, err := os.MkdirTemp("", "verifier") + if err != nil { + return err + } + + defer os.RemoveAll(basePath) + + verifier, err := newEventsVerifier(basePath) + if err != nil { + return err + } + + pChannel, err := newPerfChannel(probes, 4, 512, exec.GetTID()) + if err != nil { + return err + } + + m, err := newMonitor(ctx, true, pChannel, exec) + if err != nil { + return err + } + + defer m.Close() + + // start the monitor + if err := m.Start(); err != nil { + return err + } + + // spaw goroutine to send events to verifier to be verified + cancel := make(chan struct{}) + defer close(cancel) + + retC := make(chan error) + + go func() { + defer close(retC) + for { + select { + case runErr := <-m.ErrorChannel(): + retC <- runErr + return + + case ev, ok := <-m.EventChannel(): + if !ok { + retC <- errors.New("monitor closed unexpectedly") + return + } + + if err := verifier.validateEvent(ev.Path, ev.PID, ev.Op); err != nil { + retC <- err + return + } + continue + case <-time.After(timeout): + return + case <-cancel: + return + } + } + }() + + // add verify base path to monitor + if err := m.Add(basePath); err != nil { + return err + } + + // invoke verifier event generation from our executor + if err := exec.Run(verifier.GenerateEvents); err != nil { + return err + } + + // wait for either no new events arriving for timeout duration or + // ctx to be cancelled + select { + case err = <-retC: + if err != nil { + return err + } + case <-ctx.Done(): + return ctx.Err() + } + + // check that all events have been verified + if err := verifier.Verified(); err != nil { + return err + } + + return nil +} diff --git a/auditbeat/module/file_integrity/metricset.go b/auditbeat/module/file_integrity/metricset.go index 2c9c38d2d56..eeaaa67b365 100644 --- a/auditbeat/module/file_integrity/metricset.go +++ b/auditbeat/module/file_integrity/metricset.go @@ -71,10 +71,10 @@ type MetricSet struct { log *logp.Logger // Runtime params that are initialized on Run(). - bucket datastore.BoltBucket - scanStart time.Time - scanChan <-chan Event - fsnotifyChan <-chan Event + bucket datastore.BoltBucket + scanStart time.Time + scanChan <-chan Event + eventChan <-chan Event // Used when a hash can't be calculated nullHashes map[HashType]Digest @@ -87,7 +87,13 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } - r, err := NewEventReader(config) + logger := logp.NewLogger(moduleName) + id := base.Module().Config().ID + if id != "" { + logger = logger.With("id", id) + } + + r, err := NewEventReader(config, logger) if err != nil { return nil, fmt.Errorf("failed to initialize file event reader: %w", err) } @@ -96,7 +102,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, config: config, reader: r, - log: logp.NewLogger(moduleName), + log: logger, } ms.nullHashes = make(map[HashType]Digest, len(config.HashTypes)) @@ -118,11 +124,11 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) { return } - for ms.fsnotifyChan != nil || ms.scanChan != nil { + for ms.eventChan != nil || ms.scanChan != nil { select { - case event, ok := <-ms.fsnotifyChan: + case event, ok := <-ms.eventChan: if !ok { - ms.fsnotifyChan = nil + ms.eventChan = nil continue } @@ -161,9 +167,9 @@ func (ms *MetricSet) init(reporter mb.PushReporterV2) bool { } ms.bucket = bucket.(datastore.BoltBucket) - ms.fsnotifyChan, err = ms.reader.Start(reporter.Done()) + ms.eventChan, err = ms.reader.Start(reporter.Done()) if err != nil { - err = fmt.Errorf("failed to start fsnotify event producer: %w", err) + err = fmt.Errorf("failed to start event producer: %w", err) reporter.Error(err) ms.log.Errorw("Failed to initialize", "error", err) return false diff --git a/auditbeat/module/file_integrity/monitor/monitor.go b/auditbeat/module/file_integrity/monitor/monitor.go index 107a690d975..ae80d1a17dc 100644 --- a/auditbeat/module/file_integrity/monitor/monitor.go +++ b/auditbeat/module/file_integrity/monitor/monitor.go @@ -37,7 +37,7 @@ type Watcher interface { // New creates a new Watcher backed by fsnotify with optional recursive // logic. -func New(recursive bool, IsExcludedPath func(path string) bool) (Watcher, error) { +func New(recursive bool, isExcludedPath func(path string) bool) (Watcher, error) { watcher, err := fsnotify.NewWatcher() if err != nil { return nil, err @@ -45,7 +45,7 @@ func New(recursive bool, IsExcludedPath func(path string) bool) (Watcher, error) // Use our simulated recursive watches unless the fsnotify implementation // supports OS-provided recursive watches if recursive && watcher.SetRecursive() != nil { - return newRecursiveWatcher(watcher, IsExcludedPath), nil //nolint:nilerr // Ignore SetRecursive() errors. + return newRecursiveWatcher(watcher, isExcludedPath), nil //nolint:nilerr // Ignore SetRecursive() errors. } return (*nonRecursiveWatcher)(watcher), nil } diff --git a/auditbeat/module/file_integrity/monitor/recursive.go b/auditbeat/module/file_integrity/monitor/recursive.go index 80ab3e742ef..7a0768d6fcb 100644 --- a/auditbeat/module/file_integrity/monitor/recursive.go +++ b/auditbeat/module/file_integrity/monitor/recursive.go @@ -40,7 +40,7 @@ type recursiveWatcher struct { isExcludedPath func(path string) bool } -func newRecursiveWatcher(inner *fsnotify.Watcher, IsExcludedPath func(path string) bool) *recursiveWatcher { +func newRecursiveWatcher(inner *fsnotify.Watcher, isExcludedPath func(path string) bool) *recursiveWatcher { return &recursiveWatcher{ inner: inner, tree: FileTree{}, @@ -48,7 +48,7 @@ func newRecursiveWatcher(inner *fsnotify.Watcher, IsExcludedPath func(path strin addC: make(chan string), addErrC: make(chan error), log: logp.NewLogger(moduleName), - isExcludedPath: IsExcludedPath, + isExcludedPath: isExcludedPath, } } diff --git a/auditbeat/module/file_integrity/schema.fbs b/auditbeat/module/file_integrity/schema.fbs index 9e0863f6379..7787b225b91 100644 --- a/auditbeat/module/file_integrity/schema.fbs +++ b/auditbeat/module/file_integrity/schema.fbs @@ -12,6 +12,8 @@ enum Action : ubyte (bit_flags) { enum Source : ubyte { Scan, FSNotify, + eBPF, + KProbes } enum Type : ubyte { @@ -19,6 +21,10 @@ enum Type : ubyte { File, Dir, Symlink, + CharDevice, + BlockDevice, + FIFO, + Socket, } table Metadata { diff --git a/auditbeat/module/file_integrity/schema/Source.go b/auditbeat/module/file_integrity/schema/Source.go index 94730ce2957..66baa499187 100644 --- a/auditbeat/module/file_integrity/schema/Source.go +++ b/auditbeat/module/file_integrity/schema/Source.go @@ -26,16 +26,22 @@ type Source byte const ( SourceScan Source = 0 SourceFSNotify Source = 1 + SourceEBPF Source = 2 + SourceKProbes Source = 3 ) var EnumNamesSource = map[Source]string{ SourceScan: "Scan", SourceFSNotify: "FSNotify", + SourceEBPF: "eBPF", + SourceKProbes: "KProbes", } var EnumValuesSource = map[string]Source{ "Scan": SourceScan, "FSNotify": SourceFSNotify, + "eBPF": SourceEBPF, + "KProbes": SourceKProbes, } func (v Source) String() string { diff --git a/auditbeat/module/file_integrity/schema/Type.go b/auditbeat/module/file_integrity/schema/Type.go index 2025ee3b096..a0dc4e7a416 100644 --- a/auditbeat/module/file_integrity/schema/Type.go +++ b/auditbeat/module/file_integrity/schema/Type.go @@ -24,24 +24,36 @@ import "strconv" type Type byte const ( - TypeUnknown Type = 0 - TypeFile Type = 1 - TypeDir Type = 2 - TypeSymlink Type = 3 + TypeUnknown Type = 0 + TypeFile Type = 1 + TypeDir Type = 2 + TypeSymlink Type = 3 + TypeCharDevice Type = 4 + TypeBlockDevice Type = 5 + TypeFIFO Type = 6 + TypeSocket Type = 7 ) var EnumNamesType = map[Type]string{ - TypeUnknown: "Unknown", - TypeFile: "File", - TypeDir: "Dir", - TypeSymlink: "Symlink", + TypeUnknown: "Unknown", + TypeFile: "File", + TypeDir: "Dir", + TypeSymlink: "Symlink", + TypeCharDevice: "CharDevice", + TypeBlockDevice: "BlockDevice", + TypeFIFO: "FIFO", + TypeSocket: "Socket", } var EnumValuesType = map[string]Type{ - "Unknown": TypeUnknown, - "File": TypeFile, - "Dir": TypeDir, - "Symlink": TypeSymlink, + "Unknown": TypeUnknown, + "File": TypeFile, + "Dir": TypeDir, + "Symlink": TypeSymlink, + "CharDevice": TypeCharDevice, + "BlockDevice": TypeBlockDevice, + "FIFO": TypeFIFO, + "Socket": TypeSocket, } func (v Type) String() string { diff --git a/auditbeat/tests/system/test_file_integrity.py b/auditbeat/tests/system/test_file_integrity.py index 280d2916a55..e6b03306c3a 100644 --- a/auditbeat/tests/system/test_file_integrity.py +++ b/auditbeat/tests/system/test_file_integrity.py @@ -1,9 +1,36 @@ +import os import time import unittest import platform from auditbeat import * +def is_root(): + if 'geteuid' not in dir(os): + return False + return os.geteuid() == 0 + + +def is_version_below(version, target): + t = list(map(int, target.split('.'))) + v = list(map(int, version.split('.'))) + v += [0] * (len(t) - len(v)) + for i in range(len(t)): + if v[i] != t[i]: + return v[i] < t[i] + return False + + +# Require Linux greater or equal than 3.10.0 and arm64/amd64 arch +def is_platform_supported(): + p = platform.platform().split('-') + if p[0] != 'Linux': + return False + if is_version_below(p[1], '3.10.0'): + return False + return {'aarch64', 'arm64', 'x86_64', 'amd64'}.intersection(p) + + # Escapes a path to match what's printed in the logs def escape_path(path): return path.replace('\\', '\\\\') @@ -49,22 +76,31 @@ def wrap_except(expr): class Test(BaseTest): - def wait_output(self, min_events): self.wait_until(lambda: wrap_except(lambda: len(self.read_output()) >= min_events)) - # wait for the number of lines in the file to stay constant for a second + # wait for the number of lines in the file to stay constant for 10 seconds prev_lines = -1 while True: num_lines = self.output_lines() if prev_lines < num_lines: prev_lines = num_lines - time.sleep(1) + time.sleep(10) else: break - @unittest.skipIf(os.getenv("CI") is not None and platform.system() == 'Darwin', - 'Flaky test: https://github.com/elastic/beats/issues/24678') - def test_non_recursive(self): + def wait_startup(self, backend, dir): + if backend == "ebpf": + self.wait_log_contains("started ebpf watcher", max_timeout=30, ignore_case=True) + if backend == "kprobes": + self.wait_log_contains("Started kprobes watcher", max_timeout=30, ignore_case=True) + else: + # wait until the directories to watch are printed in the logs + # this happens when the file_integrity module starts. + # Case must be ignored under windows as capitalisation of paths + # may differ + self.wait_log_contains(escape_path(dir), max_timeout=30, ignore_case=True) + + def _test_non_recursive(self, backend): """ file_integrity monitors watched directories (non recursive). """ @@ -73,22 +109,21 @@ def test_non_recursive(self): self.temp_dir("auditbeat_test")] with PathCleanup(dirs): + extras = { + "paths": dirs, + "scan_at_start": False + } + if platform.system() == "Linux": + extras["backend"] = backend + self.render_config_template( modules=[{ "name": "file_integrity", - "extras": { - "paths": dirs, - "scan_at_start": False - } + "extras": extras }], ) proc = self.start_beat() - - # wait until the directories to watch are printed in the logs - # this happens when the file_integrity module starts. - # Case must be ignored under windows as capitalisation of paths - # may differ - self.wait_log_contains(escape_path(dirs[0]), max_timeout=30, ignore_case=True) + self.wait_startup(backend, dirs[0]) file1 = os.path.join(dirs[0], 'file.txt') self.create_file(file1, "hello world!") @@ -109,10 +144,12 @@ def test_non_recursive(self): # log entries are JSON formatted, this value shows up as an escaped json string. self.wait_log_contains("\\\"deleted\\\"") - self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(subdir)), ignore_case=True) - self.wait_output(3) - self.wait_until(lambda: any( - 'file.path' in obj and obj['file.path'].lower() == subdir.lower() for obj in self.read_output())) + + if backend == "fsnotify" or backend == "kprobes": + self.wait_output(4) + else: + # ebpf backend doesn't catch directory creation + self.wait_output(3) proc.check_kill_and_wait() self.assert_no_logged_warnings() @@ -126,7 +163,8 @@ def test_non_recursive(self): has_file(objs, file1, "430ce34d020724ed75a196dfc2ad67c77772d169") has_file(objs, file2, "d23be250530a24be33069572db67995f21244c51") - has_dir(objs, subdir) + if backend == "fsnotify" or backend == "kprobes": + has_dir(objs, subdir) file_events(objs, file1, ['created', 'deleted']) file_events(objs, file2, ['created']) @@ -134,8 +172,21 @@ def test_non_recursive(self): # assert file inside subdir is not reported assert self.log_contains(file3) is False - @unittest.skipIf(os.getenv("BUILD_ID") is not None, "Skipped as flaky: https://github.com/elastic/beats/issues/7731") - def test_recursive(self): + @unittest.skipIf(os.getenv("CI") is not None and platform.system() == 'Darwin', + 'Flaky test: https://github.com/elastic/beats/issues/24678') + def test_non_recursive__fsnotify(self): + self._test_non_recursive("fsnotify") + + @unittest.skipUnless(is_root(), "Requires root") + def test_non_recursive__ebpf(self): + self._test_non_recursive("ebpf") + + @unittest.skipUnless(is_platform_supported(), "Requires Linux 3.10.0+ and arm64/amd64 arch") + @unittest.skipUnless(is_root(), "Requires root") + def test_non_recursive__kprobes(self): + self._test_non_recursive("kprobes") + + def _test_recursive(self, backend): """ file_integrity monitors watched directories (recursive). """ @@ -143,22 +194,22 @@ def test_recursive(self): dirs = [self.temp_dir("auditbeat_test")] with PathCleanup(dirs): + extras = { + "paths": dirs, + "scan_at_start": False, + "recursive": True + } + if platform.system() == "Linux": + extras["backend"] = backend + self.render_config_template( modules=[{ "name": "file_integrity", - "extras": { - "paths": dirs, - "scan_at_start": False, - "recursive": True - } + "extras": extras }], ) proc = self.start_beat() - - # wait until the directories to watch are printed in the logs - # this happens when the file_integrity module starts - self.wait_log_contains(escape_path(dirs[0]), max_timeout=30, ignore_case=True) - self.wait_log_contains("\"recursive\":true") + self.wait_startup(backend, dirs[0]) # auditbeat_test/subdir/ subdir = os.path.join(dirs[0], "subdir") @@ -174,10 +225,13 @@ def test_recursive(self): file2 = os.path.join(subdir2, "more.txt") self.create_file(file2, "") - self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file2)), ignore_case=True) - self.wait_output(4) - self.wait_until(lambda: any( - 'file.path' in obj and obj['file.path'].lower() == subdir2.lower() for obj in self.read_output())) + if backend == "fsnotify" or backend == "kprobes": + self.wait_output(4) + self.wait_until(lambda: any( + 'file.path' in obj and obj['file.path'].lower() == subdir2.lower() for obj in self.read_output())) + else: + # ebpf backend doesn't catch directory creation + self.wait_output(2) proc.check_kill_and_wait() self.assert_no_logged_warnings() @@ -191,8 +245,92 @@ def test_recursive(self): has_file(objs, file1, "430ce34d020724ed75a196dfc2ad67c77772d169") has_file(objs, file2, "da39a3ee5e6b4b0d3255bfef95601890afd80709") - has_dir(objs, subdir) - has_dir(objs, subdir2) + if backend == "fsnotify" or backend == "kprobes": + has_dir(objs, subdir) + has_dir(objs, subdir2) file_events(objs, file1, ['created']) file_events(objs, file2, ['created']) + + def test_recursive__fsnotify(self): + self._test_recursive("fsnotify") + + @unittest.skipUnless(is_root(), "Requires root") + def test_recursive__ebpf(self): + self._test_recursive("ebpf") + + @unittest.skipUnless(is_platform_supported(), "Requires Linux 3.10.0+ and arm64/amd64 arch") + @unittest.skipUnless(is_root(), "Requires root") + def test_recursive__kprobes(self): + self._test_recursive("kprobes") + + @unittest.skipIf(platform.system() != 'Linux', 'Non linux, skipping.') + def _test_file_modified(self, backend): + """ + file_integrity tests for file modifications (chmod, chown, write, truncate, xattrs). + """ + + dirs = [self.temp_dir("auditbeat_test")] + + with PathCleanup(dirs): + self.render_config_template( + modules=[{ + "name": "file_integrity", + "extras": { + "paths": dirs, + "scan_at_start": False, + "recursive": False, + "backend": backend + } + }], + ) + proc = self.start_beat() + self.wait_startup(backend, dirs[0]) + + # Event 1: file create + f = os.path.join(dirs[0], f'file_{backend}.txt') + self.create_file(f, "hello world!") + + # FSNotify can't catch the events if operations happens too fast + time.sleep(1) + + # Event 2: chmod + os.chmod(f, 0o777) + # FSNotify can't catch the events if operations happens too fast + time.sleep(1) + + with open(f, "w") as fd: + # Event 3: write + fd.write("data") + # FSNotify can't catch the events if operations happens too fast + time.sleep(1) + + # Event 4: truncate + fd.truncate(0) + # FSNotify can't catch the events if operations happens too fast + time.sleep(1) + + # Wait N events + self.wait_output(4) + + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + # Ensure all Beater stages are used. + assert self.log_contains("Setup Beat: auditbeat") + assert self.log_contains("auditbeat start running") + assert self.log_contains("auditbeat stopped") + + @unittest.skipIf(platform.system() != 'Linux', 'Non linux, skipping.') + def test_file_modified__fsnotify(self): + self._test_file_modified("fsnotify") + + @unittest.skipIf(platform.system() != 'Linux', 'Non linux, skipping.') + @unittest.skipUnless(is_root(), "Requires root") + def test_file_modified__ebpf(self): + self._test_file_modified("ebpf") + + @unittest.skipUnless(is_platform_supported(), "Requires Linux 3.10.0+ and arm64/amd64 arch") + @unittest.skipUnless(is_root(), "Requires root") + def test_file_modified__kprobes(self): + self._test_file_modified("kprobes") diff --git a/auditbeat/tests/system/test_show_command.py b/auditbeat/tests/system/test_show_command.py index 3aa15c0aec2..843ab7e829d 100644 --- a/auditbeat/tests/system/test_show_command.py +++ b/auditbeat/tests/system/test_show_command.py @@ -98,6 +98,7 @@ def test_show_auditd_status(self): 'lost', 'backlog', 'backlog_wait_time', + 'backlog_wait_time_actual', 'features', ] diff --git a/catalog-info.yaml b/catalog-info.yaml index 4d6c956f1f3..15fc06f85c9 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -23,7 +23,7 @@ apiVersion: backstage.io/v1alpha1 kind: Resource metadata: name: buildkite-pipeline-beats - description: 'Beats Main pipeline' + description: "Beats Main pipeline" links: - title: Pipeline url: https://buildkite.com/elastic/beats @@ -37,9 +37,9 @@ spec: kind: Pipeline metadata: name: beats - description: 'Beats Main pipeline' + description: "Beats Main pipeline" spec: - branch_configuration: "main 7.* 8.* v7.* v8.*" + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/pipeline.yml" provider_settings: build_pull_request_forks: false @@ -50,9 +50,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: '!main !7.* !8.*' + cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: '!main !7.* !8.*' + skip_intermediate_builds_branch_filter: "!main !7.* !8.*" # TODO uncomment this environment variable when pipeline definition is updated # env: # ELASTIC_PR_COMMENTS_ENABLED: 'true' @@ -68,7 +68,7 @@ apiVersion: backstage.io/v1alpha1 kind: Resource metadata: name: buildkite-pipeline-metricbeat - description: 'Beats: Metricbeat pipeline' + description: "Beats: Metricbeat pipeline" links: - title: Pipeline url: https://buildkite.com/elastic/beats-metricbeat @@ -82,12 +82,13 @@ spec: kind: Pipeline metadata: name: beats-metricbeat - description: 'Beats Metricbeat pipeline' + description: "Beats Metricbeat pipeline" spec: - branch_configuration: "main 7.* 8.* v7.* v8.*" + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/metricbeat/pipeline.yml" maximum_timeout_in_minutes: 120 provider_settings: + trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot build_tags: true @@ -96,12 +97,11 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: '!main !7.* !8.*' + cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: '!main !7.* !8.*' - # TODO uncomment this environment variable when pipeline definition is updated - # env: - # ELASTIC_PR_COMMENTS_ENABLED: 'true' + skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -130,10 +130,11 @@ spec: name: filebeat description: "Filebeat pipeline" spec: -# branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/filebeat/filebeat-pipeline.yml" -# maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + maximum_timeout_in_minutes: 120 provider_settings: + trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot build_tags: true @@ -145,8 +146,8 @@ spec: cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" skip_intermediate_builds: true skip_intermediate_builds_branch_filter: "!main !7.* !8.*" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -175,10 +176,11 @@ spec: name: auditbeat description: "Auditbeat pipeline" spec: - # branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks + # branch_configuration: "main 7.17 8.*" TODO: temporarily commented to build PRs from forks pipeline_file: ".buildkite/auditbeat/auditbeat-pipeline.yml" # maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready provider_settings: + trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot build_tags: true @@ -220,10 +222,11 @@ spec: name: heartbeat description: "Heartbeat pipeline" spec: - # branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/heartbeat/heartbeat-pipeline.yml" - # maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + maximum_timeout_in_minutes: 120 provider_settings: + trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot build_tags: true @@ -235,8 +238,8 @@ spec: cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" skip_intermediate_builds: true skip_intermediate_builds_branch_filter: "!main !7.* !8.*" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -265,10 +268,11 @@ spec: name: deploy-k8s description: "Deploy K8S pipeline" spec: - # branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks + # branch_configuration: "main 7.17 8.*" TODO: temporarily commented to build PRs from forks pipeline_file: ".buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml" # maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready provider_settings: + trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot build_tags: true @@ -310,10 +314,11 @@ spec: name: beats-libbeat description: "Beats libbeat pipeline" spec: -# branch_configuration: "main 7.17 8.* v7.17 v8.*" TODO: temporarily commented to build PRs from forks + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/libbeat/pipeline.libbeat.yml" -# maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + maximum_timeout_in_minutes: 120 provider_settings: + trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot build_tags: true @@ -325,8 +330,8 @@ spec: cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" skip_intermediate_builds: true skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -355,10 +360,56 @@ spec: name: beats-packetbeat description: "Beats packetbeat pipeline" spec: -# branch_configuration: "main 7.17 8.* v7.17 v8.*" TODO: temporarily commented to build PRs from forks + branch_configuration: "main 7.17 8.*" pipeline_file: ".buildkite/packetbeat/pipeline.packetbeat.yml" -# maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-elastic-agent + description: "Beats xpack elastic agent" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-elastic-agent + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-elastic-agent + description: "Beats xpack elastic agent pipeline" + spec: + branch_configuration: "7.17" + pipeline_file: ".buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.yml" provider_settings: + trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot build_tags: true @@ -377,3 +428,268 @@ spec: access_level: MANAGE_BUILD_AND_READ everyone: access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-winlogbeat + description: "Beats winlogbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-winlogbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-winlogbeat + description: "Beats winlogbeat pipeline" + spec: + branch_configuration: "main 7.17 8.*" + pipeline_file: ".buildkite/winlogbeat/pipeline.winlogbeat.yml" + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-winlogbeat + description: "Beats x-pack winlogbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-winlogbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-winlogbeat + description: "Beats x-pack winlogbeat pipeline" + spec: + # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.winlogbeat.yml" + # maximum_timeout_in_minutes: 120 #TODO: uncomment after tests + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-packetbeat + description: "Beats x-pack packetbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-packetbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-packetbeat + description: "Beats x-pack packetbeat pipeline" + spec: + # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.packetbeat.yml" + # maximum_timeout_in_minutes: 120 #TODO: uncomment after tests + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-libbeat + description: "Beats x-pack libbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-libbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-libbeat + description: "Beats x-pack libbeat pipeline" + spec: + # branch_configuration: "main 7.17 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.libbeat.yml" + # maximum_timeout_in_minutes: 120 #TODO: uncomment after tests + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-metricbeat + description: "Beats x-pack metricbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-metricbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-metricbeat + description: "Beats x-pack metricbeat pipeline" + spec: + # branch_configuration: "7.17" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.metricbeat.yml" + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!7.17" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!7.17" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" #TODO: uncomment after tests + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: beats-xpack-elastic-agent-package-7-17 + description: Buildkite pipeline for packaging Elastic Agent v7.17 package + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-elastic-agent-package-7-17 +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-elastic-agent-package-7-17 + description: Buildkite pipeline for packaging Elastic Agent package + spec: + repository: elastic/beats + pipeline_file: ".buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml" + provider_settings: + trigger_mode: none # this pipeline is only triggered remotely + cancel_intermediate_builds: false + skip_intermediate_builds: false + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + release-eng: + access_level: BUILD_AND_READ + everyone: + access_level: READ_ONLY diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug index 842da44f3ab..e8dfaf392ab 100644 --- a/dev-tools/kubernetes/filebeat/Dockerfile.debug +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.6 as builder +FROM golang:1.21.7 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug index fd9970a5b08..473ce7484c8 100644 --- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.6 as builder +FROM golang:1.21.7 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug index 00df9d9be1d..8adf8a45901 100644 --- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.6 as builder +FROM golang:1.21.7 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/notice/overrides.json b/dev-tools/notice/overrides.json index 1484fcde52a..eee18acc0de 100644 --- a/dev-tools/notice/overrides.json +++ b/dev-tools/notice/overrides.json @@ -17,3 +17,4 @@ {"name": "github.com/awslabs/kinesis-aggregation/go/v2", "licenceType": "Apache-2.0", "url": "https://github.com/awslabs/kinesis-aggregation/blob/master/LICENSE.txt"} {"name": "github.com/dnaeon/go-vcr", "licenceType": "BSD-2-Clause"} {"name": "github.com/JohnCGriffin/overflow", "licenceType": "MIT"} +{"name": "github.com/elastic/ebpfevents", "licenceType": "Apache-2.0"} diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index e01b6c566e5..fff920b429c 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -714,9 +714,11 @@ func readZip(t *testing.T, zipFile string, inspectors ...inspector) (*packageFil } func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { - // Read the manifest file first so that the config file and layer - // names are known in advance. - manifest, err := getDockerManifest(dockerFile) + var manifest *dockerManifest + var info *dockerInfo + layers := make(map[string]*packageFile) + + manifest, err := readManifest(dockerFile) if err != nil { return nil, nil, err } @@ -727,9 +729,6 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { } defer file.Close() - var info *dockerInfo - layers := make(map[string]*packageFile) - gzipReader, err := gzip.NewReader(file) if err != nil { return nil, nil, err @@ -770,11 +769,7 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { // Read layers in order and for each file keep only the entry seen in the later layer p := &packageFile{Name: filepath.Base(dockerFile), Contents: map[string]packageEntry{}} - for _, layer := range manifest.Layers { - layerFile, found := layers[layer] - if !found { - return nil, nil, fmt.Errorf("layer not found: %s", layer) - } + for _, layerFile := range layers { for name, entry := range layerFile.Contents { // Check only files in working dir and entrypoint if strings.HasPrefix("/"+name, workingDir) || "/"+name == entrypoint { @@ -799,22 +794,21 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { return p, info, nil } -// getDockerManifest opens a gzipped tar file to read the Docker manifest.json -// that it is expected to contain. -func getDockerManifest(file string) (*dockerManifest, error) { - f, err := os.Open(file) +func readManifest(dockerFile string) (*dockerManifest, error) { + var manifest *dockerManifest + + file, err := os.Open(dockerFile) if err != nil { return nil, err } - defer f.Close() + defer file.Close() - gzipReader, err := gzip.NewReader(f) + gzipReader, err := gzip.NewReader(file) if err != nil { return nil, err } defer gzipReader.Close() - var manifest *dockerManifest tarReader := tar.NewReader(gzipReader) for { header, err := tarReader.Next() @@ -833,8 +827,7 @@ func getDockerManifest(file string) (*dockerManifest, error) { break } } - - return manifest, nil + return manifest, err } type dockerManifest struct { diff --git a/docs/devguide/testing.asciidoc b/docs/devguide/testing.asciidoc index 49d2366c920..9488fe47dce 100644 --- a/docs/devguide/testing.asciidoc +++ b/docs/devguide/testing.asciidoc @@ -50,11 +50,11 @@ In Metricbeat, run the command from within a module like this: `go test --tags i A note about tags: the `--data` flag is a custom flag added by Metricbeat and Packetbeat frameworks. It will not be present in case tags do not match, as the relevant code will not be run and silently skipped (without the tag the test file is ignored by Go compiler so the framework doesn't load). This may happen if there are different tags in the build tags of the metricset under test (i.e. the GCP billing metricset requires the `billing` tag too). -==== Running Python Tests +==== Running System (integration) Tests (Python and Go) -Python system tests are defined in the `tests/system` directory. They require a testing binary to be available and the python environment to be set up. +The system tests are defined in the `tests/system` (for legacy Python test) and on `tests/integration` (for Go tests) directory. They require a testing binary to be available and the python environment to be set up. -To create the testing binary run `mage buildSystemTestBinary`. This will create the test binary in the beat directory. To setup the testing environment run `mage pythonVirtualEnv` which will create a virtual environment with all test dependencies and print its location. To activate it, the instructions depend on your operating system. See the https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#activating-a-virtual-environment[virtualenv documentation]. +To create the testing binary run `mage buildSystemTestBinary`. This will create the test binary in the beat directory. To set up the Python testing environment run `mage pythonVirtualEnv` which will create a virtual environment with all test dependencies and print its location. To activate it, the instructions depend on your operating system. See the https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#activating-a-virtual-environment[virtualenv documentation]. To run the system and integration tests use the `mage pythonIntegTest` target, which will start the required services using https://docs.docker.com/compose/[docker-compose] and run all integration tests. Similar to Go integration tests, the individual steps can be done manually to allow selecting which tests should be run: @@ -62,12 +62,16 @@ To run the system and integration tests use the `mage pythonIntegTest` target, w ---- # Create and activate the system test virtual environment (assumes a Unix system). source $(mage pythonVirtualEnv)/bin/activate + # Pull and build the containers. Only needs to be done once unless you change the containers. mage docker:composeBuild + # Bring up all containers, wait until they are healthy, and put them in the background. mage docker:composeUp + # Run all system and integration tests. INTEGRATION_TESTS=1 pytest ./tests/system + # Stop all started containers. mage docker:composeDown ---- diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 1c76bb70919..ddc887d246f 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -92,6 +92,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -158960,6 +158961,200 @@ alias to: source.geo.region_iso_code -- +[[exported-fields-winlog]] +== Windows ETW fields + +Fields from the ETW input (Event Tracing for Windows). + + + +[float] +=== winlog + +All fields specific to the Windows Event Tracing are defined here. + + + +*`winlog.activity_id`*:: ++ +-- +A globally unique identifier that identifies the current activity. The events that are published with this identifier are part of the same activity. + + +type: keyword + +required: False + +-- + +*`winlog.channel`*:: ++ +-- +Used to enable special event processing. Channel values below 16 are reserved for use by Microsoft to enable special treatment by the ETW runtime. Channel values 16 and above will be ignored by the ETW runtime (treated the same as channel 0) and can be given user-defined semantics. + + +type: keyword + +required: False + +-- + +*`winlog.event_data`*:: ++ +-- +The event-specific data. The content of this object is specific to any provider and event. + + +type: object + +required: False + +-- + +*`winlog.flags`*:: ++ +-- +Flags that provide information about the event such as the type of session it was logged to and if the event contains extended data. + + +type: keyword + +required: False + +-- + +*`winlog.keywords`*:: ++ +-- +The keywords are used to indicate an event's membership in a set of event categories. + + +type: keyword + +required: False + +-- + +*`winlog.level`*:: ++ +-- +Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider. + + +type: keyword + +required: False + +-- + +*`winlog.opcode`*:: ++ +-- +The opcode defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. + + +type: keyword + +required: False + +-- + +*`winlog.process_id`*:: ++ +-- +Identifies the process that generated the event. + + +type: keyword + +required: False + +-- + +*`winlog.provider_guid`*:: ++ +-- +A globally unique identifier that identifies the provider that logged the event. + + +type: keyword + +required: False + +-- + +*`winlog.provider_name`*:: ++ +-- +The source of the event log record (the application or service that logged the record). + + +type: keyword + +required: False + +-- + +*`winlog.session`*:: ++ +-- +Configured session to forward ETW events from providers to consumers. + + +type: keyword + +required: False + +-- + +*`winlog.severity`*:: ++ +-- +Human-readable level of severity. + + +type: keyword + +required: False + +-- + +*`winlog.task`*:: ++ +-- +The task defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. + + +type: keyword + +required: False + +-- + +*`winlog.thread_id`*:: ++ +-- +Identifies the thread that generated the event. + + +type: keyword + +required: False + +-- + +*`winlog.version`*:: ++ +-- +Specify the version of a manifest-based event. + + +type: long + +required: False + +-- + [[exported-fields-zeek]] == Zeek fields diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index faff00e7e3d..1e9f9cac6e0 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -75,8 +75,10 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-cometd>> * <<{beatname_lc}-input-container>> * <<{beatname_lc}-input-entity-analytics>> +* <<{beatname_lc}-input-etw>> * <<{beatname_lc}-input-filestream>> * <<{beatname_lc}-input-gcp-pubsub>> +* <<{beatname_lc}-input-gcs>> * <<{beatname_lc}-input-http_endpoint>> * <<{beatname_lc}-input-httpjson>> * <<{beatname_lc}-input-journald>> @@ -90,7 +92,7 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-syslog>> * <<{beatname_lc}-input-tcp>> * <<{beatname_lc}-input-udp>> -* <<{beatname_lc}-input-gcs>> +* <<{beatname_lc}-input-websocket>> include::multiline.asciidoc[] @@ -112,10 +114,14 @@ include::inputs/input-container.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc[] +include::../../x-pack/filebeat/docs/inputs/input-etw.asciidoc[] + include::inputs/input-filestream.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc[] +include::../../x-pack/filebeat/docs/inputs/input-gcs.asciidoc[] + include::../../x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-httpjson.asciidoc[] @@ -144,4 +150,4 @@ include::inputs/input-udp.asciidoc[] include::inputs/input-unix.asciidoc[] -include::../../x-pack/filebeat/docs/inputs/input-gcs.asciidoc[] +include::../../x-pack/filebeat/docs/inputs/input-websocket.asciidoc[] diff --git a/filebeat/docs/inputs/input-filestream-file-options.asciidoc b/filebeat/docs/inputs/input-filestream-file-options.asciidoc index db00a8fe766..a3be665e28e 100644 --- a/filebeat/docs/inputs/input-filestream-file-options.asciidoc +++ b/filebeat/docs/inputs/input-filestream-file-options.asciidoc @@ -527,6 +527,9 @@ duplicated events in the output. *`native`*:: The default behaviour of {beatname_uc} is to differentiate between files using their inodes and device ids. ++ +In some cases these values can change during the lifetime of a file. +For example, when using the Linux link:https://en.wikipedia.org/wiki/Logical_Volume_Manager_%28Linux%29[LVM] (Logical Volume Manager), device numbers are allocated dynamically at module load (refer to link:https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/logical_volume_manager_administration/lv#persistent_numbers[Persistent Device Numbers] in the Red Hat Enterprise Linux documentation). To avoid the possibility of data duplication in this case, you can set `file_identity` to `path` rather than `native`. [source,yaml] ---- diff --git a/filebeat/docs/inputs/input-filestream.asciidoc b/filebeat/docs/inputs/input-filestream.asciidoc index e55ff611496..47d1b24a8e8 100644 --- a/filebeat/docs/inputs/input-filestream.asciidoc +++ b/filebeat/docs/inputs/input-filestream.asciidoc @@ -94,7 +94,7 @@ By default, {beatname_uc} identifies files based on their inodes and device IDs. However, on network shares and cloud providers these values might change during the lifetime of the file. If this happens {beatname_uc} thinks that file is new and resends the whole content -of the file. To solve this problem you can configure `file_identity` option. Possible +of the file. To solve this problem you can configure the `file_identity` option. Possible values besides the default `inode_deviceid` are `path`, `inode_marker` and `fingerprint`. WARNING: Changing `file_identity` methods between runs may result in diff --git a/filebeat/docs/running-on-kubernetes.asciidoc b/filebeat/docs/running-on-kubernetes.asciidoc index acc2905ae06..889ec8d5d8b 100644 --- a/filebeat/docs/running-on-kubernetes.asciidoc +++ b/filebeat/docs/running-on-kubernetes.asciidoc @@ -61,17 +61,17 @@ in the manifest file: ------------------------------------------------ [float] -===== Running {beatname_uc} on master nodes +===== Running {beatname_uc} on control plane nodes -Kubernetes master nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] -to limit the workloads that can run on them. To run {beatname_uc} on master nodes you may need to +Kubernetes control plane nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] +to limit the workloads that can run on them. To run {beatname_uc} on control plane nodes you may need to update the Daemonset spec to include proper tolerations: [source,yaml] ------------------------------------------------ spec: tolerations: - - key: node-role.kubernetes.io/master + - key: node-role.kubernetes.io/control-plane effect: NoSchedule ------------------------------------------------ @@ -110,7 +110,7 @@ oc patch namespace kube-system -p \ ---- + This command sets the node selector for the project to an empty string. If you -don't run this command, the default node selector will skip master nodes. +don't run this command, the default node selector will skip control plane nodes. In order to support runtime environments with Openshift (eg. CRI-O, containerd) you need to configure following path: @@ -137,7 +137,7 @@ filebeat.autodiscover: - /var/log/containers/*.log ---- -NOTE: `/var/log/containers/\*.log` is normally a symlink to `/var/log/pods/*/*.log`, +NOTE: `/var/log/containers/\*.log` is normally a symlink to `/var/log/pods/*/*.log`, so above paths can be edited accordingly diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 755db3726e7..bc5ebdc3d15 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2447,9 +2447,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/filebeat/filebeat_windows_amd64.syso b/filebeat/filebeat_windows_amd64.syso new file mode 100644 index 00000000000..c52af94f8e0 Binary files /dev/null and b/filebeat/filebeat_windows_amd64.syso differ diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index 6cedd1bd9d1..43c6ddcb19f 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -179,6 +179,9 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo return nil, err } + ok := false // used for cleanup + defer cleanup.IfNot(&ok, cleanup.IgnoreError(f.Close)) + log.Debug("newLogFileReader with config.MaxBytes:", inp.readerConfig.MaxBytes) // if the file is archived, it means that it is not going to be updated in the future @@ -203,7 +206,6 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo dbgReader, err := debug.AppendReaders(logReader) if err != nil { - f.Close() return nil, err } @@ -221,7 +223,6 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo MaxBytes: encReaderMaxBytes, }) if err != nil { - f.Close() return nil, err } @@ -233,6 +234,7 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo r = readfile.NewLimitReader(r, inp.readerConfig.MaxBytes) + ok = true // no need to close the file return r, nil } @@ -252,11 +254,11 @@ func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*o return nil, nil, fmt.Errorf("failed to open file %s, named pipes are not supported", fi.Name()) } - ok := false f, err := file.ReadOpen(path) if err != nil { return nil, nil, fmt.Errorf("failed opening %s: %w", path, err) } + ok := false defer cleanup.IfNot(&ok, cleanup.IgnoreError(f.Close)) fi, err = f.Stat() @@ -280,14 +282,13 @@ func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*o encoding, err := inp.encodingFactory(f) if err != nil { - f.Close() if errors.Is(err, transform.ErrShortSrc) { return nil, nil, fmt.Errorf("initialising encoding for '%v' failed due to file being too short", f) } return nil, nil, fmt.Errorf("initialising encoding for '%v' failed: %w", f, err) } - ok = true + ok = true // no need to close the file return f, encoding, nil } diff --git a/filebeat/input/filestream/internal/task/group_test.go b/filebeat/input/filestream/internal/task/group_test.go index 553070e5ec7..5ce15d455e3 100644 --- a/filebeat/input/filestream/internal/task/group_test.go +++ b/filebeat/input/filestream/internal/task/group_test.go @@ -241,12 +241,14 @@ func TestGroup_Go(t *testing.T) { want := uint64(2) g := NewGroup(want, time.Second, logger, "errorPrefix") - wg.Add(2) + wg.Add(1) err := g.Go(workload(1)) require.NoError(t, err) + wg.Wait() + + wg.Add(1) err = g.Go(workload(2)) require.NoError(t, err) - wg.Wait() err = g.Stop() diff --git a/filebeat/input/journald/pkg/journalfield/conv.go b/filebeat/input/journald/pkg/journalfield/conv.go index bd7403ae142..94447b773b7 100644 --- a/filebeat/input/journald/pkg/journalfield/conv.go +++ b/filebeat/input/journald/pkg/journalfield/conv.go @@ -19,11 +19,11 @@ package journalfield import ( "fmt" - "math/bits" "regexp" "strconv" "strings" + "github.com/elastic/beats/v7/libbeat/common/capabilities" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -190,72 +190,13 @@ func expandCapabilities(fields mapstr.M) { if !ok { return } - w, err := strconv.ParseUint(c, 16, 64) - if err != nil { - return - } - if w == 0 { + caps, err := capabilities.FromString(c, 16) + if err != nil || len(caps) == 0 { return } - caps := make([]string, 0, bits.OnesCount64(w)) - for i := 0; w != 0; i++ { - if w&1 != 0 { - if i < len(capTable) { - caps = append(caps, capTable[i]) - } else { - caps = append(caps, strconv.Itoa(i)) - } - } - w >>= 1 - } fields.Put("process.thread.capabilities.effective", caps) } -// include/uapi/linux/capability.h -var capTable = [...]string{ - 0: "CAP_CHOWN", - 1: "CAP_DAC_OVERRIDE", - 2: "CAP_DAC_READ_SEARCH", - 3: "CAP_FOWNER", - 4: "CAP_FSETID", - 5: "CAP_KILL", - 6: "CAP_SETGID", - 7: "CAP_SETUID", - 8: "CAP_SETPCAP", - 9: "CAP_LINUX_IMMUTABLE", - 10: "CAP_NET_BIND_SERVICE", - 11: "CAP_NET_BROADCAST", - 12: "CAP_NET_ADMIN", - 13: "CAP_NET_RAW", - 14: "CAP_IPC_LOCK", - 15: "CAP_IPC_OWNER", - 16: "CAP_SYS_MODULE", - 17: "CAP_SYS_RAWIO", - 18: "CAP_SYS_CHROOT", - 19: "CAP_SYS_PTRACE", - 20: "CAP_SYS_PACCT", - 21: "CAP_SYS_ADMIN", - 22: "CAP_SYS_BOOT", - 23: "CAP_SYS_NICE", - 24: "CAP_SYS_RESOURCE", - 25: "CAP_SYS_TIME", - 26: "CAP_SYS_TTY_CONFIG", - 27: "CAP_MKNOD", - 28: "CAP_LEASE", - 29: "CAP_AUDIT_WRITE", - 30: "CAP_AUDIT_CONTROL", - 31: "CAP_SETFCAP", - 32: "CAP_MAC_OVERRIDE", - 33: "CAP_MAC_ADMIN", - 34: "CAP_SYSLOG", - 35: "CAP_WAKE_ALARM", - 36: "CAP_BLOCK_SUSPEND", - 37: "CAP_AUDIT_READ", - 38: "CAP_PERFMON", - 39: "CAP_BPF", - 40: "CAP_CHECKPOINT_RESTORE", -} - func getStringFromFields(key string, fields mapstr.M) string { value, _ := fields.GetValue(key) str, _ := value.(string) diff --git a/filebeat/input/journald/pkg/journalfield/conv_expand_test.go b/filebeat/input/journald/pkg/journalfield/conv_expand_test.go index c43e57a1c49..09daf7c8f5b 100644 --- a/filebeat/input/journald/pkg/journalfield/conv_expand_test.go +++ b/filebeat/input/journald/pkg/journalfield/conv_expand_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build linux && cgo + package journalfield import ( @@ -228,8 +230,8 @@ var expandCapabilitiesTests = []struct { "CAP_PERFMON", "CAP_BPF", "CAP_CHECKPOINT_RESTORE", - "41", - "42", + "CAP_41", + "CAP_42", }, }, }, diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index fba8debcaea..2bea57223fe 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -197,7 +197,10 @@ def test_file_renaming(self): # expecting 6 more events self.wait_until( - lambda: self.output_has(lines=iterations1 + iterations2), max_timeout=10) + lambda: self.output_has( + lines=iterations1 + + iterations2), + max_timeout=10) filebeat.check_kill_and_wait() @@ -247,7 +250,10 @@ def test_file_disappear(self): # Let it read the file self.wait_until( - lambda: self.output_has(lines=iterations1 + iterations2), max_timeout=10) + lambda: self.output_has( + lines=iterations1 + + iterations2), + max_timeout=10) filebeat.check_kill_and_wait() @@ -317,7 +323,10 @@ def test_file_disappear_appear(self): # Let it read the file self.wait_until( - lambda: self.output_has(lines=iterations1 + iterations2), max_timeout=10) + lambda: self.output_has( + lines=iterations1 + + iterations2), + max_timeout=10) filebeat.check_kill_and_wait() @@ -468,7 +477,8 @@ def test_tail_files(self): f.write("hello world 2\n") f.flush() - # Sleep 1 second to make sure the file is persisted on disk and timestamp is in the past + # Sleep 1 second to make sure the file is persisted on disk and + # timestamp is in the past time.sleep(1) filebeat = self.start_beat() @@ -569,6 +579,7 @@ def test_encodings(self): with codecs.open(self.working_dir + "/log/test-{}".format(enc_py), "w", enc_py) as f: f.write(text + "\n") + f.close() # create the config file inputs = [] @@ -592,10 +603,11 @@ def test_encodings(self): with codecs.open(self.working_dir + "/log/test-{}".format(enc_py), "a", enc_py) as f: f.write(text + " 2" + "\n") + f.close() # wait again self.wait_until(lambda: self.output_has(lines=len(encodings) * 2), - max_timeout=15) + max_timeout=60) filebeat.check_kill_and_wait() # check that all outputs are present in the JSONs in UTF-8 diff --git a/go.mod b/go.mod index a7044889fac..d087675031e 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/eapache/go-resiliency v1.2.0 github.com/eclipse/paho.mqtt.golang v1.3.5 - github.com/elastic/elastic-agent-client/v7 v7.6.0 + github.com/elastic/elastic-agent-client/v7 v7.8.0 github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.5.0 github.com/elastic/go-licenser v0.4.1 @@ -78,7 +78,7 @@ require ( github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 github.com/elastic/go-seccomp-bpf v1.4.0 github.com/elastic/go-structform v0.0.10 - github.com/elastic/go-sysinfo v1.11.2 + github.com/elastic/go-sysinfo v1.13.1 github.com/elastic/go-ucfg v0.8.6 github.com/elastic/gosigar v0.14.2 github.com/fatih/color v1.15.0 @@ -164,7 +164,7 @@ require ( google.golang.org/api v0.128.0 google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 // indirect google.golang.org/grpc v1.58.3 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.32.0 gopkg.in/inf.v0 v0.9.1 gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect @@ -200,20 +200,24 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 + github.com/elastic/ebpfevents v0.4.0 github.com/elastic/elastic-agent-autodiscover v0.6.7 github.com/elastic/elastic-agent-libs v0.7.5 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.9.1 github.com/elastic/go-elasticsearch/v8 v8.12.0 - github.com/elastic/mito v1.8.0 + github.com/elastic/mito v1.9.0 + github.com/elastic/tk-btf v0.1.0 github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 + github.com/go-ldap/ldap/v3 v3.4.6 github.com/google/cel-go v0.19.0 github.com/googleapis/gax-go/v2 v2.12.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 + github.com/gorilla/websocket v1.4.2 github.com/icholy/digest v0.1.22 - github.com/lestrrat-go/jwx/v2 v2.0.11 + github.com/lestrrat-go/jwx/v2 v2.0.19 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 @@ -244,6 +248,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect @@ -265,6 +270,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect + github.com/cilium/ebpf v0.12.3 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -279,6 +285,7 @@ require ( github.com/fearful-symmetry/gomsr v0.0.1 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect + github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -297,7 +304,6 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect github.com/googleapis/gnostic v0.5.5 // indirect - github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/cronexpr v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -321,7 +327,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kortschak/utter v1.5.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/lestrrat-go/blackmagic v1.0.1 // indirect + github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.4 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect diff --git a/go.sum b/go.sum index 79feea75570..746d5023ae9 100644 --- a/go.sum +++ b/go.sum @@ -178,6 +178,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU= github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= @@ -237,6 +239,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= +github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 h1:7rj9qZ63knnVo2ZeepYHvHuRdG76f3tRUTdIQDzRBeI= github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20/go.mod h1:cI59GRkC2FRaFYtgbYEqMlgnnfvAwXzjojyZKXwklNg= @@ -430,6 +434,8 @@ github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLI github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= +github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -572,7 +578,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 h1:qg9VbHo1TlL0KDM0vYvBG9EY0X0Yku5WYIPoFWt8f6o= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= -github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= @@ -658,10 +663,12 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= +github.com/elastic/ebpfevents v0.4.0 h1:M80eAeJnzvGQgU9cjJqkjFca9pjM3aq/TuZxJeom4bI= +github.com/elastic/ebpfevents v0.4.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g= github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= -github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= -github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM= +github.com/elastic/elastic-agent-client/v7 v7.8.0 h1:GHFzDJIWpdgI0qDk5EcqbQJGvwTsl2E2vQK3/xe+MYQ= +github.com/elastic/elastic-agent-client/v7 v7.8.0/go.mod h1:ihtjqJzYiIltlRhNruaSSc0ogxIhqPD5hOMKq16cI1s= github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkcunPlmTZn5yg= github.com/elastic/elastic-agent-libs v0.7.5/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= @@ -693,8 +700,8 @@ github.com/elastic/go-seccomp-bpf v1.4.0 h1:6y3lYrEHrLH9QzUgOiK8WDqmPaMnnB785Wxi github.com/elastic/go-seccomp-bpf v1.4.0/go.mod h1:wIMxjTbKpWGQk4CV9WltlG6haB4brjSH/dvAohBPM1I= github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w= github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= -github.com/elastic/go-sysinfo v1.11.2 h1:mcm4OSYVMyws6+n2HIVMGkln5HOpo5Ie1ZmbbNn0jg4= -github.com/elastic/go-sysinfo v1.11.2/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ= +github.com/elastic/go-sysinfo v1.13.1 h1:U5Jlx6c/rLkR72O8wXXXo1abnGlWGJU/wbzNJ2AfQa4= +github.com/elastic/go-sysinfo v1.13.1/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ= github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0= github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= @@ -703,12 +710,14 @@ github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a h1:8WfL/X6fK11 github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/mito v1.8.0 h1:i3GOtcnNuEEH2XMqnQdPvNjIBA8m0VKuTTfvusfCfnU= -github.com/elastic/mito v1.8.0/go.mod h1:n7AvUVtYQQXb8fq87FI8z67TNzuhwBV3kHBkDT1qJYQ= +github.com/elastic/mito v1.9.0 h1:gYB+0o5bhr5/XIlKuZOdeSFvWGTKpk0v73vsFRm98u8= +github.com/elastic/mito v1.9.0/go.mod h1:n7AvUVtYQQXb8fq87FI8z67TNzuhwBV3kHBkDT1qJYQ= github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3 h1:ChPwRVv1RR4a0cxoGjKcyWjTEpxYfm5gydMIzo32cAw= github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c= github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3 h1:FzA0/n4iMt8ojGDGRoiFPSHFvvdVIvxOxyLtiFnrLBM= github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3/go.mod h1:mdtqvCSg8JOxk8PmpTNGyo6wzd4BMm4QXSfDnTXmgkE= +github.com/elastic/tk-btf v0.1.0 h1:T4rbsnfaRH/MZKSLwZFd3sndt/NexsQb0IXWtMQ9PAA= +github.com/elastic/tk-btf v0.1.0/go.mod h1:caLQPEcMbyKmPUQb2AsbX3ZAj1yCz06lOxfhn0esLR8= github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 h1:oJbI/v6q/PDOZrsruajnbbt7mujobOPDUmkePcVMkJA= github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640/go.mod h1:C26fjgblYUZyl9aRc0D4piK8WqQzeCwUdIvjN5OsTnY= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -755,8 +764,8 @@ github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -767,7 +776,11 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= +github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-faker/faker/v4 v4.2.0 h1:dGebOupKwssrODV51E0zbMrv5e2gO9VWSLNC1WDCpWg= +github.com/go-faker/faker/v4 v4.2.0/go.mod h1:F/bBy8GH9NxOxMInug5Gx4WYeG6fHJZ8Ol/dhcpRub4= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= @@ -783,6 +796,8 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A= +github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -1341,8 +1356,9 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -1351,17 +1367,16 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80= -github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= +github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= +github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8= github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.0.11 h1:ViHMnaMeaO0qV16RZWBHM7GTrAnX2aFLVKofc7FuKLQ= -github.com/lestrrat-go/jwx/v2 v2.0.11/go.mod h1:ZtPtMFlrfDrH2Y0iwfa3dRFn8VzwBrB+cyrm3IBWdDg= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lestrrat-go/jwx/v2 v2.0.19 h1:ekv1qEZE6BVct89QA+pRF6+4pCpfVrOnEJnTnT4RXoY= +github.com/lestrrat-go/jwx/v2 v2.0.19/go.mod h1:l3im3coce1lL2cDeAjqmaR+Awx+X8Ih+2k8BuHNJ4CU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -2043,7 +2058,7 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2346,7 +2361,6 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2356,6 +2370,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2366,6 +2381,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2382,6 +2398,7 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2657,8 +2674,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index 335bf29256e..eb52ad4d130 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.6 +FROM golang:1.21.7 RUN \ apt-get update \ diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 2b2f28382e9..04df2d4dbcd 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1443,9 +1443,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/heartbeat/security/security.go b/heartbeat/security/security.go index 20c0f5cc7d6..8e15102f7b8 100644 --- a/heartbeat/security/security.go +++ b/heartbeat/security/security.go @@ -26,8 +26,6 @@ import ( "strconv" "syscall" - sysinfo "github.com/elastic/go-sysinfo" - "kernel.org/pub/linux/libs/security/libcap/cap" ) @@ -36,13 +34,7 @@ func init() { // In the context of a container, where users frequently run as root, we follow BEAT_SETUID_AS to setuid/gid // and add capabilities to make this actually run as a regular user. This also helps Node.js in synthetics, which // does not want to run as root. It's also just generally more secure. - sysInfo, err := sysinfo.Host() - isContainer := false - if err == nil && sysInfo.Info().Containerized != nil { - isContainer = *sysInfo.Info().Containerized - } - - if localUserName := os.Getenv("BEAT_SETUID_AS"); isContainer && localUserName != "" && syscall.Geteuid() == 0 { + if localUserName := os.Getenv("BEAT_SETUID_AS"); localUserName != "" && syscall.Geteuid() == 0 { err := setNodeProcAttr(localUserName) if err != nil { panic(err) diff --git a/libbeat/_meta/config/setup.ilm.reference.yml.tmpl b/libbeat/_meta/config/setup.ilm.reference.yml.tmpl index 296bf0872ef..6e2aed0b828 100644 --- a/libbeat/_meta/config/setup.ilm.reference.yml.tmpl +++ b/libbeat/_meta/config/setup.ilm.reference.yml.tmpl @@ -14,9 +14,9 @@ # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 3ba0db210de..c050ff8acd8 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -154,7 +154,7 @@ func ApplyConfigTemplate(event bus.Event, configs []*conf.C, options ...ucfg.Opt var unpacked map[string]interface{} err = c.Unpack(&unpacked, opts...) if err != nil { - logp.Warn("autodiscover: Configuration template cannot be resolved: %v", err) + logp.Debug("autodiscover", "Configuration template cannot be resolved: %v", err) continue } // Repack again: diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index efe8bd48f79..4b7470b1dbd 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -197,7 +197,7 @@ func initRand() { } else { seed = n.Int64() } - rand.Seed(seed) + rand.Seed(seed) //nolint:staticcheck // need seed from cryptographically strong PRNG. } // Run initializes and runs a Beater implementation. name is the name of the @@ -824,7 +824,10 @@ func (b *Beat) configure(settings Settings) error { return fmt.Errorf("failed to get host information: %w", err) } - fqdn, err := h.FQDN() + fqdnLookupCtx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + fqdn, err := h.FQDNWithContext(fqdnLookupCtx) if err != nil { // FQDN lookup is "best effort". We log the error, fallback to // the OS-reported hostname, and move on. @@ -835,10 +838,25 @@ func (b *Beat) configure(settings Settings) error { } // initialize config manager - b.Manager, err = management.NewManager(b.Config.Management, reload.RegisterV2) + m, err := management.NewManager(b.Config.Management, reload.RegisterV2) if err != nil { return err } + b.Manager = m + + if b.Manager.AgentInfo().Version != "" { + // During the manager initialization the client to connect to the agent is + // also initialized. That makes the beat to read information sent by the + // agent, which includes the AgentInfo with the agent's package version. + // Components running under agent should report the agent's package version + // as their own version. + // In order to do so b.Info.Version needs to be set to the version the agent + // sent. As this Beat instance is initialized much before the package + // version is received, it's overridden here. So far it's early enough for + // the whole beat to report the right version. + b.Info.Version = b.Manager.AgentInfo().Version + version.SetPackageVersion(b.Info.Version) + } if err := b.Manager.CheckRawConfig(b.RawConfig); err != nil { return err @@ -1518,13 +1536,13 @@ func (bc *beatConfig) Validate() error { if bc.Pipeline.Queue.IsSet() && outputPC.Queue.IsSet() { return fmt.Errorf("top level queue and output level queue settings defined, only one is allowed") } - //elastic-agent doesn't support disk queue yet + // elastic-agent doesn't support disk queue yet if bc.Management.Enabled() && outputPC.Queue.Config().Enabled() && outputPC.Queue.Name() == diskqueue.QueueType { return fmt.Errorf("disk queue is not supported when management is enabled") } } - //elastic-agent doesn't support disk queue yet + // elastic-agent doesn't support disk queue yet if bc.Management.Enabled() && bc.Pipeline.Queue.Config().Enabled() && bc.Pipeline.Queue.Name() == diskqueue.QueueType { return fmt.Errorf("disk queue is not supported when management is enabled") } diff --git a/libbeat/common/capabilities/capabilities_linux.go b/libbeat/common/capabilities/capabilities_linux.go new file mode 100644 index 00000000000..715b86d9bc7 --- /dev/null +++ b/libbeat/common/capabilities/capabilities_linux.go @@ -0,0 +1,161 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package capabilities + +import ( + "errors" + "math/bits" + "strconv" + "strings" + + "kernel.org/pub/linux/libs/security/libcap/cap" +) + +var ( + // errInvalidCapability expresses an invalid capability ID: x < 0 || x >= 64. + errInvalidCapability = errors.New("invalid capability") +) + +// The capability set flag/vector, re-exported from +// libcap(3). Inherit, Bound & Ambient not exported since we have no +// use for it yet. +type Flag = cap.Flag + +const ( + // aka CapEff + Effective = cap.Effective + // aka CapPrm + Permitted = cap.Permitted +) + +// Fetch the capabilities of pid for a given flag/vector and convert +// it to the representation used in ECS. cap.GetPID() fetches it with +// SYS_CAPGET. +// Returns errors.ErrUnsupported on "not linux". +func FromPid(flag Flag, pid int) ([]string, error) { + set, err := cap.GetPID(pid) + if err != nil { + return nil, err + } + empty, err := isEmpty(flag, set) + if err != nil { + return nil, err + } + if empty { + return []string{}, nil + } + + sl := make([]string, 0, cap.MaxBits()) + for i := 0; i < int(cap.MaxBits()); i++ { + c := cap.Value(i) + enabled, err := set.GetFlag(flag, c) + if err != nil { + return nil, err + } + if !enabled { + continue + } + s, err := toECS(i) + // impossible since MaxBits <= 64 + if err != nil { + return nil, err + } + sl = append(sl, s) + } + + return sl, err +} + +// Convert a uint64 to the capabilities representation used in ECS. +// Returns errors.ErrUnsupported on "not linux". +func FromUint64(w uint64) ([]string, error) { + sl := make([]string, 0, bits.OnesCount64(w)) + for i := 0; w != 0; i++ { + if w&1 != 0 { + s, err := toECS(i) + // impossible since MaxBits <= 64 + if err != nil { + return nil, err + } + sl = append(sl, s) + } + w >>= 1 + } + + return sl, nil +} + +// Convert a string to the capabilities representation used in +// ECS. Example input: "1ffffffffff", 16. +// Returns errors.ErrUnsupported on "not linux". +func FromString(s string, base int) ([]string, error) { + w, err := strconv.ParseUint(s, 16, 64) + if err != nil { + return nil, err + } + + return FromUint64(w) +} + +// True if sets are equal for the given flag/vector, errors out in +// case any of the sets is malformed. +func isEqual(flag Flag, a *cap.Set, b *cap.Set) (bool, error) { + d, err := a.Cf(b) + if err != nil { + return false, err + } + + return !d.Has(flag), nil +} + +// Convert the capability ID to a string suitable to be used in +// ECS. +// If capabiliy ID X is unknown, but valid (0 <= X < 64), "CAP_X" +// will be returned instead. Fetches from an internal table built at +// startup. +var toECS = makeToECS() + +// Make toECS() which creates a map of every possible valid capability +// ID on startup. Returns errInvalidCapabilty for an invalid ID. +func makeToECS() func(int) (string, error) { + ecsNames := make(map[int]string) + + for i := 0; i < 64; i++ { + c := cap.Value(i) + if i < int(cap.MaxBits()) { + ecsNames[i] = strings.ToUpper(c.String()) + } else { + ecsNames[i] = strings.ToUpper("CAP_" + c.String()) + } + } + + return func(b int) (string, error) { + s, ok := ecsNames[b] + if !ok { + return "", errInvalidCapability + } + return s, nil + } +} + +// Like isAll(), but for the empty set, here for symmetry. +func isEmpty(flag Flag, set *cap.Set) (bool, error) { + return isEqual(flag, set, cap.NewSet()) +} diff --git a/libbeat/common/capabilities/capabilities_linux_test.go b/libbeat/common/capabilities/capabilities_linux_test.go new file mode 100644 index 00000000000..1481fc5679b --- /dev/null +++ b/libbeat/common/capabilities/capabilities_linux_test.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package capabilities + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "kernel.org/pub/linux/libs/security/libcap/cap" +) + +func TestEmpty(t *testing.T) { + sl, err := FromString("0", 16) + assert.Nil(t, err) + assert.Equal(t, len(sl), 0) + + sl, err = FromUint64(0) + assert.Nil(t, err) + assert.Equal(t, len(sl), 0) + + // assumes non root has no capabilities + if os.Geteuid() != 0 { + empty := cap.NewSet() + self := cap.GetProc() + d, err := self.Cf(empty) + assert.Nil(t, err) + assert.False(t, d.Has(cap.Effective)) + assert.False(t, d.Has(cap.Permitted)) + assert.False(t, d.Has(cap.Inheritable)) + } +} + +func TestOverflow(t *testing.T) { + sl, err := FromUint64(^uint64(0)) + assert.Nil(t, err) + assert.Equal(t, len(sl), 64) + + for _, cap := range []string{ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_IPC_LOCK", + "CAP_MAC_OVERRIDE", + } { + assertHasCap(t, sl, cap) + } + if cap.MaxBits() <= 62 { + assertHasCap(t, sl, "CAP_62") + } + if cap.MaxBits() <= 63 { + assertHasCap(t, sl, "CAP_63") + } +} + +func assertHasCap(t *testing.T, sl []string, s string) { + var found int + + for _, s2 := range sl { + if s2 == s { + found++ + } + } + + assert.Equal(t, found, 1, s) +} diff --git a/libbeat/common/capabilities/capabilities_other.go b/libbeat/common/capabilities/capabilities_other.go new file mode 100644 index 00000000000..fbd7e879772 --- /dev/null +++ b/libbeat/common/capabilities/capabilities_other.go @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !linux + +package capabilities + +import "errors" + +// Dummy value on "not linux". +type Flag = uint + +const ( + // Meaningless on "not linux". + Effective = Flag(0) + // Meaningless on "not linux". + Permitted = Flag(1) +) + +// Returns errors.ErrUnsupported on "not linux". +func FromPid(flag Flag, pid int) ([]string, error) { + return nil, errors.ErrUnsupported +} + +// Returns errors.ErrUnsupported on "not linux". +func FromUint64(w uint64) ([]string, error) { + return nil, errors.ErrUnsupported +} + +// Returns errors.ErrUnsupported on "not linux". +func FromString(s string, base int) ([]string, error) { + return nil, errors.ErrUnsupported +} diff --git a/libbeat/common/capabilities_linux.go b/libbeat/common/capabilities_linux.go deleted file mode 100644 index b2992c251ef..00000000000 --- a/libbeat/common/capabilities_linux.go +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build linux - -package common - -import ( - "errors" - "fmt" - - "github.com/elastic/go-sysinfo" - "github.com/elastic/go-sysinfo/types" -) - -// Capabilities contains the capability sets of a process -type Capabilities types.CapabilityInfo - -// Check performs a permission check for a given capabilities set -func (c Capabilities) Check(set []string) bool { - for _, capability := range set { - found := false - for _, effective := range c.Effective { - if capability == effective { - found = true - break - } - } - if !found { - return false - } - } - return true -} - -// GetCapabilities gets the capabilities of this process -func GetCapabilities() (Capabilities, error) { - p, err := sysinfo.Self() - if err != nil { - return Capabilities{}, fmt.Errorf("failed to read self process information: %w", err) - } - - if c, ok := p.(types.Capabilities); ok { - capabilities, err := c.Capabilities() - if err != nil { - return Capabilities{}, fmt.Errorf("failed to read process capabilities: %w", err) - } - return Capabilities(*capabilities), nil - } - - return Capabilities{}, errors.New("capabilities not available") -} diff --git a/libbeat/common/seccomp/policy_linux_386.go b/libbeat/common/seccomp/policy_linux_386.go index 72466698720..ac2a93a5c74 100644 --- a/libbeat/common/seccomp/policy_linux_386.go +++ b/libbeat/common/seccomp/policy_linux_386.go @@ -31,6 +31,7 @@ func init() { "_llseek", "access", "brk", + "capget", "chmod", "chown", "clock_gettime", diff --git a/libbeat/common/seccomp/policy_linux_amd64.go b/libbeat/common/seccomp/policy_linux_amd64.go index 0a05bdde927..624f48c890a 100644 --- a/libbeat/common/seccomp/policy_linux_amd64.go +++ b/libbeat/common/seccomp/policy_linux_amd64.go @@ -34,6 +34,7 @@ func init() { "arch_prctl", "bind", "brk", + "capget", "chmod", "chown", "clock_gettime", diff --git a/libbeat/docs/queueconfig.asciidoc b/libbeat/docs/queueconfig.asciidoc index 08ece0f752f..499ab9d4667 100644 --- a/libbeat/docs/queueconfig.asciidoc +++ b/libbeat/docs/queueconfig.asciidoc @@ -32,20 +32,32 @@ The memory queue waits for the output to acknowledge or drop events. If the queue is full, no new events can be inserted into the memory queue. Only after the signal from the output will the queue free up space for more events to be accepted. -The memory queue is controlled by the parameters `flush.min_events` and `flush.timeout`. If -`flush.timeout` is `0s` or `flush.min_events` is `0` or `1` then events can be sent by the output as -soon as they are available. If the output supports a `bulk_max_size` parameter it controls the -maximum batch size that can be sent. +The memory queue is controlled by the parameters `flush.min_events` and `flush.timeout`. +`flush.min_events` gives a limit on the number of events that can be included in a +single batch, and `flush.timeout` specifies how long the queue should wait to completely +fill an event request. If the output supports a `bulk_max_size` parameter, the maximum +batch size will be the smaller of `bulk_max_size` and `flush.min_events`. -If `flush.min_events` is greater than `1` and `flush.timeout` is greater than `0s`, events will only -be sent to the output when the queue contains at least `flush.min_events` events or the -`flush.timeout` period has expired. In this mode the maximum size batch that that can be sent by the -output is `flush.min_events`. If the output supports a `bulk_max_size` parameter, values of -`bulk_max_size` greater than `flush.min_events` have no effect. The value of `flush.min_events` -should be evenly divisible by `bulk_max_size` to avoid sending partial batches to the output. +`flush.min_events` is a legacy parameter, and new configurations should prefer to control +batch size with `bulk_max_size`. As of 8.13, there is never a performance advantage to +limiting batch size with `flush.min_events` instead of `bulk_max_size`. -This sample configuration forwards events to the output if 512 events are available or the oldest -available event has been waiting for 5s in the queue: +In synchronous mode, an event request is always filled as soon as events are available, +even if there are not enough events to fill the requested batch. This is useful when +latency must be minimized. To use synchronous mode, set `flush.timeout` to 0. + +For backwards compatibility, synchronous mode can also be activated by setting `flush.min_events` +to 0 or 1. In this case, batch size will be capped at 1/2 the queue capacity. + +In asynchronous mode, an event request will wait up to the specified timeout to try +and fill the requested batch completely. If the timeout expires, the queue returns a +partial batch with all available events. To use asynchronous mode, set `flush.timeout` +to a positive duration, e.g. `5s`. + +This sample configuration forwards events to the output when there are enough events +to fill the output's request (usually controlled by `bulk_max_size`, and limited to at +most 512 events by `flush.min_events`), or when events have been waiting for 5s without +filling the requested size: [source,yaml] ------------------------------------------------------------------------------ @@ -64,8 +76,7 @@ You can specify the following options in the `queue.mem` section of the +{beatna [[queue-mem-events-option]] ===== `events` -Number of events the queue can store. This value should be evenly divisible by `flush.min_events` to -avoid sending partial batches to the output. +Number of events the queue can store. The default value is 3200 events. @@ -73,11 +84,13 @@ The default value is 3200 events. [[queue-mem-flush-min-events-option]] ===== `flush.min_events` -Minimum number of events required for publishing. If this value is set to 0 or 1, events are -available to the output immediately. If this value is greater than 1 the output must wait for the -queue to accumulate this minimum number of events or for `flush.timeout` to expire before -publishing. When greater than `1` this value also defines the maximum possible batch that can be -sent by the output. +If greater than 1, specifies the maximum number of events per batch. In this case the +output must wait for the +queue to accumulate the requested number of events or for `flush.timeout` to expire before +publishing. + +If 0 or 1, sets the maximum number of events per batch to half the queue size, and sets +the queue to synchronous mode (equivalent to `flush.timeout` of 0). The default value is 1600. @@ -85,8 +98,7 @@ The default value is 1600. [[queue-mem-flush-timeout-option]] ===== `flush.timeout` -Maximum wait time for `flush.min_events` to be fulfilled. If set to 0s, events are available to the -output immediately. +Maximum wait time for event requests from the output to be fulfilled. If set to 0s, events are returned immediately. The default value is 10s. diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index 47a6f1eaf23..08da0875d41 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 13456f9cd18..1cae4fe1ad8 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.13.0 :doc-branch: main -:go-version: 1.21.6 +:go-version: 1.21.7 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/libbeat/ebpf/seccomp_linux.go b/libbeat/ebpf/seccomp_linux.go new file mode 100644 index 00000000000..9059eb0f643 --- /dev/null +++ b/libbeat/ebpf/seccomp_linux.go @@ -0,0 +1,40 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package ebpf + +import ( + "runtime" + + "github.com/elastic/beats/v7/libbeat/common/seccomp" +) + +func init() { + switch runtime.GOARCH { + case "amd64": + syscalls := []string{ + "bpf", + "eventfd2", // needed by ringbuf + "perf_event_open", // needed by tracepoints + } + if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...); err != nil { + panic(err) + } + } +} diff --git a/libbeat/ebpf/watcher_linux.go b/libbeat/ebpf/watcher_linux.go new file mode 100644 index 00000000000..e0da448d87a --- /dev/null +++ b/libbeat/ebpf/watcher_linux.go @@ -0,0 +1,183 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package ebpf + +import ( + "context" + "fmt" + "sync" + + "github.com/elastic/ebpfevents" +) + +var ( + gWatcherOnce sync.Once + gWatcher Watcher +) + +type client struct { + name string + mask EventMask + records chan ebpfevents.Record +} + +// EventMask is a mask of ebpfevents.EventType which is used to control which event types clients will receive. +type EventMask uint64 + +// Watcher observes kernel events, using ebpf probes from the ebpfevents library, and sends the +// events to subscribing clients. +// +// A single global watcher can exist, and can deliver events to multiple clients. Clients subscribe +// to the watcher, and all ebpf events that match their mask will be sent to their channel. +type Watcher struct { + sync.Mutex + cancel context.CancelFunc + loader *ebpfevents.Loader + clients map[string]client + status status + err error +} + +type status int + +const ( + stopped status = iota + started +) + +// GetWatcher creates the watcher, if required, and returns a reference to the global Watcher. +func GetWatcher() (*Watcher, error) { + gWatcher.Lock() + defer gWatcher.Unlock() + + // Try to load the probe once on startup so consumers can error out. + gWatcherOnce.Do(func() { + if gWatcher.status == stopped { + l, err := ebpfevents.NewLoader() + if err != nil { + gWatcher.err = fmt.Errorf("init ebpf loader: %w", err) + return + } + _ = l.Close() + } + }) + + return &gWatcher, gWatcher.err +} + +// Subscribe to receive events from the watcher. +func (w *Watcher) Subscribe(clientName string, events EventMask) <-chan ebpfevents.Record { + w.Lock() + defer w.Unlock() + + if w.status == stopped { + w.startLocked() + } + + w.clients[clientName] = client{ + name: clientName, + mask: events, + records: make(chan ebpfevents.Record, w.loader.BufferLen()), + } + + return w.clients[clientName].records +} + +// Unsubscribe the client with the given name. +func (w *Watcher) Unsubscribe(clientName string) { + w.Lock() + defer w.Unlock() + + delete(w.clients, clientName) + + if w.nclients() == 0 { + w.stopLocked() + } +} + +func (w *Watcher) startLocked() { + if w.status == started { + return + } + + loader, err := ebpfevents.NewLoader() + if err != nil { + w.err = fmt.Errorf("start ebpf loader: %w", err) + return + } + + w.loader = loader + w.clients = make(map[string]client) + + records := make(chan ebpfevents.Record, loader.BufferLen()) + var ctx context.Context + ctx, w.cancel = context.WithCancel(context.Background()) + + go w.loader.EventLoop(ctx, records) + go func(ctx context.Context) { + for { + select { + case record := <-records: + if record.Error != nil { + for _, client := range w.clients { + client.records <- record + } + continue + } + for _, client := range w.clients { + if client.mask&EventMask(record.Event.Type) != 0 { + client.records <- record + } + } + continue + case <-ctx.Done(): + return + } + } + }(ctx) + + w.status = started +} + +func (w *Watcher) stopLocked() { + if w.status == stopped { + return + } + w.close() + w.status = stopped +} + +func (w *Watcher) nclients() int { + return len(w.clients) +} + +func (w *Watcher) close() { + if w.cancel != nil { + w.cancel() + } + + if w.loader != nil { + _ = w.loader.Close() + } + + for _, cl := range w.clients { + close(cl.records) + } +} diff --git a/libbeat/ebpf/watcher_test.go b/libbeat/ebpf/watcher_test.go new file mode 100644 index 00000000000..13d27ffd52c --- /dev/null +++ b/libbeat/ebpf/watcher_test.go @@ -0,0 +1,61 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package ebpf + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +const allEvents = EventMask(math.MaxUint64) + +func TestWatcherStartStop(t *testing.T) { + w, err := GetWatcher() + if err != nil { + t.Skipf("skipping ebpf watcher test: %v", err) + } + assert.Equal(t, gWatcher.status, stopped) + assert.Equal(t, 0, gWatcher.nclients()) + + _ = w.Subscribe("test-1", allEvents) + assert.Equal(t, gWatcher.status, started) + assert.Equal(t, 1, gWatcher.nclients()) + + _ = w.Subscribe("test-2", allEvents) + assert.Equal(t, 2, gWatcher.nclients()) + + w.Unsubscribe("test-2") + assert.Equal(t, 1, gWatcher.nclients()) + + w.Unsubscribe("dummy") + assert.Equal(t, 1, gWatcher.nclients()) + + assert.Equal(t, gWatcher.status, started) + w.Unsubscribe("test-1") + assert.Equal(t, 0, gWatcher.nclients()) + assert.Equal(t, gWatcher.status, stopped) + + _ = w.Subscribe("new", allEvents) + assert.Equal(t, 1, gWatcher.nclients()) + assert.Equal(t, gWatcher.status, started) + w.Unsubscribe("new") +} diff --git a/libbeat/idxmgmt/lifecycle/config.go b/libbeat/idxmgmt/lifecycle/config.go index b25d5ecff6d..5d1f4e9435b 100644 --- a/libbeat/idxmgmt/lifecycle/config.go +++ b/libbeat/idxmgmt/lifecycle/config.go @@ -33,9 +33,9 @@ type Config struct { // used only for testing policyRaw *Policy - // CheckExists can disable the check for an existing policy. Check required - // read_ilm privileges. If check is disabled the policy will only be - // installed if Overwrite is enabled. + // CheckExists can disable the check for an existing policy. This check + // requires read_ilm privileges. If CheckExists is set to false, the policy + // will not be installed, even if Overwrite is enabled. CheckExists bool `config:"check_exists"` // Enable always overwrite policy mode. This required manage_ilm privileges. diff --git a/libbeat/idxmgmt/lifecycle/mock_client_handler.go b/libbeat/idxmgmt/lifecycle/mock_client_handler_test.go similarity index 100% rename from libbeat/idxmgmt/lifecycle/mock_client_handler.go rename to libbeat/idxmgmt/lifecycle/mock_client_handler_test.go diff --git a/libbeat/publisher/agent.go b/libbeat/management/agent.go similarity index 84% rename from libbeat/publisher/agent.go rename to libbeat/management/agent.go index e1551dd8cbd..84f5a1e9f6b 100644 --- a/libbeat/publisher/agent.go +++ b/libbeat/management/agent.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package publisher +package management import ( "github.com/elastic/beats/v7/libbeat/common/atomic" @@ -47,14 +47,15 @@ func UnderAgent() bool { return underAgent.Load() } -// LogWithTrace returns true when not running under Elastic Agent always or -// only true if running under Elastic Agent with trace logging enabled. -func LogWithTrace() bool { - agent := underAgent.Load() - if agent { - trace := underAgentTrace.Load() - return trace +// TraceLevelEnabled returns true when the "trace log level" is enabled. +// +// It always returns true when not running under Elastic Agent. +// Otherwise it returns true when the trace level is enabled +func TraceLevelEnabled() bool { + if underAgent.Load() { + return underAgentTrace.Load() } + // Always true when not running under the Elastic Agent. return true } diff --git a/libbeat/management/management.go b/libbeat/management/management.go index 88faa48f540..177642b3398 100644 --- a/libbeat/management/management.go +++ b/libbeat/management/management.go @@ -82,9 +82,12 @@ type Manager interface { // // Calls to 'CheckRawConfig()' or 'SetPayload()' will be ignored after calling stop. // - // Note: Stop will not call 'UnregisterAction()' automaticallty. + // Note: Stop will not call 'UnregisterAction()' automatically. Stop() + // AgentInfo returns the information of the agent to which the manager is connected. + AgentInfo() client.AgentInfo + // SetStopCallback accepts a function that need to be called when the manager want to shutdown the // beats. This is needed when you want your beats to be gracefully shutdown remotely by the Elastic Agent // when a policy doesn't need to run this beat. @@ -190,6 +193,7 @@ func (n *fallbackManager) Stop() { // but that does not mean the Beat is being managed externally, // hence it will always return false. func (n *fallbackManager) Enabled() bool { return false } +func (n *fallbackManager) AgentInfo() client.AgentInfo { return client.AgentInfo{} } func (n *fallbackManager) Start() error { return nil } func (n *fallbackManager) CheckRawConfig(cfg *config.C) error { return nil } func (n *fallbackManager) RegisterAction(action client.Action) {} diff --git a/libbeat/outputs/kafka/kafka.go b/libbeat/outputs/kafka/kafka.go index 0c856ea425d..d004bd16ba3 100644 --- a/libbeat/outputs/kafka/kafka.go +++ b/libbeat/outputs/kafka/kafka.go @@ -18,9 +18,12 @@ package kafka import ( + "fmt" + "github.com/Shopify/sarama" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/codec" "github.com/elastic/beats/v7/libbeat/outputs/outil" @@ -84,7 +87,27 @@ func makeKafka( return outputs.Success(kConfig.Queue, kConfig.BulkMaxSize, retry, client) } +// buildTopicSelector builds the topic selector for standalone Beat and when +// running under Elastic-Agent based on cfg. +// +// When running standalone the topic selector works as expected and documented. +// When running under Elastic-Agent, dynamic topic selection is not supported, +// so a constant selector using the `topic` value is returned. func buildTopicSelector(cfg *config.C) (outil.Selector, error) { + topicCfg := struct { + Topic string `config:"topic" yaml:"topic"` + }{} + + if err := cfg.Unpack(&topicCfg); err != nil { + return outil.Selector{}, fmt.Errorf("cannot unpack Kafka config to read the topic: %w", err) + } + + if management.UnderAgent() { + exprSelector := outil.ConstSelectorExpr(topicCfg.Topic, outil.SelectorKeepCase) + selector := outil.MakeSelector(exprSelector) + return selector, nil + } + return outil.BuildSelectorFromConfig(cfg, outil.Settings{ Key: "topic", MultiKey: "topics", diff --git a/libbeat/outputs/kafka/kafka_test.go b/libbeat/outputs/kafka/kafka_test.go new file mode 100644 index 00000000000..0717f8a0d51 --- /dev/null +++ b/libbeat/outputs/kafka/kafka_test.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kafka + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestBuildTopicSelector(t *testing.T) { + testCases := []struct { + name string + topic string + expected string + underAgent bool + }{ + { + name: "static topic", + topic: "a test", + expected: "a test", + underAgent: true, + }, + { + name: "dynamic topic under agent", + topic: "%{[foo]}", + expected: "%{[foo]}", + underAgent: true, + }, + { + name: "dynamic topic standalone", + topic: "%{[foo]}", + expected: "bar", + underAgent: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + topicCfg := struct { + Topic string `config:"topic" yaml:"topic"` + }{ + Topic: tc.topic, + } + + configC := config.MustNewConfigFrom(topicCfg) + if tc.underAgent { + previous := management.UnderAgent() + management.SetUnderAgent(true) + defer management.SetUnderAgent(previous) + } + + selector, err := buildTopicSelector(configC) + if err != nil { + t.Fatalf("could not build topic selector: %s", err) + } + + event := beat.Event{Fields: mapstr.M{"foo": "bar"}} + topic, err := selector.Select(&event) + if err != nil { + t.Fatalf("could not use selector: %s", err) + } + + if topic != tc.expected { + t.Fatalf("expecting topic to be '%s', got '%s' instead", tc.expected, topic) + } + }) + } + + t.Run("fail unpacking config", func(t *testing.T) { + _, err := buildTopicSelector(nil) + if err == nil { + t.Error("unpack must fail with a nil *config.C") + } + }) +} diff --git a/libbeat/outputs/util.go b/libbeat/outputs/util.go index ce8765b5c2e..cab6b99aebe 100644 --- a/libbeat/outputs/util.go +++ b/libbeat/outputs/util.go @@ -20,7 +20,7 @@ package outputs import ( "fmt" - "github.com/elastic/beats/v7/libbeat/publisher" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/publisher/queue" "github.com/elastic/beats/v7/libbeat/publisher/queue/diskqueue" "github.com/elastic/beats/v7/libbeat/publisher/queue/memqueue" @@ -46,7 +46,7 @@ func Success(cfg config.Namespace, batchSize, retry int, clients ...Client) (Gro } q = memqueue.FactoryForSettings(settings) case diskqueue.QueueType: - if publisher.UnderAgent() { + if management.UnderAgent() { return Group{}, fmt.Errorf("disk queue not supported under agent") } settings, err := diskqueue.SettingsForUserConfig(cfg.Config()) diff --git a/libbeat/processors/actions/append.go b/libbeat/processors/actions/append.go index e7429449db4..1bf2caad45f 100644 --- a/libbeat/processors/actions/append.go +++ b/libbeat/processors/actions/append.go @@ -21,10 +21,10 @@ import ( "fmt" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" - "github.com/elastic/beats/v7/libbeat/publisher" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -82,7 +82,7 @@ func (f *appendProcessor) Run(event *beat.Event) (*beat.Event, error) { err := f.appendValues(f.config.TargetField, f.config.Fields, f.config.Values, event) if err != nil { errMsg := fmt.Errorf("failed to append fields in append processor: %w", err) - if publisher.LogWithTrace() { + if management.TraceLevelEnabled() { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { diff --git a/libbeat/processors/actions/copy_fields.go b/libbeat/processors/actions/copy_fields.go index 373c939388c..0f4fab309a3 100644 --- a/libbeat/processors/actions/copy_fields.go +++ b/libbeat/processors/actions/copy_fields.go @@ -22,10 +22,10 @@ import ( "fmt" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" - "github.com/elastic/beats/v7/libbeat/publisher" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -79,7 +79,7 @@ func (f *copyFields) Run(event *beat.Event) (*beat.Event, error) { err := f.copyField(field.From, field.To, event) if err != nil { errMsg := fmt.Errorf("Failed to copy fields in copy_fields processor: %w", err) - if publisher.LogWithTrace() { + if management.TraceLevelEnabled() { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { diff --git a/libbeat/processors/actions/decode_base64_field.go b/libbeat/processors/actions/decode_base64_field.go index fdfd038b05d..c45166beb11 100644 --- a/libbeat/processors/actions/decode_base64_field.go +++ b/libbeat/processors/actions/decode_base64_field.go @@ -24,10 +24,10 @@ import ( "strings" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" - "github.com/elastic/beats/v7/libbeat/publisher" cfg "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -84,7 +84,7 @@ func (f *decodeBase64Field) Run(event *beat.Event) (*beat.Event, error) { err := f.decodeField(event) if err != nil { errMsg := fmt.Errorf("failed to decode base64 fields in processor: %w", err) - if publisher.LogWithTrace() { + if management.TraceLevelEnabled() { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { diff --git a/libbeat/processors/actions/decompress_gzip_field.go b/libbeat/processors/actions/decompress_gzip_field.go index cdd73185420..8d463600c21 100644 --- a/libbeat/processors/actions/decompress_gzip_field.go +++ b/libbeat/processors/actions/decompress_gzip_field.go @@ -25,9 +25,9 @@ import ( "io" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" - "github.com/elastic/beats/v7/libbeat/publisher" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -76,7 +76,7 @@ func (f *decompressGzipField) Run(event *beat.Event) (*beat.Event, error) { err := f.decompressGzipField(event) if err != nil { errMsg := fmt.Errorf("Failed to decompress field in decompress_gzip_field processor: %w", err) - if publisher.LogWithTrace() { + if management.TraceLevelEnabled() { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { diff --git a/libbeat/processors/actions/rename.go b/libbeat/processors/actions/rename.go index 15d2627540b..4c49174bf54 100644 --- a/libbeat/processors/actions/rename.go +++ b/libbeat/processors/actions/rename.go @@ -22,10 +22,10 @@ import ( "fmt" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" - "github.com/elastic/beats/v7/libbeat/publisher" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -84,7 +84,7 @@ func (f *renameFields) Run(event *beat.Event) (*beat.Event, error) { err := f.renameField(field.From, field.To, event) if err != nil { errMsg := fmt.Errorf("Failed to rename fields in processor: %w", err) - if publisher.LogWithTrace() { + if management.TraceLevelEnabled() { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { diff --git a/libbeat/processors/actions/replace.go b/libbeat/processors/actions/replace.go index 07e5ba0fcc0..df4aa03fc86 100644 --- a/libbeat/processors/actions/replace.go +++ b/libbeat/processors/actions/replace.go @@ -23,10 +23,10 @@ import ( "regexp" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" - "github.com/elastic/beats/v7/libbeat/publisher" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -86,7 +86,7 @@ func (f *replaceString) Run(event *beat.Event) (*beat.Event, error) { err := f.replaceField(field.Field, field.Pattern, field.Replacement, event) if err != nil { errMsg := fmt.Errorf("Failed to replace fields in processor: %w", err) - if publisher.LogWithTrace() { + if management.TraceLevelEnabled() { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { diff --git a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc index 9e61cac2e8c..c6dbdd5600a 100644 --- a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc +++ b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc @@ -21,6 +21,11 @@ The following cloud providers are supported: - Openstack Nova - Hetzner Cloud +NOTE: `huawei` is an alias for `openstack`. Huawei cloud runs on OpenStack platform, and when +viewed from a metadata API standpoint, it is impossible to differentiate it from OpenStack. If you know that your +deployments run on Huawei Cloud exclusively, and you wish to have `cloud.provider` value as `huawei`, you can achieve +this by overwriting the value using an `add_fields` processor. + The Alibaba Cloud and Tencent cloud providers are disabled by default, because they require to access a remote host. The `providers` setting allows users to select a list of default providers to query. @@ -53,10 +58,9 @@ List of names the `providers` setting supports: - "digitalocean" for Digital Ocean (enabled by default). - "aws", or "ec2" for Amazon Web Services (enabled by default). - "gcp" for Google Copmute Enging (enabled by default). -- "openstack", or "nova" for Openstack Nova (enabled by default). +- "openstack", "nova", or "huawei" for Openstack Nova (enabled by default). - "openstack-ssl", or "nova-ssl" for Openstack Nova when SSL metadata APIs are enabled (enabled by default). - "tencent", or "qcloud" for Tencent Cloud (disabled by default). -- "huawei" for Huawei Cloud (enabled by default). - "hetzner" for Hetzner Cloud (enabled by default). The third optional configuration setting is `overwrite`. When `overwrite` is @@ -128,20 +132,6 @@ _Tencent Cloud_ } ------------------------------------------------------------------------------- -_Huawei Cloud_ - -[source,json] -------------------------------------------------------------------------------- -{ - "cloud": { - "availability_zone": "cn-east-2b", - "instance.id": "37da9890-8289-4c58-ba34-a8271c4a8216", - "provider": "huawei", - "region": "cn-east-2" - } -} -------------------------------------------------------------------------------- - _Alibaba Cloud_ This metadata is only available when VPC is selected as the network type of the diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go index 1f428372b86..ea945ce4bba 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go @@ -59,10 +59,16 @@ var ec2MetadataFetcher = provider{ Create: func(_ string, config *conf.C) (metadataFetcher, error) { ec2Schema := func(m map[string]interface{}) mapstr.M { - m["service"] = mapstr.M{ - "name": "EC2", + meta := mapstr.M{ + "cloud": mapstr.M{ + "service": mapstr.M{ + "name": "EC2", + }, + }, } - return mapstr.M{"cloud": m} + + meta.DeepUpdate(m) + return meta } fetcher, err := newGenericMetadataFetcher(config, "aws", ec2Schema, fetchRawProviderMetadata) @@ -109,12 +115,12 @@ func fetchRawProviderMetadata( _, _ = result.metadata.Put("orchestrator.cluster.name", clusterName) } - _, _ = result.metadata.Put("instance.id", instanceIdentity.InstanceIdentityDocument.InstanceID) - _, _ = result.metadata.Put("machine.type", instanceIdentity.InstanceIdentityDocument.InstanceType) - _, _ = result.metadata.Put("region", awsRegion) - _, _ = result.metadata.Put("availability_zone", instanceIdentity.InstanceIdentityDocument.AvailabilityZone) - _, _ = result.metadata.Put("account.id", accountID) - _, _ = result.metadata.Put("image.id", instanceIdentity.InstanceIdentityDocument.ImageID) + _, _ = result.metadata.Put("cloud.instance.id", instanceIdentity.InstanceIdentityDocument.InstanceID) + _, _ = result.metadata.Put("cloud.machine.type", instanceIdentity.InstanceIdentityDocument.InstanceType) + _, _ = result.metadata.Put("cloud.region", awsRegion) + _, _ = result.metadata.Put("cloud.availability_zone", instanceIdentity.InstanceIdentityDocument.AvailabilityZone) + _, _ = result.metadata.Put("cloud.account.id", accountID) + _, _ = result.metadata.Put("cloud.image.id", instanceIdentity.InstanceIdentityDocument.ImageID) } diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go index 229902b9612..76ddea084a7 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go @@ -35,6 +35,14 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +func init() { + // Disable IMDS when the real AWS SDK IMDS client is used, + // so tests are isolated from the environment. Otherwise, + // tests for non-EC2 providers may fail when the tests are + // run within an EC2 VM. + os.Setenv("AWS_EC2_METADATA_DISABLED", "true") +} + type MockIMDSClient struct { GetInstanceIdentityDocumentFunc func(ctx context.Context, params *imds.GetInstanceIdentityDocumentInput, optFns ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error) } @@ -153,11 +161,11 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "region": regionDoc1, "availability_zone": availabilityZoneDoc1, "service": mapstr.M{"name": "EC2"}, - "orchestrator": mapstr.M{ - "cluster": mapstr.M{ - "name": clusterNameValue, - "id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue), - }, + }, + "orchestrator": mapstr.M{ + "cluster": mapstr.M{ + "name": clusterNameValue, + "id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue), }, }, }, @@ -198,6 +206,12 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "cloud": mapstr.M{ "instance": mapstr.M{"id": instanceIDDoc2}, }, + "orchestrator": mapstr.M{ + "cluster": mapstr.M{ + "name": clusterNameValue, + "id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue), + }, + }, }, }, { @@ -244,11 +258,11 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "region": regionDoc1, "availability_zone": availabilityZoneDoc1, "service": mapstr.M{"name": "EC2"}, - "orchestrator": mapstr.M{ - "cluster": mapstr.M{ - "name": clusterNameValue, - "id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue), - }, + }, + "orchestrator": mapstr.M{ + "cluster": mapstr.M{ + "name": clusterNameValue, + "id": fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", regionDoc1, accountIDDoc1, clusterNameValue), }, }, }, diff --git a/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go index 3cc089a4f0b..60737d22c61 100644 --- a/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go @@ -31,7 +31,7 @@ import ( ) func hetznerMetadataHandler() http.HandlerFunc { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { if r.RequestURI == hetznerMetadataInstanceIDURI { _, _ = w.Write([]byte("111111")) return @@ -50,7 +50,7 @@ func hetznerMetadataHandler() http.HandlerFunc { } http.Error(w, "not found", http.StatusNotFound) - }) + } } func TestRetrieveHetznerMetadata(t *testing.T) { diff --git a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud.go b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud.go deleted file mode 100644 index 36683e74a13..00000000000 --- a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud.go +++ /dev/null @@ -1,81 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package add_cloud_metadata - -import ( - "encoding/json" - - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -type hwMeta struct { - ImageName string `json:"image_name"` - VpcID string `json:"vpc_id"` -} - -type hwMetadata struct { - UUID string `json:"uuid"` - AvailabilityZone string `json:"availability_zone"` - RegionID string `json:"region_id"` - Meta *hwMeta `json:"meta"` - ProjectID string `json:"project_id"` - Name string `json:"name"` -} - -// Huawei Cloud Metadata Service -// Document https://support.huaweicloud.com/usermanual-ecs/ecs_03_0166.html -var huaweiMetadataFetcher = provider{ - Name: "huawei-cloud", - - Local: true, - - Create: func(_ string, c *conf.C) (metadataFetcher, error) { - metadataHost := "169.254.169.254" - huaweiCloudMetadataJSONURI := "/openstack/latest/meta_data.json" - - huaweiCloudSchema := func(m map[string]interface{}) mapstr.M { - m["service"] = mapstr.M{ - "name": "ECS", - } - return mapstr.M{"cloud": m} - } - - urls, err := getMetadataURLs(c, metadataHost, []string{ - huaweiCloudMetadataJSONURI, - }) - if err != nil { - return nil, err - } - responseHandlers := map[string]responseHandler{ - urls[0]: func(all []byte, result *result) error { - data := new(hwMetadata) - err := json.Unmarshal(all, data) - if err != nil { - return err - } - result.metadata.Put("instance.id", data.UUID) - result.metadata.Put("region", data.RegionID) - result.metadata.Put("availability_zone", data.AvailabilityZone) - return nil - }, - } - fetcher := &httpMetadataFetcher{"huawei", nil, responseHandlers, huaweiCloudSchema} - return fetcher, nil - }, -} diff --git a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go deleted file mode 100644 index 0ae6fc332f0..00000000000 --- a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package add_cloud_metadata - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/beat" - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -func initHuaweiCloudTestServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/openstack/latest/meta_data.json" { - w.Write([]byte(`{ - "random_seed": "CWIZtYK4y5pzMtShTtCKx16qB1DsA/2kL0US4u1fHxedODNr7gos4RgdE/z9eHucnltnlJfDY1remfGL60yzTsvEIWPdECOpPaJm1edIYQaUvQzdeQwKcOQAHjUP5wLQzGA3j3Pw10p7u+M7glHEwNRoEY1WsbVYwzyOOkBnqb+MJ1aOhiRnfNtHOxjLNBSDvjHaQZzoHL+1YNAxDYFezE83nE2m3ciVwZO7xWpdKDQ+W5hYBUsYAWODRMOYqIR/5ZLsfAfxE2DhK+NvuMyJ5yjO+ObQf0DN5nRUSrM5ajs84UVMr9ylJuT78ckh83CLSttsjzXJ+sr07ZFsB6/6NABzziFL7Xn8z/mEBVmFXBiBgg7KcWSoH756w42VSdUezwTy9lW0spRmdvNBKV/PzrYyy0FMiGXXZwMOCyBD05CBRJlsPorwxZLlfRVmNvsTuMYB8TG3UUbFhoR8Bd5en+EC3ncH3QIUDWn0oVg28BVjWe5rADVQLX1h83ti6GD08YUGaxoNPXnJLZfiaucSacby2mG31xysxd8Tg0qPRq7744a1HPVryuauWR9pF0+qDmtskhenxK0FR+TQ4w0fRxTigteBsXx1pQu0iz+B8rP68uokU2faCC2IMHY2Tf9RPCe6Eef0/DdQhBft88PuJLwq52o/0qZ/n9HFL6LdgCU=", - "uuid": "37da9890-8289-4c58-ba34-a8271c4a8216", - "availability_zone": "cn-east-2b", - "enterprise_project_id": "0", - "launch_index": 0, - "instance_type": "c3.large.2", - "meta": { - "os_bit": "64", - "image_name": "CentOS 7.4", - "vpc_id": "6dad7f50-db1d-4cce-b095-d27bc837d4bb" - }, - "region_id": "cn-east-2", - "project_id": "c09b8baf28b845a9b53ed37575cfd61f", - "name": "hwdev-test-1" - }`)) - return - } - - http.Error(w, "not found", http.StatusNotFound) - })) -} - -func TestRetrieveHuaweiCloudMetadata(t *testing.T) { - logp.TestingSetup() - - server := initHuaweiCloudTestServer() - defer server.Close() - - config, err := conf.NewConfigFrom(map[string]interface{}{ - "providers": []string{"huawei"}, - "host": server.Listener.Addr().String(), - }) - - if err != nil { - t.Fatal(err) - } - - p, err := New(config) - if err != nil { - t.Fatal(err) - } - - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } - - expected := mapstr.M{ - "cloud": mapstr.M{ - "provider": "huawei", - "instance": mapstr.M{ - "id": "37da9890-8289-4c58-ba34-a8271c4a8216", - }, - "region": "cn-east-2", - "availability_zone": "cn-east-2b", - "service": mapstr.M{ - "name": "ECS", - }, - }, - } - assert.Equal(t, expected, actual.Fields) -} diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go index 09c52d15066..feacd5acc3e 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go @@ -31,26 +31,26 @@ import ( ) func openstackNovaMetadataHandler() http.HandlerFunc { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { if r.RequestURI == osMetadataInstanceIDURI { - w.Write([]byte("i-0000ffac")) + _, _ = w.Write([]byte("i-0000ffac")) return } if r.RequestURI == osMetadataInstanceTypeURI { - w.Write([]byte("m1.xlarge")) + _, _ = w.Write([]byte("m1.xlarge")) return } if r.RequestURI == osMetadataHostnameURI { - w.Write([]byte("testvm01.stack.cloud")) + _, _ = w.Write([]byte("testvm01.stack.cloud")) return } if r.RequestURI == osMetadataZoneURI { - w.Write([]byte("az-test-2")) + _, _ = w.Write([]byte("az-test-2")) return } http.Error(w, "not found", http.StatusNotFound) - }) + } } func TestRetrieveOpenstackNovaMetadata(t *testing.T) { diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go index 304fc492e20..2e067fc09c4 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go @@ -33,15 +33,15 @@ import ( func initQCloudTestServer() *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.RequestURI == "/meta-data/instance-id" { - w.Write([]byte("ins-qcloudv5")) + _, _ = w.Write([]byte("ins-qcloudv5")) return } if r.RequestURI == "/meta-data/placement/region" { - w.Write([]byte("china-south-gz")) + _, _ = w.Write([]byte("china-south-gz")) return } if r.RequestURI == "/meta-data/placement/zone" { - w.Write([]byte("gz-azone2")) + _, _ = w.Write([]byte("gz-azone2")) return } diff --git a/libbeat/processors/add_cloud_metadata/providers.go b/libbeat/processors/add_cloud_metadata/providers.go index 55e68f75607..77c4c7042ad 100644 --- a/libbeat/processors/add_cloud_metadata/providers.go +++ b/libbeat/processors/add_cloud_metadata/providers.go @@ -64,7 +64,7 @@ var cloudMetaProviders = map[string]provider{ "nova-ssl": openstackNovaSSLMetadataFetcher, "qcloud": qcloudMetadataFetcher, "tencent": qcloudMetadataFetcher, - "huawei": huaweiMetadataFetcher, + "huawei": openstackNovaMetadataFetcher, "hetzner": hetznerMetadataFetcher, } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index db3cbbc5ee3..5fe28194b55 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -18,6 +18,7 @@ package add_host_metadata import ( + "context" "fmt" "sync" "time" @@ -25,6 +26,7 @@ import ( "github.com/gofrs/uuid" "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/go-sysinfo" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/features" @@ -35,7 +37,6 @@ import ( "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-system-metrics/metric/system/host" - "github.com/elastic/go-sysinfo" ) const processorName = "add_host_metadata" @@ -96,7 +97,7 @@ func New(cfg *config.C) (beat.Processor, error) { } // create a unique ID for this instance of the processor - cbIDStr := "" + var cbIDStr string cbID, err := uuid.NewV4() // if we fail, fall back to the processor name, hope for the best. if err != nil { @@ -178,7 +179,10 @@ func (p *addHostMetadata) loadData(checkCache bool, useFQDN bool) error { hostname := h.Info().Hostname if useFQDN { - fqdn, err := h.FQDN() + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + fqdn, err := h.FQDNWithContext(ctx) if err != nil { // FQDN lookup is "best effort". If it fails, we monitor the failure, fallback to // the OS-reported hostname, and move on. diff --git a/libbeat/processors/cache/file_store.go b/libbeat/processors/cache/file_store.go index 1ab4ab21ae4..d3820600acf 100644 --- a/libbeat/processors/cache/file_store.go +++ b/libbeat/processors/cache/file_store.go @@ -185,6 +185,7 @@ func (c *fileStore) readState() { // through all the elements. If any survive the filter, we // were alive, otherwise delete the file. + c.log.Infow("reading state from file", "id", c.id, "path", c.path) dec := json.NewDecoder(f) for { var e CacheEntry @@ -209,6 +210,7 @@ func (c *fileStore) readState() { heap.Push(&c.expiries, &e) } + c.log.Infow("got state from file", "id", c.id, "entries", len(c.cache)) if len(c.cache) != 0 { return } @@ -243,6 +245,7 @@ func (c *fileStore) writeState(final bool) { if !c.dirty { return } + c.log.Infow("write state to file", "id", c.id, "path", c.path) if len(c.cache) == 0 && final { err := os.Remove(c.path) if err != nil { @@ -278,6 +281,7 @@ func (c *fileStore) writeState(final bool) { if err != nil { c.log.Errorw("failed to finalize writing state", "error", err) } + c.log.Infow("write state to file sync and replace succeeded", "id", c.id, "path", c.path) }() enc := json.NewEncoder(f) @@ -297,4 +301,5 @@ func (c *fileStore) writeState(final bool) { } // Only mark as not dirty if we succeeded in the write. c.dirty = false + c.log.Infow("write state to file succeeded", "id", c.id, "path", c.path) } diff --git a/libbeat/processors/fingerprint/config.go b/libbeat/processors/fingerprint/config.go index dc36b6bceff..2f31691e741 100644 --- a/libbeat/processors/fingerprint/config.go +++ b/libbeat/processors/fingerprint/config.go @@ -17,13 +17,15 @@ package fingerprint +import "encoding/json" + // Config for fingerprint processor. type Config struct { - Method hashMethod `config:"method"` // Hash function to use for fingerprinting - Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from - TargetField string `config:"target_field"` // Target field for the fingerprint - Encoding encodingMethod `config:"encoding"` // Encoding to use for target field value - IgnoreMissing bool `config:"ignore_missing"` // Ignore missing fields? + Method namedHashMethod `config:"method"` // Hash function to use for fingerprinting + Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from + TargetField string `config:"target_field"` // Target field for the fingerprint + Encoding namedEncodingMethod `config:"encoding"` // Encoding to use for target field value + IgnoreMissing bool `config:"ignore_missing"` // Ignore missing fields? } func defaultConfig() Config { @@ -34,3 +36,16 @@ func defaultConfig() Config { IgnoreMissing: false, } } + +func (c *Config) MarshalJSON() ([]byte, error) { + type Alias Config + return json.Marshal(&struct { + Method string + Encoding string + *Alias + }{ + Method: c.Method.Name, + Encoding: c.Encoding.Name, + Alias: (*Alias)(c), + }) +} diff --git a/libbeat/processors/fingerprint/encode.go b/libbeat/processors/fingerprint/encode.go index 843c7bd5d29..dd04068df73 100644 --- a/libbeat/processors/fingerprint/encode.go +++ b/libbeat/processors/fingerprint/encode.go @@ -24,16 +24,26 @@ import ( "strings" ) +type namedEncodingMethod struct { + Name string + Encode encodingMethod +} type encodingMethod func([]byte) string -var encodings = map[string]encodingMethod{ - "hex": hex.EncodeToString, - "base32": base32.StdEncoding.EncodeToString, - "base64": base64.StdEncoding.EncodeToString, +var encodings = map[string]namedEncodingMethod{} + +func init() { + for _, e := range []namedEncodingMethod{ + {Name: "hex", Encode: hex.EncodeToString}, + {Name: "base32", Encode: base32.StdEncoding.EncodeToString}, + {Name: "base64", Encode: base64.StdEncoding.EncodeToString}, + } { + encodings[e.Name] = e + } } // Unpack creates the encodingMethod from the given string -func (e *encodingMethod) Unpack(str string) error { +func (e *namedEncodingMethod) Unpack(str string) error { str = strings.ToLower(str) m, found := encodings[str] diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 3f22082bad4..fdbcf158b27 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -60,7 +60,7 @@ func New(cfg *config.C) (beat.Processor, error) { p := &fingerprint{ config: config, - hash: config.Method, + hash: config.Method.Hash, fields: fields, } @@ -75,7 +75,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { return nil, makeErrComputeFingerprint(err) } - encodedHash := p.config.Encoding(hashFn.Sum(nil)) + encodedHash := p.config.Encoding.Encode(hashFn.Sum(nil)) if _, err := event.PutValue(p.config.TargetField, encodedHash); err != nil { return nil, makeErrComputeFingerprint(err) @@ -85,8 +85,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { } func (p *fingerprint) String() string { - //nolint:staticcheck // https://github.com/elastic/beats/issues/35174 - json, _ := json.Marshal(p.config) + json, _ := json.Marshal(&p.config) return procName + "=" + string(json) } diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index ead0bc2c005..5f6bdb70b5e 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -18,6 +18,7 @@ package fingerprint import ( + "fmt" "math/rand" "strconv" "testing" @@ -77,6 +78,7 @@ func TestWithConfig(t *testing.T) { Fields: test.input.Clone(), } newEvent, err := p.Run(testEvent) + assert.NoError(t, err) v, err := newEvent.GetValue("fingerprint") assert.NoError(t, err) assert.Equal(t, test.want, v) @@ -459,6 +461,18 @@ func TestIgnoreMissing(t *testing.T) { } } +func TestProcessorStringer(t *testing.T) { + testConfig, err := config.NewConfigFrom(mapstr.M{ + "fields": []string{"field1"}, + "encoding": "hex", + "method": "md5", + }) + require.NoError(t, err) + p, err := New(testConfig) + require.NoError(t, err) + require.Equal(t, `fingerprint={"Method":"md5","Encoding":"hex","Fields":["field1"],"TargetField":"fingerprint","IgnoreMissing":false}`, fmt.Sprint(p)) +} + func BenchmarkHashMethods(b *testing.B) { events := nRandomEvents(100000) @@ -472,8 +486,8 @@ func BenchmarkHashMethods(b *testing.B) { b.Run(method, func(b *testing.B) { b.ResetTimer() - for _, e := range events { - _, err := p.Run(&e) + for i := range events { + _, err := p.Run(&events[i]) if err != nil { b.Fatal(err) } @@ -491,7 +505,7 @@ func nRandomEvents(num int) []beat.Event { charsetLen := len(charset) b := make([]byte, 200) - var events []beat.Event + events := make([]beat.Event, num) for i := 0; i < num; i++ { for j := range b { b[j] = charset[prng.Intn(charsetLen)] diff --git a/libbeat/processors/fingerprint/hash.go b/libbeat/processors/fingerprint/hash.go index 1c4af0d0161..1c8cf146a14 100644 --- a/libbeat/processors/fingerprint/hash.go +++ b/libbeat/processors/fingerprint/hash.go @@ -28,19 +28,29 @@ import ( "github.com/cespare/xxhash/v2" ) +type namedHashMethod struct { + Name string + Hash hashMethod +} type hashMethod func() hash.Hash -var hashes = map[string]hashMethod{ - "md5": md5.New, - "sha1": sha1.New, - "sha256": sha256.New, - "sha384": sha512.New384, - "sha512": sha512.New, - "xxhash": newXxHash, +var hashes = map[string]namedHashMethod{} + +func init() { + for _, h := range []namedHashMethod{ + {Name: "md5", Hash: md5.New}, + {Name: "sha1", Hash: sha1.New}, + {Name: "sha256", Hash: sha256.New}, + {Name: "sha384", Hash: sha512.New384}, + {Name: "sha512", Hash: sha512.New}, + {Name: "xxhash", Hash: newXxHash}, + } { + hashes[h.Name] = h + } } // Unpack creates the hashMethod from the given string -func (f *hashMethod) Unpack(str string) error { +func (f *namedHashMethod) Unpack(str string) error { str = strings.ToLower(str) m, found := hashes[str] diff --git a/libbeat/processors/urldecode/urldecode.go b/libbeat/processors/urldecode/urldecode.go index 66844003b5c..59ed552e2ae 100644 --- a/libbeat/processors/urldecode/urldecode.go +++ b/libbeat/processors/urldecode/urldecode.go @@ -23,10 +23,10 @@ import ( "net/url" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/checks" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" - "github.com/elastic/beats/v7/libbeat/publisher" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -83,7 +83,7 @@ func (p *urlDecode) Run(event *beat.Event) (*beat.Event, error) { err := p.decodeField(field.From, field.To, event) if err != nil { errMsg := fmt.Errorf("failed to decode fields in urldecode processor: %w", err) - if publisher.LogWithTrace() { + if management.TraceLevelEnabled() { p.log.Debug(errMsg.Error()) } if p.config.FailOnError { diff --git a/libbeat/publisher/pipeline/client_test.go b/libbeat/publisher/pipeline/client_test.go index a3f0c822b9e..015d8f70c9d 100644 --- a/libbeat/publisher/pipeline/client_test.go +++ b/libbeat/publisher/pipeline/client_test.go @@ -128,9 +128,9 @@ func TestClient(t *testing.T) { // a small in-memory queue with a very short flush interval q := memqueue.NewQueue(l, nil, memqueue.Settings{ - Events: 5, - FlushMinEvents: 1, - FlushTimeout: time.Millisecond, + Events: 5, + MaxGetRequest: 1, + FlushTimeout: time.Millisecond, }, 5) // model a processor that we're going to make produce errors after diff --git a/libbeat/publisher/processing/default.go b/libbeat/publisher/processing/default.go index e3a2c961a92..14f75fde004 100644 --- a/libbeat/publisher/processing/default.go +++ b/libbeat/publisher/processing/default.go @@ -25,11 +25,11 @@ import ( "github.com/elastic/beats/v7/libbeat/common/fleetmode" "github.com/elastic/beats/v7/libbeat/ecs" "github.com/elastic/beats/v7/libbeat/features" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/mapping" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/actions" "github.com/elastic/beats/v7/libbeat/processors/timeseries" - "github.com/elastic/beats/v7/libbeat/publisher" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -388,7 +388,7 @@ func (b *builder) Create(cfg beat.ProcessingConfig, drop bool) (beat.Processor, } // setup 10: debug print final event (P) - if b.log.IsDebug() || publisher.UnderAgent() { + if b.log.IsDebug() || management.UnderAgent() { processors.add(debugPrintProcessor(b.info, b.log)) } diff --git a/libbeat/publisher/processing/processors.go b/libbeat/publisher/processing/processors.go index 7e76fa57914..69fb5090e4c 100644 --- a/libbeat/publisher/processing/processors.go +++ b/libbeat/publisher/processing/processors.go @@ -27,9 +27,9 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/outputs/codec/json" "github.com/elastic/beats/v7/libbeat/processors" - "github.com/elastic/beats/v7/libbeat/publisher" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -102,7 +102,7 @@ func (p *group) Close() error { } func (p *group) String() string { - var s []string + s := make([]string, 0, len(p.list)) for _, p := range p.list { s = append(s, p.String()) } @@ -200,7 +200,7 @@ func debugPrintProcessor(info beat.Info, log *logp.Logger) *processorFn { EscapeHTML: false, }) return newProcessor("debugPrint", func(event *beat.Event) (*beat.Event, error) { - if publisher.LogWithTrace() { + if management.TraceLevelEnabled() { mux.Lock() defer mux.Unlock() diff --git a/libbeat/publisher/queue/memqueue/ackloop.go b/libbeat/publisher/queue/memqueue/ackloop.go index f61439a6d50..1a964d8bb45 100644 --- a/libbeat/publisher/queue/memqueue/ackloop.go +++ b/libbeat/publisher/queue/memqueue/ackloop.go @@ -25,26 +25,28 @@ package memqueue type ackLoop struct { broker *broker - // A list of ACK channels given to queue consumers, + // A list of batches given to queue consumers, // used to maintain sequencing of event acknowledgements. - ackChans chanList + pendingBatches batchList +} - processACK func(chanList, int) +func newACKLoop(broker *broker) *ackLoop { + return &ackLoop{broker: broker} } func (l *ackLoop) run() { + b := l.broker for { - nextBatchChan := l.ackChans.nextBatchChannel() + nextBatchChan := l.pendingBatches.nextBatchChannel() select { - case <-l.broker.done: + case <-b.ctx.Done(): // The queue is shutting down. return - case chanList := <-l.broker.scheduledACKs: - // A new batch has been generated, add its ACK channel to the end of - // the pending list. - l.ackChans.concat(&chanList) + case chanList := <-b.consumedChan: + // New batches have been generated, add them to the pending list + l.pendingBatches.concat(&chanList) case <-nextBatchChan: // The oldest outstanding batch has been acknowledged, advance our @@ -57,11 +59,11 @@ func (l *ackLoop) run() { // handleBatchSig collects and handles a batch ACK/Cancel signal. handleBatchSig // is run by the ackLoop. func (l *ackLoop) handleBatchSig() int { - lst := l.collectAcked() + ackedBatches := l.collectAcked() count := 0 - for current := lst.front(); current != nil; current = current.next { - count += current.count + for batch := ackedBatches.front(); batch != nil; batch = batch.next { + count += batch.count } if count > 0 { @@ -70,11 +72,12 @@ func (l *ackLoop) handleBatchSig() int { } // report acks to waiting clients - l.processACK(lst, count) + l.processACK(ackedBatches, count) } - for !lst.empty() { - releaseACKChan(lst.pop()) + for !ackedBatches.empty() { + // Release finished batch structs into the shared memory pool + releaseBatch(ackedBatches.pop()) } // return final ACK to EventLoop, in order to clean up internal buffer @@ -84,23 +87,63 @@ func (l *ackLoop) handleBatchSig() int { return count } -func (l *ackLoop) collectAcked() chanList { - lst := chanList{} +func (l *ackLoop) collectAcked() batchList { + ackedBatches := batchList{} - acks := l.ackChans.pop() - lst.append(acks) + acks := l.pendingBatches.pop() + ackedBatches.append(acks) done := false - for !l.ackChans.empty() && !done { - acks := l.ackChans.front() + for !l.pendingBatches.empty() && !done { + acks := l.pendingBatches.front() select { case <-acks.doneChan: - lst.append(l.ackChans.pop()) + ackedBatches.append(l.pendingBatches.pop()) default: done = true } } - return lst + return ackedBatches +} + +// Called by ackLoop. This function exists to decouple the work of collecting +// and running producer callbacks from logical deletion of the events, so +// input callbacks can't block the queue by occupying the runLoop goroutine. +func (l *ackLoop) processACK(lst batchList, N int) { + ackCallbacks := []func(){} + // First we traverse the entries we're about to remove, collecting any callbacks + // we need to run. + lst.reverse() + for !lst.empty() { + batch := lst.pop() + + // Traverse entries from last to first, so we can acknowledge the most recent + // ones first and skip subsequent producer callbacks. + for i := batch.count - 1; i >= 0; i-- { + entry := batch.rawEntry(i) + if entry.producer == nil { + continue + } + + if entry.producerID <= entry.producer.state.lastACK { + // This index was already acknowledged on a previous iteration, skip. + entry.producer = nil + continue + } + producerState := entry.producer.state + count := int(entry.producerID - producerState.lastACK) + ackCallbacks = append(ackCallbacks, func() { producerState.cb(count) }) + entry.producer.state.lastACK = entry.producerID + entry.producer = nil + } + } + // Signal runLoop to delete the events + l.broker.deleteChan <- N + + // The events have been removed; notify their listeners. + for _, f := range ackCallbacks { + f() + } } diff --git a/libbeat/publisher/queue/memqueue/broker.go b/libbeat/publisher/queue/memqueue/broker.go index ac5b9dc6615..e1d0fd46c00 100644 --- a/libbeat/publisher/queue/memqueue/broker.go +++ b/libbeat/publisher/queue/memqueue/broker.go @@ -18,6 +18,7 @@ package memqueue import ( + "context" "io" "sync" "time" @@ -35,12 +36,23 @@ const ( maxInputQueueSizeRatio = 0.1 ) +// broker is the main implementation type for the memory queue. An active queue +// consists of two goroutines: runLoop, which handles all public API requests +// and owns the buffer state, and ackLoop, which listens for acknowledgments of +// consumed events and runs any appropriate completion handlers. type broker struct { - done chan struct{} + settings Settings + logger *logp.Logger - logger *logp.Logger + ctx context.Context + ctxCancel context.CancelFunc - bufSize int + // The ring buffer backing the queue. All buffer positions should be taken + // modulo the size of this array. + buf []queueEntry + + // wait group for queue workers (runLoop and ackLoop) + wg sync.WaitGroup /////////////////////////// // api channels @@ -55,35 +67,49 @@ type broker struct { // sent so far that have not yet reached a consumer. cancelChan chan producerCancelRequest + // Metrics() sends requests to metricChan to expose internal queue + // metrics to external callers. + metricChan chan metricsRequest + /////////////////////////// // internal channels - // When events are sent to consumers, the ACK channels for their batches - // are collected into chanLists and sent to scheduledACKs. - // These are then read by ackLoop and concatenated to its internal - // chanList of all outstanding ACK channels. - scheduledACKs chan chanList + // Batches sent to consumers are also collected and forwarded to ackLoop + // through this channel so ackLoop can monitor them for acknowledgments. + consumedChan chan batchList - // A callback that should be invoked when ACKs are processed. + // ackCallback is a configurable callback to invoke when ACKs are processed. // ackLoop calls this function when it advances the consumer ACK position. // Right now this forwards the notification to queueACKed() in // the pipeline observer, which updates the beats registry if needed. ackCallback func(eventCount int) - // This channel is used to request/return metrics where such metrics require insight into - // the actual eventloop itself. This seems like it might be overkill, but it seems that - // all communication between the broker and the eventloops - // happens via channels, so we're doing it this way. - metricChan chan metricsRequest + // When batches are acknowledged, ackLoop saves any metadata needed + // for producer callbacks and such, then notifies runLoop that it's + // safe to free these events and advance the queue by sending the + // acknowledged event count to this channel. + deleteChan chan int - // wait group for worker shutdown - wg sync.WaitGroup + /////////////////////////////// + // internal goroutine state + + // The goroutine that manages the queue's core run state + runLoop *runLoop + + // The goroutine that manages ack notifications and callbacks + ackLoop *ackLoop } type Settings struct { - Events int - FlushMinEvents int - FlushTimeout time.Duration + // The number of events the queue can hold. + Events int + + // The most events that will ever be returned from one Get request. + MaxGetRequest int + + // If positive, the amount of time the queue will wait to fill up + // a batch if a Get request asks for more events than we have. + FlushTimeout time.Duration } type queueEntry struct { @@ -95,24 +121,22 @@ type queueEntry struct { } type batch struct { - queue *broker - entries []queueEntry - doneChan chan batchDoneMsg -} + queue *broker + + // Next batch in the containing batchList + next *batch -// batchACKState stores the metadata associated with a batch of events sent to -// a consumer. When the consumer ACKs that batch, a batchAckMsg is sent on -// ackChan and received by -type batchACKState struct { - next *batchACKState - doneChan chan batchDoneMsg - start, count int // number of events waiting for ACK - entries []queueEntry + // Position and length of the events within the queue buffer + start, count int + + // batch.Done() sends to doneChan, where ackLoop reads it and handles + // acknowledgment / cleanup. + doneChan chan batchDoneMsg } -type chanList struct { - head *batchACKState - tail *batchACKState +type batchList struct { + head *batch + tail *batch } // FactoryForSettings is a simple wrapper around NewQueue so a concrete @@ -137,23 +161,46 @@ func NewQueue( settings Settings, inputQueueSize int, ) *broker { - var ( - sz = settings.Events - minEvents = settings.FlushMinEvents - flushTimeout = settings.FlushTimeout - ) + b := newQueue(logger, ackCallback, settings, inputQueueSize) - chanSize := AdjustInputQueueSize(inputQueueSize, sz) + // Start the queue workers + b.wg.Add(2) + go func() { + defer b.wg.Done() + b.runLoop.run() + }() + go func() { + defer b.wg.Done() + b.ackLoop.run() + }() - if minEvents < 1 { - minEvents = 1 - } - if minEvents > 1 && flushTimeout <= 0 { - minEvents = 1 - flushTimeout = 0 + return b +} + +// newQueue does most of the work of creating a queue from the given +// parameters, but doesn't start the runLoop or ackLoop workers. This +// lets us perform more granular / deterministic tests by controlling +// when the workers are active. +func newQueue( + logger *logp.Logger, + ackCallback func(eventCount int), + settings Settings, + inputQueueSize int, +) *broker { + chanSize := AdjustInputQueueSize(inputQueueSize, settings.Events) + + // Backwards compatibility: an old way to select synchronous queue + // behavior was to set "flush.min_events" to 0 or 1, in which case the + // timeout was disabled and the max get request was half the queue. + // (Otherwise, it would make sense to leave FlushTimeout unchanged here.) + if settings.MaxGetRequest <= 1 { + settings.FlushTimeout = 0 + settings.MaxGetRequest = (settings.Events + 1) / 2 } - if minEvents > sz { - minEvents = sz + + // Can't request more than the full queue + if settings.MaxGetRequest > settings.Events { + settings.MaxGetRequest = settings.Events } if logger == nil { @@ -161,52 +208,33 @@ func NewQueue( } b := &broker{ - done: make(chan struct{}), - logger: logger, + settings: settings, + logger: logger, + + buf: make([]queueEntry, settings.Events), // broker API channels pushChan: make(chan pushRequest, chanSize), getChan: make(chan getRequest), cancelChan: make(chan producerCancelRequest, 5), + metricChan: make(chan metricsRequest), - // internal broker and ACK handler channels - scheduledACKs: make(chan chanList), + // internal runLoop and ackLoop channels + consumedChan: make(chan batchList), + deleteChan: make(chan int), ackCallback: ackCallback, - metricChan: make(chan metricsRequest), } + b.ctx, b.ctxCancel = context.WithCancel(context.Background()) - var eventLoop interface { - run() - processACK(chanList, int) - } - - if minEvents > 1 { - eventLoop = newBufferingEventLoop(b, sz, minEvents, flushTimeout) - } else { - eventLoop = newDirectEventLoop(b, sz) - } - - b.bufSize = sz - ackLoop := &ackLoop{ - broker: b, - processACK: eventLoop.processACK} - - b.wg.Add(2) - go func() { - defer b.wg.Done() - eventLoop.run() - }() - go func() { - defer b.wg.Done() - ackLoop.run() - }() + b.runLoop = newRunLoop(b) + b.ackLoop = newACKLoop(b) return b } func (b *broker) Close() error { - close(b.done) + b.ctxCancel() return nil } @@ -216,7 +244,7 @@ func (b *broker) QueueType() string { func (b *broker) BufferConfig() queue.BufferConfig { return queue.BufferConfig{ - MaxEvents: b.bufSize, + MaxEvents: len(b.buf), } } @@ -225,9 +253,9 @@ func (b *broker) Producer(cfg queue.ProducerConfig) queue.Producer { } func (b *broker) Get(count int) (queue.Batch, error) { - responseChan := make(chan getResponse, 1) + responseChan := make(chan *batch, 1) select { - case <-b.done: + case <-b.ctx.Done(): return nil, io.EOF case b.getChan <- getRequest{ entryCount: count, responseChan: responseChan}: @@ -235,18 +263,14 @@ func (b *broker) Get(count int) (queue.Batch, error) { // if request has been sent, we have to wait for a response resp := <-responseChan - return &batch{ - queue: b, - entries: resp.entries, - doneChan: resp.ackChan, - }, nil + return resp, nil } func (b *broker) Metrics() (queue.Metrics, error) { responseChan := make(chan memQueueMetrics, 1) select { - case <-b.done: + case <-b.ctx.Done(): return queue.Metrics{}, io.EOF case b.metricChan <- metricsRequest{ responseChan: responseChan}: @@ -255,43 +279,43 @@ func (b *broker) Metrics() (queue.Metrics, error) { return queue.Metrics{ EventCount: opt.UintWith(uint64(resp.currentQueueSize)), - EventLimit: opt.UintWith(uint64(b.bufSize)), + EventLimit: opt.UintWith(uint64(len(b.buf))), UnackedConsumedEvents: opt.UintWith(uint64(resp.occupiedRead)), OldestEntryID: resp.oldestEntryID, }, nil } -var ackChanPool = sync.Pool{ +var batchPool = sync.Pool{ New: func() interface{} { - return &batchACKState{ + return &batch{ doneChan: make(chan batchDoneMsg, 1), } }, } -func newBatchACKState(start, count int, entries []queueEntry) *batchACKState { - ch := ackChanPool.Get().(*batchACKState) - ch.next = nil - ch.start = start - ch.count = count - ch.entries = entries - return ch +func newBatch(queue *broker, start, count int) *batch { + batch := batchPool.Get().(*batch) + batch.next = nil + batch.queue = queue + batch.start = start + batch.count = count + return batch } -func releaseACKChan(c *batchACKState) { - c.next = nil - ackChanPool.Put(c) +func releaseBatch(b *batch) { + b.next = nil + batchPool.Put(b) } -func (l *chanList) prepend(ch *batchACKState) { - ch.next = l.head - l.head = ch +func (l *batchList) prepend(b *batch) { + b.next = l.head + l.head = b if l.tail == nil { - l.tail = ch + l.tail = b } } -func (l *chanList) concat(other *chanList) { +func (l *batchList) concat(other *batchList) { if other.head == nil { return } @@ -305,31 +329,31 @@ func (l *chanList) concat(other *chanList) { l.tail = other.tail } -func (l *chanList) append(ch *batchACKState) { +func (l *batchList) append(b *batch) { if l.head == nil { - l.head = ch + l.head = b } else { - l.tail.next = ch + l.tail.next = b } - l.tail = ch + l.tail = b } -func (l *chanList) empty() bool { +func (l *batchList) empty() bool { return l.head == nil } -func (l *chanList) front() *batchACKState { +func (l *batchList) front() *batch { return l.head } -func (l *chanList) nextBatchChannel() chan batchDoneMsg { +func (l *batchList) nextBatchChannel() chan batchDoneMsg { if l.head == nil { return nil } return l.head.doneChan } -func (l *chanList) pop() *batchACKState { +func (l *batchList) pop() *batch { ch := l.head if ch != nil { l.head = ch.next @@ -342,9 +366,9 @@ func (l *chanList) pop() *batchACKState { return ch } -func (l *chanList) reverse() { +func (l *batchList) reverse() { tmp := *l - *l = chanList{} + *l = batchList{} for !tmp.empty() { l.prepend(tmp.pop()) @@ -364,11 +388,18 @@ func AdjustInputQueueSize(requested, mainQueueSize int) (actual int) { } func (b *batch) Count() int { - return len(b.entries) + return b.count +} + +// Return a pointer to the queueEntry for the i-th element of this batch +func (b *batch) rawEntry(i int) *queueEntry { + // Indexes wrap around the end of the queue buffer + return &b.queue.buf[(b.start+i)%len(b.queue.buf)] } +// Return the event referenced by the i-th element of this batch func (b *batch) Entry(i int) interface{} { - return b.entries[i].event + return b.rawEntry(i).event } func (b *batch) FreeEntries() { diff --git a/libbeat/publisher/queue/memqueue/config.go b/libbeat/publisher/queue/memqueue/config.go index 5e4f78ae41c..7d9593b30e3 100644 --- a/libbeat/publisher/queue/memqueue/config.go +++ b/libbeat/publisher/queue/memqueue/config.go @@ -26,19 +26,23 @@ import ( ) type config struct { - Events int `config:"events" validate:"min=32"` - FlushMinEvents int `config:"flush.min_events" validate:"min=0"` - FlushTimeout time.Duration `config:"flush.timeout"` + Events int `config:"events" validate:"min=32"` + // This field is named MaxGetRequest because its logical effect is to give + // a maximum on the number of events a Get request can return, but the + // user-exposed name is "flush.min_events" for backwards compatibility, + // since it used to control buffer size in the internal buffer chain. + MaxGetRequest int `config:"flush.min_events" validate:"min=0"` + FlushTimeout time.Duration `config:"flush.timeout"` } var defaultConfig = config{ - Events: 3200, - FlushMinEvents: 1600, - FlushTimeout: 10 * time.Second, + Events: 3200, + MaxGetRequest: 1600, + FlushTimeout: 10 * time.Second, } func (c *config) Validate() error { - if c.FlushMinEvents > c.Events { + if c.MaxGetRequest > c.Events { return errors.New("flush.min_events must be less events") } return nil @@ -55,8 +59,8 @@ func SettingsForUserConfig(cfg *c.C) (Settings, error) { } //nolint:gosimple // Actually want this conversion to be explicit since the types aren't definitionally equal. return Settings{ - Events: config.Events, - FlushMinEvents: config.FlushMinEvents, - FlushTimeout: config.FlushTimeout, + Events: config.Events, + MaxGetRequest: config.MaxGetRequest, + FlushTimeout: config.FlushTimeout, }, nil } diff --git a/libbeat/publisher/queue/memqueue/eventloop.go b/libbeat/publisher/queue/memqueue/eventloop.go deleted file mode 100644 index ccb50565365..00000000000 --- a/libbeat/publisher/queue/memqueue/eventloop.go +++ /dev/null @@ -1,560 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memqueue - -import ( - "time" - - "github.com/elastic/beats/v7/libbeat/publisher/queue" - "github.com/elastic/elastic-agent-libs/logp" -) - -// directEventLoop implements the broker main event loop. It buffers events, -// but tries to forward events as early as possible. -type directEventLoop struct { - broker *broker - buf ringBuffer - deleteChan chan int - - // pendingACKs aggregates a list of ACK channels for batches that have been sent - // to consumers, which is then sent to the broker's scheduledACKs channel. - pendingACKs chanList - - nextEntryID queue.EntryID -} - -// bufferingEventLoop implements the broker main event loop. -// Events in the buffer are forwarded to consumers only if the buffer is full or on flush timeout. -type bufferingEventLoop struct { - broker *broker - deleteChan chan int - - // The current buffer that incoming events are appended to. When it gets - // full enough, or enough time has passed, it is added to flushList. - // Events will still be added to buf even after it is in flushList, until - // either it reaches minEvents or a consumer requests it. - buf *batchBuffer - - // flushList is the list of buffers that are ready to be sent to consumers. - flushList flushList - - // pendingACKs aggregates a list of ACK channels for batches that have been sent - // to consumers, which is then sent to the broker's scheduledACKs channel. - pendingACKs chanList - - // The number of events currently waiting in the queue, including - // those that have not yet been acked. - eventCount int - - // The next entry ID that will be read by a consumer, and the next - // entry ID that has been consumed and is waiting for acknowledgment. - // We need to track these here because bufferingEventLoop discards - // its event buffers when they are sent to consumers, so we can't - // look directly at the event itself to get the current id like we - // do in the unbuffered loop. - nextConsumedID queue.EntryID - nextACKedID queue.EntryID - - minEvents int - maxEvents int - flushTimeout time.Duration - - // buffer flush timer state - timer *time.Timer - idleC <-chan time.Time - - nextEntryID queue.EntryID -} - -type flushList struct { - head *batchBuffer - tail *batchBuffer - count int -} - -func newDirectEventLoop(b *broker, size int) *directEventLoop { - l := &directEventLoop{ - broker: b, - deleteChan: make(chan int), - } - l.buf.init(b.logger, size) - - return l -} - -func (l *directEventLoop) run() { - var ( - broker = l.broker - buf = &l.buf - ) - - for { - var pushChan chan pushRequest - // Push requests are enabled if the queue isn't yet full. - if !l.buf.Full() { - pushChan = l.broker.pushChan - } - - var getChan chan getRequest - // Get requests are enabled if there are events in the queue - // that haven't yet been sent to a consumer. - if buf.Avail() > 0 { - getChan = l.broker.getChan - } - - var schedACKs chan chanList - // Sending pending ACKs to the broker's scheduled ACKs - // channel is enabled if it is nonempty. - if !l.pendingACKs.empty() { - schedACKs = l.broker.scheduledACKs - } - - select { - case <-broker.done: - return - - case req := <-pushChan: // producer pushing new event - l.insert(&req) - - case count := <-l.deleteChan: - l.buf.removeEntries(count) - - case req := <-l.broker.cancelChan: // producer cancelling active events - l.handleCancel(&req) - // re-enable pushRequest if buffer can take new events - - case req := <-getChan: // consumer asking for next batch - l.handleGetRequest(&req) - - case req := <-l.broker.metricChan: // broker asking for queue metrics - l.handleMetricsRequest(&req) - - case schedACKs <- l.pendingACKs: - // on send complete list of pending batches has been forwarded -> clear list - l.pendingACKs = chanList{} - } - } -} - -func (l *directEventLoop) handleMetricsRequest(req *metricsRequest) { - // If the queue is empty, we report the "oldest" ID as the next - // one that will be assigned. Otherwise, we report the ID attached - // to the oldest queueEntry. - oldestEntryID := l.nextEntryID - if oldestEntry := l.buf.OldestEntry(); oldestEntry != nil { - oldestEntryID = oldestEntry.id - } - - req.responseChan <- memQueueMetrics{ - currentQueueSize: l.buf.Items(), - occupiedRead: l.buf.reserved, - oldestEntryID: oldestEntryID, - } -} - -func (l *directEventLoop) insert(req *pushRequest) { - log := l.broker.logger - - if req.producer != nil && req.producer.state.cancelled { - reportCancelledState(log, req) - } else { - req.resp <- l.nextEntryID - l.buf.insert(queueEntry{ - event: req.event, - id: l.nextEntryID, - producer: req.producer, - producerID: req.producerID}) - l.nextEntryID++ - } -} - -func (l *directEventLoop) handleCancel(req *producerCancelRequest) { - // log := l.broker.logger - // log.Debug("handle cancel request") - - var removed int - - if producer := req.producer; producer != nil { - producer.state.cancelled = true - removed = l.buf.cancel(producer) - } - - // signal cancel request being finished - if req.resp != nil { - req.resp <- producerCancelResponse{removed: removed} - } -} - -func (l *directEventLoop) handleGetRequest(req *getRequest) { - // log := l.broker.logger - // log.Debugf("try reserve %v events", req.sz) - - start, buf := l.buf.reserve(req.entryCount) - count := len(buf) - if count == 0 { - panic("empty batch returned") - } - - ackCH := newBatchACKState(start, count, l.buf.entries) - - req.responseChan <- getResponse{ackCH.doneChan, buf} - l.pendingACKs.append(ackCH) -} - -// processACK is called by the ackLoop to process the list of acked batches -func (l *directEventLoop) processACK(lst chanList, N int) { - log := l.broker.logger - { - start := time.Now() - log.Debug("handle ACKs: ", N) - defer func() { - log.Debug("handle ACK took: ", time.Since(start)) - }() - } - - entries := l.buf.entries - - firstIndex := lst.front().start - - // We want to acknowledge N events starting at position firstIndex - // in the entries array. - // We iterate over the events from last to first, so we encounter the - // highest producer IDs first and can skip subsequent callbacks to the - // same producer. - producerCallbacks := []func(){} - for i := N - 1; i >= 0; i-- { - // idx is the index in entries of the i-th event after firstIndex, wrapping - // around the end of the array. - idx := (firstIndex + i) % len(entries) - entry := &entries[idx] - - producer := entry.producer - - // Set the producer in the entires array to nil to mark it as visited; a nil - // producer indicates that an entry requires no more ack processing (either - // because it has already been ACKed, or because its producer does not listen to ACKs). - entry.producer = nil - if producer == nil || entry.producerID <= producer.state.lastACK { - // This has a lower index than the previous ACK for this producer, - // so it was covered in the previous call and we can skip it. - continue - } - // This update is safe because lastACK is only used from the event loop. - count := int(entry.producerID - producer.state.lastACK) - producer.state.lastACK = entry.producerID - - producerCallbacks = append(producerCallbacks, func() { producer.state.cb(count) }) - } - l.deleteChan <- N - for _, f := range producerCallbacks { - f() - } -} - -func newBufferingEventLoop(b *broker, size int, minEvents int, flushTimeout time.Duration) *bufferingEventLoop { - l := &bufferingEventLoop{ - broker: b, - deleteChan: make(chan int), - maxEvents: size, - minEvents: minEvents, - flushTimeout: flushTimeout, - } - l.buf = newBatchBuffer(l.minEvents) - - l.timer = time.NewTimer(flushTimeout) - if !l.timer.Stop() { - <-l.timer.C - } - - return l -} - -func (l *bufferingEventLoop) run() { - broker := l.broker - - for { - var pushChan chan pushRequest - // Push requests are enabled if the queue isn't yet full. - if l.eventCount < l.maxEvents { - pushChan = l.broker.pushChan - } - - var getChan chan getRequest - // Get requests are enabled if the queue has events that - // weren't yet sent to consumers. - if !l.flushList.empty() { - getChan = l.broker.getChan - } - - var schedACKs chan chanList - // Enable sending to the scheduled ACKs channel if we have - // something to send. - if !l.pendingACKs.empty() { - schedACKs = l.broker.scheduledACKs - } - - select { - case <-broker.done: - return - - case req := <-pushChan: // producer pushing new event - l.handleInsert(&req) - - case req := <-l.broker.cancelChan: // producer cancelling active events - l.handleCancel(&req) - - case req := <-getChan: // consumer asking for next batch - l.handleGetRequest(&req) - - case schedACKs <- l.pendingACKs: - l.pendingACKs = chanList{} - - case count := <-l.deleteChan: - l.handleDelete(count) - - case req := <-l.broker.metricChan: // broker asking for queue metrics - l.handleMetricsRequest(&req) - - case <-l.idleC: - l.idleC = nil - l.timer.Stop() - if l.buf.length() > 0 { - l.flushBuffer() - } - } - } -} - -func (l *bufferingEventLoop) handleMetricsRequest(req *metricsRequest) { - req.responseChan <- memQueueMetrics{ - currentQueueSize: l.eventCount, - occupiedRead: int(l.nextConsumedID - l.nextACKedID), - oldestEntryID: l.nextACKedID, - } -} - -func (l *bufferingEventLoop) handleInsert(req *pushRequest) { - if l.insert(req, l.nextEntryID) { - // Send back the new event id. - req.resp <- l.nextEntryID - - l.nextEntryID++ - l.eventCount++ - - L := l.buf.length() - if !l.buf.flushed { - if L < l.minEvents { - l.startFlushTimer() - } else { - l.stopFlushTimer() - l.flushBuffer() - l.buf = newBatchBuffer(l.minEvents) - } - } else if L >= l.minEvents { - l.buf = newBatchBuffer(l.minEvents) - } - } -} - -func (l *bufferingEventLoop) insert(req *pushRequest, id queue.EntryID) bool { - if req.producer != nil && req.producer.state.cancelled { - reportCancelledState(l.broker.logger, req) - return false - } - - l.buf.add(queueEntry{ - event: req.event, - id: id, - producer: req.producer, - producerID: req.producerID, - }) - return true -} - -func (l *bufferingEventLoop) handleCancel(req *producerCancelRequest) { - removed := 0 - if producer := req.producer; producer != nil { - // remove from actively flushed buffers - for buf := l.flushList.head; buf != nil; buf = buf.next { - removed += buf.cancel(producer) - } - if !l.buf.flushed { - removed += l.buf.cancel(producer) - } - - producer.state.cancelled = true - } - - if req.resp != nil { - req.resp <- producerCancelResponse{removed: removed} - } - - // remove flushed but empty buffers: - tmpList := flushList{} - for l.flushList.head != nil { - b := l.flushList.head - l.flushList.head = b.next - - if b.length() > 0 { - tmpList.add(b) - } - } - l.flushList = tmpList - l.eventCount -= removed -} - -func (l *bufferingEventLoop) handleGetRequest(req *getRequest) { - buf := l.flushList.head - if buf == nil { - panic("get from non-flushed buffers") - } - - count := buf.length() - if count == 0 { - panic("empty buffer in flush list") - } - - if sz := req.entryCount; sz > 0 { - if sz < count { - count = sz - } - } - - if count == 0 { - panic("empty batch returned") - } - - entries := buf.entries[:count] - acker := newBatchACKState(0, count, entries) - - req.responseChan <- getResponse{acker.doneChan, entries} - l.pendingACKs.append(acker) - - l.nextConsumedID += queue.EntryID(len(entries)) - buf.entries = buf.entries[count:] - if buf.length() == 0 { - l.advanceFlushList() - } -} - -func (l *bufferingEventLoop) handleDelete(count int) { - l.nextACKedID += queue.EntryID(count) - l.eventCount -= count -} - -func (l *bufferingEventLoop) startFlushTimer() { - if l.idleC == nil { - l.timer.Reset(l.flushTimeout) - l.idleC = l.timer.C - } -} - -func (l *bufferingEventLoop) stopFlushTimer() { - if l.idleC != nil { - l.idleC = nil - if !l.timer.Stop() { - <-l.timer.C - } - } -} - -func (l *bufferingEventLoop) advanceFlushList() { - l.flushList.pop() - if l.flushList.count == 0 && l.buf.flushed { - l.buf = newBatchBuffer(l.minEvents) - } -} - -func (l *bufferingEventLoop) flushBuffer() { - l.buf.flushed = true - l.flushList.add(l.buf) -} - -// Called by ackLoop. This function exists to decouple the work of collecting -// and running producer callbacks from logical deletion of the events, so -// input callbacks can't block the main queue goroutine. -func (l *bufferingEventLoop) processACK(lst chanList, N int) { - ackCallbacks := []func(){} - // First we traverse the entries we're about to remove, collecting any callbacks - // we need to run. - lst.reverse() - for !lst.empty() { - current := lst.pop() - entries := current.entries - - // Traverse entries from last to first, so we can acknowledge the most recent - // ones first and skip subsequent producer callbacks. - for i := len(entries) - 1; i >= 0; i-- { - entry := &entries[i] - if entry.producer == nil { - continue - } - - if entry.producerID <= entry.producer.state.lastACK { - // This index was already acknowledged on a previous iteration, skip. - entry.producer = nil - continue - } - producerState := entry.producer.state - count := int(entry.producerID - producerState.lastACK) - ackCallbacks = append(ackCallbacks, func() { producerState.cb(count) }) - entry.producer.state.lastACK = entry.producerID - entry.producer = nil - } - } - // Signal the queue to delete the events - l.deleteChan <- N - - // The events have been removed; notify their listeners. - for _, f := range ackCallbacks { - f() - } -} - -func (l *flushList) pop() { - l.count-- - if l.count > 0 { - l.head = l.head.next - } else { - l.head = nil - l.tail = nil - } -} - -func (l *flushList) empty() bool { - return l.head == nil -} - -func (l *flushList) add(b *batchBuffer) { - l.count++ - b.next = nil - if l.tail == nil { - l.head = b - l.tail = b - } else { - l.tail.next = b - l.tail = b - } -} - -func reportCancelledState(log *logp.Logger, req *pushRequest) { - // do not add waiting events if producer did send cancel signal - if cb := req.producer.state.dropCB; cb != nil { - cb(req.event) - } -} diff --git a/libbeat/publisher/queue/memqueue/internal_api.go b/libbeat/publisher/queue/memqueue/internal_api.go index 07485af99b4..ae93a5df0d5 100644 --- a/libbeat/publisher/queue/memqueue/internal_api.go +++ b/libbeat/publisher/queue/memqueue/internal_api.go @@ -46,13 +46,8 @@ type producerCancelResponse struct { // consumer -> broker API type getRequest struct { - entryCount int // request entryCount events from the broker - responseChan chan getResponse // channel to send response to -} - -type getResponse struct { - ackChan chan batchDoneMsg - entries []queueEntry + entryCount int // request entryCount events from the broker + responseChan chan *batch // channel to send response to } type batchDoneMsg struct{} diff --git a/libbeat/publisher/queue/memqueue/queue_test.go b/libbeat/publisher/queue/memqueue/queue_test.go index 531acdce3e9..141514483f3 100644 --- a/libbeat/publisher/queue/memqueue/queue_test.go +++ b/libbeat/publisher/queue/memqueue/queue_test.go @@ -46,10 +46,10 @@ func TestProduceConsumer(t *testing.T) { maxEvents := 1024 minEvents := 32 - rand.Seed(seed) - events := rand.Intn(maxEvents-minEvents) + minEvents - batchSize := rand.Intn(events-8) + 4 - bufferSize := rand.Intn(batchSize*2) + 4 + randGen := rand.New(rand.NewSource(seed)) + events := randGen.Intn(maxEvents-minEvents) + minEvents + batchSize := randGen.Intn(events-8) + 4 + bufferSize := randGen.Intn(batchSize*2) + 4 // events := 4 // batchSize := 1 @@ -90,9 +90,9 @@ func TestProduceConsumer(t *testing.T) { func TestProducerDoesNotBlockWhenCancelled(t *testing.T) { q := NewQueue(nil, nil, Settings{ - Events: 2, // Queue size - FlushMinEvents: 1, // make sure the queue won't buffer events - FlushTimeout: time.Millisecond, + Events: 2, // Queue size + MaxGetRequest: 1, // make sure the queue won't buffer events + FlushTimeout: time.Millisecond, }, 0) p := q.Producer(queue.ProducerConfig{ @@ -155,9 +155,9 @@ func TestQueueMetricsDirect(t *testing.T) { // Test the directEventLoop directSettings := Settings{ - Events: maxEvents, - FlushMinEvents: 1, - FlushTimeout: 0, + Events: maxEvents, + MaxGetRequest: 1, + FlushTimeout: 0, } t.Logf("Testing directEventLoop") queueTestWithSettings(t, directSettings, eventsToTest, "directEventLoop") @@ -169,9 +169,9 @@ func TestQueueMetricsBuffer(t *testing.T) { maxEvents := 10 // Test Buffered Event Loop bufferedSettings := Settings{ - Events: maxEvents, - FlushMinEvents: eventsToTest, // The buffered event loop can only return FlushMinEvents per Get() - FlushTimeout: time.Millisecond, + Events: maxEvents, + MaxGetRequest: eventsToTest, // The buffered event loop can only return FlushMinEvents per Get() + FlushTimeout: time.Millisecond, } t.Logf("Testing bufferedEventLoop") queueTestWithSettings(t, bufferedSettings, eventsToTest, "bufferedEventLoop") @@ -219,9 +219,9 @@ func TestProducerCancelRemovesEvents(t *testing.T) { func makeTestQueue(sz, minEvents int, flushTimeout time.Duration) queuetest.QueueFactory { return func(_ *testing.T) queue.Queue { return NewQueue(nil, nil, Settings{ - Events: sz, - FlushMinEvents: minEvents, - FlushTimeout: flushTimeout, + Events: sz, + MaxGetRequest: minEvents, + FlushTimeout: flushTimeout, }, 0) } } @@ -343,12 +343,12 @@ func TestEntryIDs(t *testing.T) { }) t.Run("acking in forward order with bufferedEventLoop reports the right event IDs", func(t *testing.T) { - testQueue := NewQueue(nil, nil, Settings{Events: 1000, FlushMinEvents: 2, FlushTimeout: time.Microsecond}, 0) + testQueue := NewQueue(nil, nil, Settings{Events: 1000, MaxGetRequest: 2, FlushTimeout: time.Microsecond}, 0) testForward(testQueue) }) t.Run("acking in reverse order with bufferedEventLoop reports the right event IDs", func(t *testing.T) { - testQueue := NewQueue(nil, nil, Settings{Events: 1000, FlushMinEvents: 2, FlushTimeout: time.Microsecond}, 0) + testQueue := NewQueue(nil, nil, Settings{Events: 1000, MaxGetRequest: 2, FlushTimeout: time.Microsecond}, 0) testBackward(testQueue) }) } diff --git a/libbeat/publisher/queue/memqueue/ringbuf.go b/libbeat/publisher/queue/memqueue/ringbuf.go deleted file mode 100644 index d593cc2351b..00000000000 --- a/libbeat/publisher/queue/memqueue/ringbuf.go +++ /dev/null @@ -1,210 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memqueue - -import ( - "fmt" - - "github.com/elastic/elastic-agent-libs/logp" -) - -// Internal event ring buffer. -// The ring is split into 2 contiguous regions. -// Events are appended to region A until it grows to the end of the internal -// buffer. Then region B is created at the beginning of the internal buffer, -// and events are inserted there until region A is emptied. When A becomes empty, -// we rename region B to region A, and the cycle repeats every time we wrap around -// the internal array storage. -type ringBuffer struct { - logger *logp.Logger - - entries []queueEntry - - // The underlying array is divided up into two contiguous regions. - regA, regB region - - // The number of events awaiting ACK at the beginning of region A. - reserved int -} - -// region represents a contiguous region in ringBuffer's internal storage (i.e. -// one that does not cross the end of the array). -type region struct { - // The starting position of this region within the full event buffer. - index int - - // The number of events currently stored in this region. - size int -} - -func (b *ringBuffer) init(logger *logp.Logger, size int) { - *b = ringBuffer{ - logger: logger, - entries: make([]queueEntry, size), - } -} - -// Returns true if the ringBuffer is full after handling -// the given insertion, false otherwise. -func (b *ringBuffer) insert(entry queueEntry) { - // always insert into region B, if region B exists. - // That is, we have 2 regions and region A is currently processed by consumers - if b.regB.size > 0 { - // log.Debug(" - push into B region") - - idx := b.regB.index + b.regB.size - avail := b.regA.index - idx - if avail > 0 { - b.entries[idx] = entry - b.regB.size++ - } - return - } - - // region B does not exist yet, check if region A is available for use - idx := b.regA.index + b.regA.size - if b.regA.index+b.regA.size >= len(b.entries) { - // region A extends to the end of the buffer - if b.regA.index > 0 { - // If there is space before region A, create - // region B there. - b.regB = region{index: 0, size: 1} - b.entries[0] = entry - } - return - } - - // space available in region A -> let's append the event - // log.Debug(" - push into region A") - b.entries[idx] = entry - b.regA.size++ -} - -// cancel removes all buffered events matching `st`, not yet reserved by -// any consumer -func (b *ringBuffer) cancel(producer *ackProducer) int { - cancelledB := b.cancelRegion(producer, b.regB) - b.regB.size -= cancelledB - - cancelledA := b.cancelRegion(producer, region{ - index: b.regA.index + b.reserved, - size: b.regA.size - b.reserved, - }) - b.regA.size -= cancelledA - - return cancelledA + cancelledB -} - -// cancelRegion removes the events in the specified range having -// the specified produceState. It returns the number of events -// removed. -func (b *ringBuffer) cancelRegion(producer *ackProducer, reg region) int { - start := reg.index - end := start + reg.size - entries := b.entries[start:end] - - toEntries := entries[:0] - - // filter loop - for i := 0; i < reg.size; i++ { - if entries[i].producer == producer { - continue // remove - } - toEntries = append(toEntries, entries[i]) - } - - // re-initialize old buffer elements to help garbage collector - entries = entries[len(toEntries):] - for i := range entries { - entries[i] = queueEntry{} - } - - return len(entries) -} - -// reserve returns up to `sz` events from the brokerBuffer, -// exclusively marking the events as 'reserved'. Subsequent calls to `reserve` -// will only return enqueued and non-reserved events from the buffer. -// If `sz == -1`, all available events will be reserved. -func (b *ringBuffer) reserve(sz int) (int, []queueEntry) { - use := b.regA.size - b.reserved - - if sz > 0 && use > sz { - use = sz - } - - start := b.regA.index + b.reserved - end := start + use - b.reserved += use - return start, b.entries[start:end] -} - -// Remove the specified number of previously-reserved buffer entries from the -// start of region A. Called by the event loop when events are ACKed by -// consumers. -func (b *ringBuffer) removeEntries(count int) { - if b.regA.size < count { - panic(fmt.Errorf("commit region to big (commit region=%v, buffer size=%v)", - count, b.regA.size, - )) - } - - // clear region, so published events can be collected by the garbage collector: - end := b.regA.index + count - for i := b.regA.index; i < end; i++ { - b.entries[i] = queueEntry{} - } - - b.regA.index = end - b.regA.size -= count - b.reserved -= count - if b.regA.size == 0 { - // region A is empty, transfer region B into region A - b.regA = b.regB - b.regB.index = 0 - b.regB.size = 0 - } -} - -// Number of events that consumers can currently request. -func (b *ringBuffer) Avail() int { - return b.regA.size - b.reserved -} - -func (b *ringBuffer) Full() bool { - if b.regB.size > 0 { - return b.regA.index == (b.regB.index + b.regB.size) - } - return b.regA.size == len(b.entries) -} - -func (b *ringBuffer) Size() int { - return len(b.entries) -} - -// Items returns the count of events currently in the buffer -func (b *ringBuffer) Items() int { - return b.regA.size + b.regB.size -} - -func (b *ringBuffer) OldestEntry() *queueEntry { - if b.regA.size == 0 { - return nil - } - return &b.entries[b.regA.index] -} diff --git a/libbeat/publisher/queue/memqueue/runloop.go b/libbeat/publisher/queue/memqueue/runloop.go new file mode 100644 index 00000000000..0f7788c6209 --- /dev/null +++ b/libbeat/publisher/queue/memqueue/runloop.go @@ -0,0 +1,307 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memqueue + +import ( + "time" + + "github.com/elastic/beats/v7/libbeat/publisher/queue" +) + +// runLoop internal state. These fields could mostly be local variables +// in runLoop.run(), but they're exposed here to facilitate testing. In a +// live queue, only the runLoop goroutine should read or write these fields. +type runLoop struct { + broker *broker + + // The index of the beginning of the current ring buffer within its backing + // array. If the queue isn't empty, bufPos points to the oldest remaining + // event. + bufPos int + + // The total number of events in the queue. + eventCount int + + // The number of consumed events waiting for acknowledgment. The next Get + // request will return events starting at position + // (bufPos + consumedCount) % len(buf). + consumedCount int + + // The list of batches that have been consumed and are waiting to be sent + // to ackLoop for acknowledgment handling. (This list doesn't contain all + // outstanding batches, only the ones not yet forwarded to ackLoop.) + consumedBatches batchList + + // If there aren't enough events ready to fill an incoming get request, + // the queue may block based on its flush settings. When this happens, + // pendingGetRequest stores the request until we're ready to handle it. + pendingGetRequest *getRequest + + // This timer tracks the configured flush timeout when we will respond + // to a pending getRequest even if we can't fill the requested event count. + // It is active if and only if pendingGetRequest is non-nil. + getTimer *time.Timer + + // TODO (https://github.com/elastic/beats/issues/37893): entry IDs were a + // workaround for an external project that no longer exists. At this point + // they just complicate the API and should be removed. + nextEntryID queue.EntryID +} + +func newRunLoop(broker *broker) *runLoop { + var timer *time.Timer + + // Create the timer we'll use for get requests, but stop it until a + // get request is active. + if broker.settings.FlushTimeout > 0 { + timer = time.NewTimer(broker.settings.FlushTimeout) + if !timer.Stop() { + <-timer.C + } + } + return &runLoop{ + broker: broker, + getTimer: timer, + } +} + +func (l *runLoop) run() { + for l.broker.ctx.Err() == nil { + l.runIteration() + } +} + +// Perform one iteration of the queue's main run loop. Broken out into a +// standalone helper function to allow testing of loop invariants. +func (l *runLoop) runIteration() { + var pushChan chan pushRequest + // Push requests are enabled if the queue isn't yet full. + if l.eventCount < len(l.broker.buf) { + pushChan = l.broker.pushChan + } + + var getChan chan getRequest + // Get requests are enabled if the queue has events that weren't yet sent + // to consumers, and no existing request is active. + if l.pendingGetRequest == nil && l.eventCount > l.consumedCount { + getChan = l.broker.getChan + } + + var consumedChan chan batchList + // Enable sending to the scheduled ACKs channel if we have + // something to send. + if !l.consumedBatches.empty() { + consumedChan = l.broker.consumedChan + } + + var timeoutChan <-chan time.Time + // Enable the timeout channel if a get request is waiting for events + if l.pendingGetRequest != nil { + timeoutChan = l.getTimer.C + } + + select { + case <-l.broker.ctx.Done(): + return + + case req := <-pushChan: // producer pushing new event + l.handleInsert(&req) + + case req := <-l.broker.cancelChan: // producer cancelling active events + l.handleCancel(&req) + + case req := <-getChan: // consumer asking for next batch + l.handleGetRequest(&req) + + case consumedChan <- l.consumedBatches: + // We've sent all the pending batches to the ackLoop for processing, + // clear the pending list. + l.consumedBatches = batchList{} + + case count := <-l.broker.deleteChan: + l.handleDelete(count) + + case req := <-l.broker.metricChan: // asking broker for queue metrics + l.handleMetricsRequest(&req) + + case <-timeoutChan: + // The get timer has expired, handle the blocked request + l.getTimer.Stop() + l.handleGetReply(l.pendingGetRequest) + l.pendingGetRequest = nil + } +} + +func (l *runLoop) handleGetRequest(req *getRequest) { + if req.entryCount <= 0 || req.entryCount > l.broker.settings.MaxGetRequest { + req.entryCount = l.broker.settings.MaxGetRequest + } + if l.getRequestShouldBlock(req) { + l.pendingGetRequest = req + l.getTimer.Reset(l.broker.settings.FlushTimeout) + return + } + l.handleGetReply(req) +} + +func (l *runLoop) getRequestShouldBlock(req *getRequest) bool { + if l.broker.settings.FlushTimeout <= 0 { + // Never block if the flush timeout isn't positive + return false + } + eventsAvailable := l.eventCount - l.consumedCount + // Block if the available events aren't enough to fill the request + return eventsAvailable < req.entryCount +} + +// Respond to the given get request without blocking or waiting for more events +func (l *runLoop) handleGetReply(req *getRequest) { + eventsAvailable := l.eventCount - l.consumedCount + batchSize := req.entryCount + if eventsAvailable < batchSize { + batchSize = eventsAvailable + } + + startIndex := l.bufPos + l.consumedCount + batch := newBatch(l.broker, startIndex, batchSize) + + // Send the batch to the caller and update internal state + req.responseChan <- batch + l.consumedBatches.append(batch) + l.consumedCount += batchSize +} + +func (l *runLoop) handleDelete(count int) { + // Clear the internal event pointers so they can be garbage collected + for i := 0; i < count; i++ { + index := (l.bufPos + i) % len(l.broker.buf) + l.broker.buf[index].event = nil + } + + // Advance position and counters + l.bufPos = (l.bufPos + count) % len(l.broker.buf) + l.eventCount -= count + l.consumedCount -= count +} + +func (l *runLoop) handleInsert(req *pushRequest) { + if l.insert(req, l.nextEntryID) { + // Send back the new event id. + req.resp <- l.nextEntryID + + l.nextEntryID++ + l.eventCount++ + + // See if this gave us enough for a new batch + l.maybeUnblockGetRequest() + } +} + +// Checks if we can handle pendingGetRequest yet, and handles it if so +func (l *runLoop) maybeUnblockGetRequest() { + // If a get request is blocked waiting for more events, check if + // we should unblock it. + if getRequest := l.pendingGetRequest; getRequest != nil { + available := l.eventCount - l.consumedCount + if available >= getRequest.entryCount { + l.pendingGetRequest = nil + if !l.getTimer.Stop() { + <-l.getTimer.C + } + l.handleGetReply(getRequest) + } + } +} + +// Returns true if the event was inserted, false if insertion was cancelled. +func (l *runLoop) insert(req *pushRequest, id queue.EntryID) bool { + if req.producer != nil && req.producer.state.cancelled { + reportCancelledState(req) + return false + } + + index := (l.bufPos + l.eventCount) % len(l.broker.buf) + l.broker.buf[index] = queueEntry{ + event: req.event, + id: id, + producer: req.producer, + producerID: req.producerID, + } + return true +} + +func (l *runLoop) handleMetricsRequest(req *metricsRequest) { + oldestEntryID := l.nextEntryID + if l.eventCount > 0 { + index := l.bufPos % len(l.broker.buf) + oldestEntryID = l.broker.buf[index].id + } + + req.responseChan <- memQueueMetrics{ + currentQueueSize: l.eventCount, + occupiedRead: l.consumedCount, + oldestEntryID: oldestEntryID, + } +} + +func (l *runLoop) handleCancel(req *producerCancelRequest) { + var removedCount int + + // Traverse all unconsumed events in the buffer, removing any with + // the specified producer. As we go we condense all the remaining + // events to be sequential. + buf := l.broker.buf + startIndex := l.bufPos + l.consumedCount + unconsumedEventCount := l.eventCount - l.consumedCount + for i := 0; i < unconsumedEventCount; i++ { + readIndex := (startIndex + i) % len(buf) + if buf[readIndex].producer == req.producer { + // The producer matches, skip this event + removedCount++ + } else { + // Move the event to its final position after accounting for any + // earlier indices that were removed. + // (Count backwards from (startIndex + i), not from readIndex, to avoid + // sign issues when the buffer wraps.) + writeIndex := (startIndex + i - removedCount) % len(buf) + buf[writeIndex] = buf[readIndex] + } + } + + // Clear the event pointers at the end of the buffer so we don't keep + // old events in memory by accident. + for i := 0; i < removedCount; i++ { + index := (l.bufPos + l.eventCount - removedCount + i) % len(buf) + buf[index].event = nil + } + + // Subtract removed events from the internal event count + l.eventCount -= removedCount + + // signal cancel request being finished + if req.resp != nil { + req.resp <- producerCancelResponse{removed: removedCount} + } +} + +func reportCancelledState(req *pushRequest) { + // do not add waiting events if producer did send cancel signal + if cb := req.producer.state.dropCB; cb != nil { + cb(req.event) + } +} diff --git a/libbeat/publisher/queue/memqueue/runloop_test.go b/libbeat/publisher/queue/memqueue/runloop_test.go new file mode 100644 index 00000000000..9b3a467647a --- /dev/null +++ b/libbeat/publisher/queue/memqueue/runloop_test.go @@ -0,0 +1,114 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memqueue + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestFlushSettingsDoNotBlockFullBatches(t *testing.T) { + // In previous versions of the queue, setting flush.min_events (currently + // corresponding to memqueue.Settings.MaxGetRequest) to a high value would + // delay get requests even if the number of requested events was immediately + // available. This test verifies that Get requests that can be completely + // filled do not wait for the flush timer. + + broker := newQueue( + logp.NewLogger("testing"), + nil, + Settings{ + Events: 1000, + MaxGetRequest: 500, + FlushTimeout: 10 * time.Second, + }, + 10) + + producer := newProducer(broker, nil, nil, false) + rl := broker.runLoop + for i := 0; i < 100; i++ { + // Pair each publish call with an iteration of the run loop so we + // get a response. + go rl.runIteration() + _, ok := producer.Publish(i) + require.True(t, ok, "Queue publish call must succeed") + } + + // The queue now has 100 events, but MaxGetRequest is 500. + // In the old queue, a Get call now would block until the flush + // timer expires. With current changes, it should return + // immediately on any request size up to 100. + go func() { + // Run the Get asynchronously so the test itself doesn't block if + // there's a logical error. + _, _ = broker.Get(100) + }() + rl.runIteration() + assert.Nil(t, rl.pendingGetRequest, "Queue should have no pending get request since the request should succeed immediately") + assert.Equal(t, 100, rl.consumedCount, "Queue should have a consumedCount of 100 after a consumer requested all its events") +} + +func TestFlushSettingsBlockPartialBatches(t *testing.T) { + // The previous test confirms that Get requests are handled immediately if + // there are enough events. This one uses the same setup to confirm that + // Get requests are delayed if there aren't enough events. + + broker := newQueue( + logp.NewLogger("testing"), + nil, + Settings{ + Events: 1000, + MaxGetRequest: 500, + FlushTimeout: 10 * time.Second, + }, + 10) + + producer := newProducer(broker, nil, nil, false) + rl := broker.runLoop + for i := 0; i < 100; i++ { + // Pair each publish call with an iteration of the run loop so we + // get a response. + go rl.runIteration() + _, ok := producer.Publish("some event") + require.True(t, ok, "Queue publish call must succeed") + } + + // The queue now has 100 events, and a positive flush timeout, so a + // request for 101 events should block. + go func() { + // Run the Get asynchronously so the test itself doesn't block if + // there's a logical error. + _, _ = broker.Get(101) + }() + rl.runIteration() + assert.NotNil(t, rl.pendingGetRequest, "Queue should have a pending get request since the queue doesn't have the requested event count") + assert.Equal(t, 0, rl.consumedCount, "Queue should have a consumedCount of 0 since the Get request couldn't be completely filled") + + // Now confirm that adding one more event unblocks the request + go func() { + _, _ = producer.Publish("some event") + }() + rl.runIteration() + assert.Nil(t, rl.pendingGetRequest, "Queue should have no pending get request since adding an event should unblock the previous one") + assert.Equal(t, 101, rl.consumedCount, "Queue should have a consumedCount of 101 after adding an event unblocked the pending get request") +} diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 100ccd3f013..4360aa0c192 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -46,7 +46,7 @@ export PATH := ./bin:$(PATH) GOFILES = $(shell find . -type f -name '*.go' 2>/dev/null) GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "*/vendor/*" 2>/dev/null) GOFILES_ALL = $(GOFILES) $(shell find $(ES_BEATS) -type f -name '*.go' 2>/dev/null) -GOPACKAGES_STRESSTESTS=$(shell find . -name '*.go' 2>/dev/null | xargs grep -l '\+build.*stresstest' | xargs -n1 dirname | uniq) +GOPACKAGES_STRESSTESTS=$(shell find . -type d \( -name "stress" \) 2>/dev/null) SHELL=bash ES_HOST?=elasticsearch ES_PORT?=9200 @@ -87,7 +87,7 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/arm64 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/arm !darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 !linux/ppc64 ## @building Space-separated list of GOOS/GOARCH pairs to exclude (unsupported by GO and generated by GOX) in the "make crosscompile" build. GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 diff --git a/libbeat/tests/integration/cmd_keystore_test.go b/libbeat/tests/integration/cmd_keystore_test.go index eb4b697cafa..efb9b91a1c9 100644 --- a/libbeat/tests/integration/cmd_keystore_test.go +++ b/libbeat/tests/integration/cmd_keystore_test.go @@ -100,19 +100,23 @@ func TestKeystoreRemoveMultipleExistingKeys(t *testing.T) { mockbeat.Stop() mockbeat.Start("keystore", "add", "key1", "--stdin") - fmt.Fprintf(os.Stdin, "pass1") + + fmt.Fprintf(mockbeat.stdin, "pass1") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err := mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") mockbeat.Start("keystore", "add", "key2", "--stdin") - fmt.Fprintf(os.Stdin, "pass2") + fmt.Fprintf(mockbeat.stdin, "pass2") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err = mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") mockbeat.Start("keystore", "add", "key3", "--stdin") - fmt.Fprintf(os.Stdin, "pass3") + fmt.Fprintf(mockbeat.stdin, "pass3") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err = mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") @@ -138,19 +142,22 @@ func TestKeystoreList(t *testing.T) { mockbeat.Stop() mockbeat.Start("keystore", "add", "key1", "--stdin") - fmt.Fprintf(os.Stdin, "pass1") + fmt.Fprintf(mockbeat.stdin, "pass1") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err := mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") mockbeat.Start("keystore", "add", "key2", "--stdin") - fmt.Fprintf(os.Stdin, "pass2") + fmt.Fprintf(mockbeat.stdin, "pass2") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err = mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") mockbeat.Start("keystore", "add", "key3", "--stdin") - fmt.Fprintf(os.Stdin, "pass3") + fmt.Fprintf(mockbeat.stdin, "pass3") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err = mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") @@ -186,7 +193,8 @@ func TestKeystoreAddSecretFromStdin(t *testing.T) { require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") mockbeat.Start("keystore", "add", "key1", "--stdin") - fmt.Fprintf(os.Stdin, "pass1") + fmt.Fprintf(mockbeat.stdin, "pass1") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err = mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") @@ -202,13 +210,15 @@ func TestKeystoreUpdateForce(t *testing.T) { require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") mockbeat.Start("keystore", "add", "key1", "--stdin") - fmt.Fprintf(os.Stdin, "pass1") + fmt.Fprintf(mockbeat.stdin, "pass1") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err = mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") mockbeat.Start("keystore", "add", "key1", "--force", "--stdin") - fmt.Fprintf(os.Stdin, "pass2") + fmt.Fprintf(mockbeat.stdin, "pass2") + require.NoError(t, mockbeat.stdin.Close(), "could not close mockbeat stdin") procState, err = mockbeat.Process.Wait() require.NoError(t, err) require.Equal(t, 0, procState.ExitCode(), "incorrect exit code") diff --git a/libbeat/tests/integration/framework.go b/libbeat/tests/integration/framework.go index 046c578d7cd..9657fbaeaff 100644 --- a/libbeat/tests/integration/framework.go +++ b/libbeat/tests/integration/framework.go @@ -30,6 +30,7 @@ import ( "net/http" "net/url" "os" + "os/exec" "path/filepath" "regexp" "strings" @@ -55,6 +56,7 @@ type BeatProc struct { logFileOffset int64 t *testing.T tempDir string + stdin io.WriteCloser stdout *os.File stderr *os.File Process *os.Process @@ -90,7 +92,7 @@ type Total struct { Value int `json:"value"` } -// NewBeat createa a new Beat process from the system tests binary. +// NewBeat creates a new Beat process from the system tests binary. // It sets some required options like the home path, logging, etc. // `tempDir` will be used as home and logs directory for the Beat // `args` will be passed as CLI arguments to the Beat @@ -98,10 +100,12 @@ func NewBeat(t *testing.T, beatName, binary string, args ...string) *BeatProc { require.FileExistsf(t, binary, "beat binary must exists") tempDir := createTempDir(t) configFile := filepath.Join(tempDir, beatName+".yml") + stdoutFile, err := os.Create(filepath.Join(tempDir, "stdout")) require.NoError(t, err, "error creating stdout file") stderrFile, err := os.Create(filepath.Join(tempDir, "stderr")) require.NoError(t, err, "error creating stderr file") + p := BeatProc{ Binary: binary, baseArgs: append([]string{ @@ -213,15 +217,27 @@ func (b *BeatProc) Start(args ...string) { func (b *BeatProc) startBeat() { b.cmdMutex.Lock() defer b.cmdMutex.Unlock() + _, _ = b.stdout.Seek(0, 0) _ = b.stdout.Truncate(0) _, _ = b.stderr.Seek(0, 0) _ = b.stderr.Truncate(0) - var procAttr os.ProcAttr - procAttr.Files = []*os.File{os.Stdin, b.stdout, b.stderr} - process, err := os.StartProcess(b.fullPath, b.Args, &procAttr) + + cmd := exec.Cmd{ + Path: b.fullPath, + Args: b.Args, + Stdout: b.stdout, + Stderr: b.stderr, + } + + var err error + b.stdin, err = cmd.StdinPipe() + require.NoError(b.t, err, "could not get cmd StdinPipe") + + err = cmd.Start() require.NoError(b.t, err, "error starting beat process") - b.Process = process + + b.Process = cmd.Process } // waitBeatToExit blocks until the Beat exits, it returns @@ -515,6 +531,10 @@ func (b *BeatProc) LoadMeta() (Meta, error) { return m, nil } +func (b *BeatProc) Stdin() io.WriteCloser { + return b.stdin +} + func GetESURL(t *testing.T, scheme string) url.URL { t.Helper() diff --git a/libbeat/tests/integration/mockserver.go b/libbeat/tests/integration/mockserver.go index 0a396cb7839..763467819fa 100644 --- a/libbeat/tests/integration/mockserver.go +++ b/libbeat/tests/integration/mockserver.go @@ -38,18 +38,18 @@ type unitKey struct { } // NewMockServer creates a GRPC server to mock the Elastic-Agent. -// On the first check in call it will send the first element of `unit` +// On the first check-in call it will send the first element of `unit` // as the expected unit, on successive calls, if the Beat has reached // that state, it will move on to sending the next state. // It will also validate the features. // // if `observedCallback` is not nil, it will be called on every -// check in receiving the `proto.CheckinObserved` sent by the +// check-in receiving the `proto.CheckinObserved` sent by the // Beat and index from `units` that was last sent to the Beat. // // If `delay` is not zero, when the Beat state matches the last // sent units, the server will wait for `delay` before sending the -// the next state. This will block the check in call from the Beat. +// next state. This will block the check-in call from the Beat. func NewMockServer( units [][]*proto.UnitExpected, featuresIdxs []uint64, @@ -58,7 +58,7 @@ func NewMockServer( delay time.Duration, ) *mock.StubServerV2 { i := 0 - agentInfo := &proto.CheckinAgentInfo{ + agentInfo := &proto.AgentInfo{ Id: "elastic-agent-id", Version: version.GetDefaultVersion(), Snapshot: true, diff --git a/libbeat/version/helper.go b/libbeat/version/helper.go index 5ed206d8a6c..92b2ed2cb4c 100644 --- a/libbeat/version/helper.go +++ b/libbeat/version/helper.go @@ -17,23 +17,36 @@ package version -import "time" +import ( + "sync/atomic" + "time" +) + +var ( + packageVersion atomic.Value + buildTime = "unknown" + commit = "unknown" + qualifier = "" +) -// GetDefaultVersion returns the current libbeat version. -// This method is in a separate file as the version.go file is auto generated +// GetDefaultVersion returns the current version. +// If running in stand-alone mode, it's the libbeat version. If running in +// managed mode, a.k.a under the agent, it's the package version set using +// SetPackageVersion. If SetPackageVersion haven't been called, it reports the +// libbeat version +// +// This method is in a separate file as the version.go file is auto-generated. func GetDefaultVersion() string { + if v, ok := packageVersion.Load().(string); ok && v != "" { + return v + } + if qualifier == "" { return defaultBeatVersion } return defaultBeatVersion + "-" + qualifier } -var ( - buildTime = "unknown" - commit = "unknown" - qualifier = "" -) - // BuildTime exposes the compile-time build time information. // It will represent the zero time instant if parsing fails. func BuildTime() time.Time { @@ -48,3 +61,10 @@ func BuildTime() time.Time { func Commit() string { return commit } + +// SetPackageVersion sets the package version, overriding the defaultBeatVersion. +func SetPackageVersion(version string) { + // Currently, the Elastic Agent does not perform any validation on the + // package version, therefore, no validation is done here either. + packageVersion.Store(version) +} diff --git a/libbeat/version/version.go b/libbeat/version/version.go index 48850647a6d..e1c849d3bda 100644 --- a/libbeat/version/version.go +++ b/libbeat/version/version.go @@ -18,4 +18,4 @@ // Code generated by dev-tools/set_version package version -const defaultBeatVersion = "8.13.0" +const defaultBeatVersion = "8.14.0" diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index baf372f1859..31f13aeea2c 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.6 +FROM golang:1.21.7 RUN \ apt update \ diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 03e0b65272e..ab1ee57979f 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -17,11 +17,11 @@ services: # Used by base tests elasticsearch: - image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.11.2}-1 + image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.12.1}-1 build: context: ./module/elasticsearch/_meta args: - ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-8.11.2} + ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-8.12.1} environment: - "ES_JAVA_OPTS=-Xms256m -Xmx256m" - "transport.host=127.0.0.1" @@ -38,11 +38,11 @@ services: # Used by base tests kibana: - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.11.2}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.12.1}-1 build: context: ./module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-8.11.2} + KIBANA_VERSION: ${KIBANA_VERSION:-8.12.1} healthcheck: test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 @@ -53,11 +53,11 @@ services: # Used by base tests metricbeat: - image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-8.11.2}-1 + image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-8.12.1}-1 build: context: ./module/beat/_meta args: - BEAT_VERSION: ${BEAT_VERSION:-8.11.2} + BEAT_VERSION: ${BEAT_VERSION:-8.12.1} command: '-e' ports: - 5066:5066 diff --git a/metricbeat/docs/modules/oracle.asciidoc b/metricbeat/docs/modules/oracle.asciidoc index f524967cce5..3436caa9cc2 100644 --- a/metricbeat/docs/modules/oracle.asciidoc +++ b/metricbeat/docs/modules/oracle.asciidoc @@ -60,19 +60,24 @@ Then, Metricbeat can be launched. *Host Configuration* -The following two types of host configurations are supported: +The following types of host configuration are supported: -1. Old style host configuration for backwards compatibility: +1. An old-style Oracle connection string, for backwards compatibility: a. `hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"]` b. `hosts: ["user/password@0.0.0.0:1521/ORCLPDB1.localdomain as sysdba"]` -2. DSN host configuration: +2. DSN configuration as a URL: + a. `hosts: ["oracle://user:pass@0.0.0.0:1521/ORCLPDB1.localdomain?sysdba=1"]` + +3. DSN configuration as a logfmt-encoded parameter list: a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']` b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']` -DSN host configuration is the recommended way to configure the Oracle Metricbeat Module as it supports the usage of special characters in the password. +DSN host configuration is the recommended configuration type as it supports the use of special characters in the password. + +In a URL any special characters should be URL encoded. -Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`. +In the logfmt-encoded DSN format, if the password contains a backslash character (`\`), it must be escaped with another backslash. For example, if the password is `my\_password`, it must be written as `my\\_password`. [float] == Metricsets diff --git a/metricbeat/docs/modules/sql.asciidoc b/metricbeat/docs/modules/sql.asciidoc index 9c27c0bc4ba..d8e0e15b617 100644 --- a/metricbeat/docs/modules/sql.asciidoc +++ b/metricbeat/docs/modules/sql.asciidoc @@ -871,19 +871,26 @@ Then, Metricbeat can be launched. ===== Host Configuration for Oracle -The following two types of host configurations are supported: +The following types of host configuration are supported: -1. DSN host configuration as URL: +1. An old-style Oracle connection string, for backwards compatibility: a. `hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"]` b. `hosts: ["user/password@0.0.0.0:1521/ORCLPDB1.localdomain as sysdba"]` -2. DSN host configuration: +2. DSN configuration as a URL: + a. `hosts: ["oracle://user:pass@0.0.0.0:1521/ORCLPDB1.localdomain?sysdba=1"]` + +3. DSN configuration as a logfmt-encoded parameter list: a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']` b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']` -Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`. +DSN host configuration is the recommended configuration type as it supports the use of special characters in the password. + +In a URL any special characters should be URL encoded. -The username and password to connect to the database can be provided as values to `username` and `password` keys of `sql.yml`. +In the logfmt-encoded DSN format, if the password contains a backslash character (`\`), it must be escaped with another backslash. For example, if the password is `my\_password`, it must be written as `my\\_password`. + +The username and password to connect to the database can be provided as values to the `username` and `password` keys of `sql.yml`. [source,yml] ---- @@ -901,6 +908,7 @@ The username and password to connect to the database can be provided as values t response_format: variables ---- + :edit_url: [float] diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 2fcad7f2cf1..e2ac0be9664 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -70,17 +70,17 @@ in the manifest file: ------------------------------------------------ [float] -===== Running {beatname_uc} on master nodes +===== Running {beatname_uc} on control plane nodes -Kubernetes master nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] -to limit the workloads that can run on them. To run {beatname_uc} on master nodes you may need to +Kubernetes control plane nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] +to limit the workloads that can run on them. To run {beatname_uc} on control plane nodes you may need to update the Daemonset spec to include proper tolerations: [source,yaml] ------------------------------------------------ spec: tolerations: - - key: node-role.kubernetes.io/master + - key: node-role.kubernetes.io/control-plane effect: NoSchedule ------------------------------------------------ @@ -166,7 +166,7 @@ oc patch namespace kube-system -p \ ---- + This command sets the node selector for the project to an empty string. If you -don't run this command, the default node selector will skip master nodes. +don't run this command, the default node selector will skip control plane nodes. NOTE: for openshift versions prior to the version 4.x additionally you need to modify the `DaemonSet` container spec in the manifest file to enable the container to run as privileged: [source,yaml] diff --git a/metricbeat/helper/socket/ptable_linux.go b/metricbeat/helper/socket/ptable_linux.go index 88fff488bc2..ffe585f7094 100644 --- a/metricbeat/helper/socket/ptable_linux.go +++ b/metricbeat/helper/socket/ptable_linux.go @@ -20,17 +20,22 @@ package socket import ( - "github.com/elastic/beats/v7/libbeat/common" + "kernel.org/pub/linux/libs/security/libcap/cap" ) -var requiredCapabilities = []string{"sys_ptrace", "dac_read_search"} - // isPrivileged checks if this process has privileges to read sockets // of all users func isPrivileged() (bool, error) { - capabilities, err := common.GetCapabilities() + set := cap.GetProc() + + ptrace, err := set.GetFlag(cap.Effective, cap.SYS_PTRACE) + if err != nil { + return false, err + } + dac_read_search, err := set.GetFlag(cap.Effective, cap.DAC_READ_SEARCH) if err != nil { return false, err } - return capabilities.Check(requiredCapabilities), nil + + return ptrace && dac_read_search, nil } diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index d6b8b9e9475..6659ca29276 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2201,9 +2201,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/metricbeat/module/elasticsearch/ingest_pipeline/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/ingest_pipeline/_meta/docs.asciidoc index c387a6ee085..a02a76a6326 100644 --- a/metricbeat/module/elasticsearch/ingest_pipeline/_meta/docs.asciidoc +++ b/metricbeat/module/elasticsearch/ingest_pipeline/_meta/docs.asciidoc @@ -17,6 +17,6 @@ processor-level metrics will be collected during 25% of the time. This can be co - module: elasticsearch period: 10s metricsets: - - ingest + - ingest_pipeline ingest.processor_sample_rate: 0.1 # decrease to 10% of fetches ---- diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 55df10b1294..a46a3dbb3e2 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.6 +FROM golang:1.21.7 COPY test/main.go main.go diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index ef6e428ae87..4df0b81a912 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.21.6 AS build-env +FROM golang:1.21.7 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index 1f0881c14ec..3db3cccbab2 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.21.6 +FROM golang:1.21.7 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/metricbeat/tests/system/test_reload.py b/metricbeat/tests/system/test_reload.py index 29d82bbf82b..99aa8e2c2f2 100644 --- a/metricbeat/tests/system/test_reload.py +++ b/metricbeat/tests/system/test_reload.py @@ -42,7 +42,8 @@ def test_reload(self): self.wait_until(lambda: self.output_lines() > 0) proc.check_kill_and_wait() - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + # windows is disabled, see https://github.com/elastic/beats/issues/37841 + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os") def test_start_stop(self): """ Test if module is properly started and stopped diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 6e5c1d0bab4..17075d9da65 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.6 +FROM golang:1.21.7 RUN \ apt-get update \ diff --git a/packetbeat/_meta/config/beat.reference.yml.tmpl b/packetbeat/_meta/config/beat.reference.yml.tmpl index 649ec0e8dee..033aa1e5106 100644 --- a/packetbeat/_meta/config/beat.reference.yml.tmpl +++ b/packetbeat/_meta/config/beat.reference.yml.tmpl @@ -78,6 +78,11 @@ packetbeat.interfaces.internal_networks: # can stay enabled even after beat is shut down. #packetbeat.interfaces.auto_promisc_mode: true +# By default Ingest pipelines are not updated if a pipeline with the same ID +# already exists. If this option is enabled Packetbeat overwrites pipelines +# every time a new Elasticsearch connection is established. +#packetbeat.overwrite_pipelines: false + {{- template "windows_npcap.yml.tmpl" .}} {{header "Flows"}} diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go index 725f3eebc33..d8c223f1789 100644 --- a/packetbeat/beater/packetbeat.go +++ b/packetbeat/beater/packetbeat.go @@ -25,13 +25,16 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" + "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/service" "github.com/elastic/beats/v7/packetbeat/config" + "github.com/elastic/beats/v7/packetbeat/module" "github.com/elastic/beats/v7/packetbeat/protos" // Add packetbeat default processors @@ -80,10 +83,11 @@ func initialConfig() config.Config { // Beater object. Contains all objects needed to run the beat type packetbeat struct { - config *conf.C - factory *processorFactory - done chan struct{} - stopOnce sync.Once + config *conf.C + factory *processorFactory + overwritePipelines bool + done chan struct{} + stopOnce sync.Once } // New returns a new Packetbeat beat.Beater. @@ -98,15 +102,35 @@ func New(b *beat.Beat, rawConfig *conf.C) (beat.Beater, error) { return nil, err } + var overwritePipelines bool + if !b.Manager.Enabled() { + // Pipeline overwrite is only enabled on standalone packetbeat + // since pipelines are managed by fleet otherwise. + config, err := configurator(rawConfig) + if err != nil { + return nil, err + } + overwritePipelines = config.OverwritePipelines + b.OverwritePipelinesCallback = func(esConfig *conf.C) error { + esClient, err := eslegclient.NewConnectedClient(esConfig, "Packetbeat") + if err != nil { + return err + } + _, err = module.UploadPipelines(b.Info, esClient, overwritePipelines) + return err + } + } + return &packetbeat{ - config: rawConfig, - factory: factory, - done: make(chan struct{}), + config: rawConfig, + factory: factory, + overwritePipelines: overwritePipelines, + done: make(chan struct{}), }, nil } // Run starts the packetbeat network capture, decoding and event publication, sending -// events to b.Publisher. If b is mananaged, packetbeat is registered with the +// events to b.Publisher. If b is managed, packetbeat is registered with the // reload.Registry and handled by fleet. Otherwise it is run until cancelled or a // fatal error. func (pb *packetbeat) Run(b *beat.Beat) error { @@ -138,11 +162,28 @@ func (pb *packetbeat) Run(b *beat.Beat) error { } if !b.Manager.Enabled() { + if b.Config.Output.Name() == "elasticsearch" { + _, err := elasticsearch.RegisterConnectCallback(func(esClient *eslegclient.Connection) error { + _, err := module.UploadPipelines(b.Info, esClient, pb.overwritePipelines) + return err + }) + if err != nil { + return err + } + } else { + logp.L().Warn(pipelinesWarning) + } + return pb.runStatic(b, pb.factory) } return pb.runManaged(b, pb.factory) } +const pipelinesWarning = "Packetbeat is unable to load the ingest pipelines for the configured" + + " modules because the Elasticsearch output is not configured/enabled. If you have" + + " already loaded the ingest pipelines or are using Logstash pipelines, you" + + " can ignore this warning." + // runStatic constructs a packetbeat runner and starts it, returning on cancellation // or the first fatal error. func (pb *packetbeat) runStatic(b *beat.Beat, factory *processorFactory) error { diff --git a/packetbeat/config/config.go b/packetbeat/config/config.go index 13d00b89e44..7d579af635b 100644 --- a/packetbeat/config/config.go +++ b/packetbeat/config/config.go @@ -33,14 +33,15 @@ import ( var errFanoutGroupAFPacketOnly = errors.New("fanout_group is only valid with af_packet type") type Config struct { - Interface *InterfaceConfig `config:"interfaces"` - Interfaces []InterfaceConfig `config:"interfaces"` - Flows *Flows `config:"flows"` - Protocols map[string]*conf.C `config:"protocols"` - ProtocolsList []*conf.C `config:"protocols"` - Procs procs.ProcsConfig `config:"procs"` - IgnoreOutgoing bool `config:"ignore_outgoing"` - ShutdownTimeout time.Duration `config:"shutdown_timeout"` + Interface *InterfaceConfig `config:"interfaces"` + Interfaces []InterfaceConfig `config:"interfaces"` + Flows *Flows `config:"flows"` + Protocols map[string]*conf.C `config:"protocols"` + ProtocolsList []*conf.C `config:"protocols"` + Procs procs.ProcsConfig `config:"procs"` + IgnoreOutgoing bool `config:"ignore_outgoing"` + ShutdownTimeout time.Duration `config:"shutdown_timeout"` + OverwritePipelines bool `config:"overwrite_pipelines"` // Only used by standalone Packetbeat. } // FromStatic initializes a configuration given a config.C diff --git a/packetbeat/docs/howto/howto.asciidoc b/packetbeat/docs/howto/howto.asciidoc index cdadf3cb7b3..b7284ab3024 100644 --- a/packetbeat/docs/howto/howto.asciidoc +++ b/packetbeat/docs/howto/howto.asciidoc @@ -23,6 +23,8 @@ include::{libbeat-dir}/howto/load-dashboards.asciidoc[] include::{libbeat-dir}/shared-geoip.asciidoc[] +include::load-ingest-pipelines.asciidoc[] + :standalone: include::{libbeat-dir}/shared-env-vars.asciidoc[] :standalone!: diff --git a/packetbeat/docs/howto/load-ingest-pipelines.asciidoc b/packetbeat/docs/howto/load-ingest-pipelines.asciidoc new file mode 100644 index 00000000000..acca824829c --- /dev/null +++ b/packetbeat/docs/howto/load-ingest-pipelines.asciidoc @@ -0,0 +1,28 @@ +[[load-ingest-pipelines]] +== Load ingest pipelines + +{beatname_uc} modules are implemented using {es} ingest node +pipelines. The events receive their transformations within +{es}. The ingest node pipelines must be loaded +into {es}. This can happen one of several ways. + +[id="{beatname_lc}-load-pipeline-auto"] +[float] +=== On connection to {es} + +{beatname_uc} will send ingest pipelines automatically to {es} if the +{es} output is enabled. + +Make sure the user specified in +{beatname_lc}.yml+ is +<>. + +If {beatname_uc} is sending events to {ls} or another output you need +to load the ingest pipelines with the `setup` command or manually. + +[id="{beatname_lc}-load-pipeline-manual"] +[float] +=== Manually install pipelines + +Pipelines can be loaded them into {es} with the `_ingest/pipeline` REST API +call. The user making the REST API call will need to have the `ingest_admin` +role assigned to them. diff --git a/packetbeat/docs/modules.asciidoc b/packetbeat/docs/modules.asciidoc new file mode 100644 index 00000000000..8e72454f9cf --- /dev/null +++ b/packetbeat/docs/modules.asciidoc @@ -0,0 +1,41 @@ +[id="{beatname_lc}-modules"] +[role="xpack"] += Modules + +[partintro] +-- +This section contains detailed information about the available network packet +log processing modules contained in {beatname_uc}. + +{beatname_uc} modules are implemented using Elasticsearch Ingest Node pipelines. +The events receive their transformations within Elasticsearch. All events are +sent through {beatname_uc}'s "routing" pipeline that routes events to specific +module pipelines based on their network protocol. + +{beatname_uc}'s default config file contains the option to send all events to +the routing pipeline. If you remove this option then the module processing +will not be applied. + +[source,yaml,subs="attributes"] +---- +output.elasticsearch.pipeline: packetbeat-%{[agent.version]}-routing +---- + +The general goal of each module is to transform events by renaming fields to +comply with the {ecs-ref}/index.html[Elastic Common Schema] (ECS). The modules +may also apply additional categorization, tagging, and parsing as necessary. +about how to configure the language in `packetbeat`, refer to <>. + +[id="{beatname_lc}-modules-setup"] +[float] +=== Setup of Ingest Node pipelines + +{beatname_uc}'s Ingest Node pipelines must be installed to Elasticsearch if you +want to apply the module processing to events. The simplest way to get started +is to use the Elasticsearch output and {beatname_uc} will automatically install +the pipelines when it first connects to Elasticsearch. + +Installation Methods + +1. <<{beatname_lc}-load-pipeline-auto>> +2. <<{beatname_lc}-load-pipeline-manual>> diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index c5cb4d95d6b..c48b4a1b01d 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -1650,3 +1650,12 @@ Example configuration: ------------------------------------------------------------------------------------- packetbeat.shutdown_timeout: 5s ------------------------------------------------------------------------------------- + +[float] +==== `overwrite_pipelines` + +By default Ingest pipelines are not updated if a pipeline with the same ID +already exists. If this option is enabled {beatname_uc} overwrites pipelines +every time a new Elasticsearch connection is established. + +The default value is `false`. diff --git a/packetbeat/docs/protocol-metrics-packetbeat.asciidoc b/packetbeat/docs/protocol-metrics-packetbeat.asciidoc index d36cce38e44..1b84eeb37b6 100644 --- a/packetbeat/docs/protocol-metrics-packetbeat.asciidoc +++ b/packetbeat/docs/protocol-metrics-packetbeat.asciidoc @@ -2,7 +2,7 @@ === Protocol-Specific Metrics Packetbeat exposes per-protocol metrics under the <>. -These metrics are exposed under the `/inputs` path. They can be used to +These metrics are exposed under the `/inputs/` path. They can be used to observe the activity of Packetbeat for the monitored protocol. [float] diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 50c8a19310c..00e4f9dd47b 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -29,19 +29,20 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/target/build" packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage" - // mage:import + //mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" - // mage:import + //mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" - // mage:import + //mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/notests" - // mage:import + //mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) func init() { common.RegisterCheckDeps(Update) unittest.RegisterPythonTestDeps(packetbeat.FieldsYML, Dashboards) + packetbeat.SelectLogic = devtools.OSSProject devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch." } diff --git a/packetbeat/module/pipeline.go b/packetbeat/module/pipeline.go new file mode 100644 index 00000000000..9e6d2384938 --- /dev/null +++ b/packetbeat/module/pipeline.go @@ -0,0 +1,188 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package module + +import ( + "embed" + "encoding/json" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/joeshaw/multierror" + "gopkg.in/yaml.v2" + + "github.com/elastic/beats/v7/filebeat/fileset" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" + "github.com/elastic/elastic-agent-libs/logp" +) + +// PipelinesFS is used from the x-pack/packetbeat code to inject modules. The +// OSS version does not have modules. +var PipelinesFS *embed.FS + +var errNoFS = errors.New("no embedded file system") + +const logName = "pipeline" + +type pipeline struct { + id string + contents map[string]interface{} +} + +// UploadPipelines reads all pipelines embedded in the Packetbeat executable +// and adapts the pipeline for a given ES version, converts to JSON if +// necessary and creates or updates ingest pipeline in ES. The IDs of pipelines +// uploaded to ES are returned in loaded. +func UploadPipelines(info beat.Info, esClient *eslegclient.Connection, overwritePipelines bool) (loaded []string, err error) { + pipelines, err := readAll(info) + if err != nil { + return nil, err + } + return load(esClient, pipelines, overwritePipelines) +} + +// readAll reads pipelines from the the embedded filesystem and +// returns a slice of pipelines suitable for sending to Elasticsearch +// with load. +func readAll(info beat.Info) (pipelines []pipeline, err error) { + p, err := readDir(".", info) + if err == errNoFS { //nolint:errorlint // Bad linter! This is never wrapped. + return nil, nil + } + return p, err +} + +func readDir(dir string, info beat.Info) (pipelines []pipeline, err error) { + if PipelinesFS == nil { + return nil, errNoFS + } + dirEntries, err := PipelinesFS.ReadDir(dir) + if err != nil { + return nil, err + } + for _, de := range dirEntries { + if de.IsDir() { + subPipelines, err := readDir(path.Join(dir, de.Name()), info) + if err != nil { + return nil, err + } + pipelines = append(pipelines, subPipelines...) + continue + } + p, err := readFile(path.Join(dir, de.Name()), info) + if err == errNoFS { //nolint:errorlint // Bad linter! This is never wrapped. + continue + } + if err != nil { + return nil, err + } + pipelines = append(pipelines, p) + } + return pipelines, nil +} + +func readFile(filename string, info beat.Info) (p pipeline, err error) { + if PipelinesFS == nil { + return pipeline{}, errNoFS + } + contents, err := PipelinesFS.ReadFile(filename) + if err != nil { + return pipeline{}, err + } + updatedContent, err := applyTemplates(info.IndexPrefix, info.Version, filename, contents) + if err != nil { + return pipeline{}, err + } + ds, _, _ := strings.Cut(filename, string(os.PathSeparator)) + p = pipeline{ + id: fileset.FormatPipelineID(info.IndexPrefix, "", "", ds, info.Version), + contents: updatedContent, + } + return p, nil +} + +// load uses esClient to load pipelines to Elasticsearch cluster. +// The IDs of loaded pipelines will be returned in loaded. +// load will only overwrite existing pipelines if overwritePipelines is +// true. An error in loading one of the pipelines will cause the +// successfully loaded ones to be deleted. +func load(esClient *eslegclient.Connection, pipelines []pipeline, overwritePipelines bool) (loaded []string, err error) { + log := logp.NewLogger(logName) + + for _, pipeline := range pipelines { + err = fileset.LoadPipeline(esClient, pipeline.id, pipeline.contents, overwritePipelines, log) + if err != nil { + err = fmt.Errorf("error loading pipeline %s: %w", pipeline.id, err) + break + } + loaded = append(loaded, pipeline.id) + } + + if err != nil { + errs := multierror.Errors{err} + for _, id := range loaded { + err = fileset.DeletePipeline(esClient, id) + if err != nil { + errs = append(errs, err) + } + } + return nil, errs.Err() + } + return loaded, nil +} + +func applyTemplates(prefix string, version string, filename string, original []byte) (converted map[string]interface{}, err error) { + vars := map[string]interface{}{ + "builtin": map[string]interface{}{ + "prefix": prefix, + "module": "", + "fileset": "", + "beatVersion": version, + }, + } + + encodedString, err := fileset.ApplyTemplate(vars, string(original), true) + if err != nil { + return nil, fmt.Errorf("failed to apply template: %w", err) + } + + var content map[string]interface{} + switch extension := strings.ToLower(filepath.Ext(filename)); extension { + case ".json": + if err = json.Unmarshal([]byte(encodedString), &content); err != nil { + return nil, fmt.Errorf("error JSON decoding the pipeline file: %s: %w", filename, err) + } + case ".yaml", ".yml": + if err = yaml.Unmarshal([]byte(encodedString), &content); err != nil { + return nil, fmt.Errorf("error YAML decoding the pipeline file: %s: %w", filename, err) + } + newContent, err := fileset.FixYAMLMaps(content) + if err != nil { + return nil, fmt.Errorf("failed to sanitize the YAML pipeline file: %s: %w", filename, err) + } + content = newContent.(map[string]interface{}) + default: + return nil, fmt.Errorf("unsupported extension '%s' for pipeline file: %s", extension, filename) + } + return content, nil +} diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 1e013fb081f..6eaee863da0 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -78,6 +78,11 @@ packetbeat.interfaces.internal_networks: # can stay enabled even after beat is shut down. #packetbeat.interfaces.auto_promisc_mode: true +# By default Ingest pipelines are not updated if a pipeline with the same ID +# already exists. If this option is enabled Packetbeat overwrites pipelines +# every time a new Elasticsearch connection is established. +#packetbeat.overwrite_pipelines: false + # =================================== Flows ==================================== packetbeat.flows: @@ -1817,9 +1822,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/packetbeat/protos/protos.go b/packetbeat/protos/protos.go index 26e2783d216..887c349e5b7 100644 --- a/packetbeat/protos/protos.go +++ b/packetbeat/protos/protos.go @@ -173,16 +173,9 @@ func (s ProtocolsStruct) configureProtocol(test bool, device string, pub reporte if device != "" { // This could happen earlier, but let any errors be found first. - var protocol struct { - Device string `config:"interface"` - } - err := config.Unpack(&protocol) - if err != nil { + if isValid, err := validateProtocolDevice(device, config); !isValid || err != nil { return err } - if protocol.Device != "" && protocol.Device != device { - return nil - } } var client beat.Client @@ -205,6 +198,24 @@ func (s ProtocolsStruct) configureProtocol(test bool, device string, pub reporte return nil } +func validateProtocolDevice(device string, config *conf.C) (bool, error) { + var protocol struct { + Interface struct { + Device string `config:"device"` + } `config:"interface"` + } + + if err := config.Unpack(&protocol); err != nil { + return false, err + } + + if protocol.Interface.Device != "" && protocol.Interface.Device != device { + return false, nil + } + + return true, nil +} + func (s ProtocolsStruct) register(proto Protocol, client beat.Client, plugin Plugin) { if _, exists := s.all[proto]; exists { logp.Warn("Protocol (%s) plugin will overwritten by another plugin", proto.String()) diff --git a/packetbeat/protos/protos_test.go b/packetbeat/protos/protos_test.go index 3637d139407..39c11c7847c 100644 --- a/packetbeat/protos/protos_test.go +++ b/packetbeat/protos/protos_test.go @@ -24,6 +24,8 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/common" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/go-ucfg" "github.com/stretchr/testify/assert" ) @@ -200,3 +202,58 @@ func TestGetUDP(t *testing.T) { assert.NotNil(t, udp) assert.Contains(t, udp.GetPorts(), 53) } + +func TestValidateProtocolDevice(t *testing.T) { + tcs := []struct { + testCase, device string + config map[string]interface{} + expectedValid bool + expectedErr string + }{ + { + "DeviceIsIncorrect", + "eth0", + map[string]interface{}{ + "interface": map[string]interface{}{ + "device": "eth1", + }, + }, + false, + "", + }, + { + "DeviceIsCorrect", + "eth1", + map[string]interface{}{ + "interface": map[string]interface{}{ + "device": "eth1", + }, + }, + true, + "", + }, + { + "ConfigIsInvalid", + "eth0", + map[string]interface{}{ + "interface": "eth1", + }, + false, + "required 'object', but found 'string' in field 'interface'", + }, + } + + for _, tc := range tcs { + tc := tc + t.Run(tc.testCase, func(t *testing.T) { + cfg := (*conf.C)(ucfg.MustNewFrom(tc.config)) + isValid, err := validateProtocolDevice(tc.device, cfg) + assert.Equal(t, tc.expectedValid, isValid) + if tc.expectedErr == "" { + assert.Nil(t, err) + } else { + assert.EqualError(t, err, tc.expectedErr) + } + }) + } +} diff --git a/packetbeat/scripts/mage/config.go b/packetbeat/scripts/mage/config.go index 5213f4f1f87..f41b50ffff7 100644 --- a/packetbeat/scripts/mage/config.go +++ b/packetbeat/scripts/mage/config.go @@ -30,11 +30,18 @@ func device(goos string) string { return "default_route" } +// SelectLogic configures the types of project logic to use (OSS vs X-Pack). +// It is set in the packetbeat and x-pack/packetbeat magefiles. +var SelectLogic devtools.ProjectType + // ConfigFileParams returns the default ConfigFileParams for generating // packetbeat*.yml files. func ConfigFileParams() devtools.ConfigFileParams { p := devtools.DefaultConfigFileParams() p.Templates = append(p.Templates, devtools.OSSBeatDir("_meta/config/*.tmpl")) + if SelectLogic == devtools.XPackProject { + p.Templates = append(p.Templates, devtools.XPackBeatDir("_meta/config/*.tmpl")) + } p.ExtraVars = map[string]interface{}{ "device": device, } diff --git a/testing/certutil/certutil.go b/testing/certutil/certutil.go new file mode 100644 index 00000000000..422bf4969d4 --- /dev/null +++ b/testing/certutil/certutil.go @@ -0,0 +1,186 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package certutil + +import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" +) + +// TODO: move it to a more generic place. Probably elastic-agent-client. +// Moving it to the agent-client would allow to have a mock.StubServerV2 with +// TLS out of the box. With that, we could also remove the +// `management.insecure_grpc_url_for_testing` flag from the beats. +// This can also be expanded to save the certificates and keys to disk, making +// an tool for us to generate certificates whenever we need. + +// NewRootCA generates a new x509 Certificate and returns: +// - the private key +// - the certificate +// - the certificate in PEM format as a byte slice. +// +// If any error occurs during the generation process, a non-nil error is returned. +func NewRootCA() (*ecdsa.PrivateKey, *x509.Certificate, []byte, error) { + rootKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not create private key: %w", err) + } + + notBefore := time.Now() + notAfter := notBefore.Add(3 * time.Hour) + + rootTemplate := x509.Certificate{ + DNSNames: []string{"localhost"}, + SerialNumber: big.NewInt(1653), + Subject: pkix.Name{ + Organization: []string{"Gallifrey"}, + CommonName: "localhost", + }, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + + rootCertRawBytes, err := x509.CreateCertificate( + rand.Reader, &rootTemplate, &rootTemplate, &rootKey.PublicKey, rootKey) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not create CA: %w", err) + } + + rootPrivKeyDER, err := x509.MarshalECPrivateKey(rootKey) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not marshal private key: %w", err) + } + + // PEM private key + var rootPrivBytesOut []byte + rootPrivateKeyBuff := bytes.NewBuffer(rootPrivBytesOut) + err = pem.Encode(rootPrivateKeyBuff, &pem.Block{ + Type: "EC PRIVATE KEY", Bytes: rootPrivKeyDER}) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not pem.Encode private key: %w", err) + } + + // PEM certificate + var rootCertBytesOut []byte + rootCertPemBuff := bytes.NewBuffer(rootCertBytesOut) + err = pem.Encode(rootCertPemBuff, &pem.Block{ + Type: "CERTIFICATE", Bytes: rootCertRawBytes}) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not pem.Encode certificate: %w", err) + } + + // tls.Certificate + rootTLSCert, err := tls.X509KeyPair( + rootCertPemBuff.Bytes(), rootPrivateKeyBuff.Bytes()) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not create key pair: %w", err) + } + + rootCACert, err := x509.ParseCertificate(rootTLSCert.Certificate[0]) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not parse certificate: %w", err) + } + + return rootKey, rootCACert, rootCertPemBuff.Bytes(), nil +} + +// GenerateChildCert generates a x509 Certificate as a child of caCert and +// returns the following: +// - the certificate in PEM format as a byte slice +// - the private key in PEM format as a byte slice +// - the certificate and private key as a tls.Certificate +// +// If any error occurs during the generation process, a non-nil error is returned. +func GenerateChildCert(name string, caPrivKey *ecdsa.PrivateKey, caCert *x509.Certificate) ( + []byte, []byte, *tls.Certificate, error) { + + notBefore := time.Now() + notAfter := notBefore.Add(3 * time.Hour) + + certTemplate := &x509.Certificate{ + DNSNames: []string{name}, + SerialNumber: big.NewInt(1658), + Subject: pkix.Name{ + Organization: []string{"Gallifrey"}, + CommonName: name, + }, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + } + + privateKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not create private key: %w", err) + } + + certRawBytes, err := x509.CreateCertificate( + rand.Reader, certTemplate, caCert, &privateKey.PublicKey, caPrivKey) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not create CA: %w", err) + } + + privateKeyDER, err := x509.MarshalECPrivateKey(privateKey) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not marshal private key: %w", err) + } + + // PEM private key + var privBytesOut []byte + privateKeyBuff := bytes.NewBuffer(privBytesOut) + err = pem.Encode(privateKeyBuff, &pem.Block{ + Type: "EC PRIVATE KEY", Bytes: privateKeyDER}) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not pem.Encode private key: %w", err) + } + privateKeyPemBytes := privateKeyBuff.Bytes() + + // PEM certificate + var certBytesOut []byte + certBuff := bytes.NewBuffer(certBytesOut) + err = pem.Encode(certBuff, &pem.Block{ + Type: "CERTIFICATE", Bytes: certRawBytes}) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not pem.Encode certificate: %w", err) + } + certPemBytes := certBuff.Bytes() + + // TLS Certificate + tlsCert, err := tls.X509KeyPair(certPemBytes, privateKeyPemBytes) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not create key pair: %w", err) + } + + return privateKeyPemBytes, certPemBytes, &tlsCert, nil +} diff --git a/testing/environments/latest.yml b/testing/environments/latest.yml index 2ce1f719210..76718b63b7b 100644 --- a/testing/environments/latest.yml +++ b/testing/environments/latest.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.11.2 + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.1 healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -19,7 +19,7 @@ services: - "script.context.template.cache_max_size=2000" logstash: - image: docker.elastic.co/logstash/logstash:8.11.2 + image: docker.elastic.co/logstash/logstash:8.12.1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 300 @@ -29,7 +29,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.11.2 + image: docker.elastic.co/kibana/kibana:8.12.1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5601"] retries: 300 diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index c1e25d376f6..4029dd7fa97 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-yil7wib0-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-772867d3-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.13.0-yil7wib0-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.13.0-772867d3-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.13.0-yil7wib0-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.13.0-772867d3-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/winlogbeat/docs/howto/load-ingest-pipelines.asciidoc b/winlogbeat/docs/howto/load-ingest-pipelines.asciidoc index fa795f0b6b2..0d7f842249e 100644 --- a/winlogbeat/docs/howto/load-ingest-pipelines.asciidoc +++ b/winlogbeat/docs/howto/load-ingest-pipelines.asciidoc @@ -24,7 +24,7 @@ to load the ingest pipelines with the `setup` command or manually. === setup command On a machine that has {beatname_uc} installed and has {es} configured -as the outup, run the `setup` command with the `--pipelines` option +as the output, run the `setup` command with the `--pipelines` option specified. For example, the following command loads the ingest pipelines: diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 8b7bad94c23..a6a042f4aba 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1233,9 +1233,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 45d1c4af851..a0352454e09 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -92,6 +92,11 @@ auditbeat.modules: # Auditbeat will ignore files unless they match a pattern. #include_files: #- '/\.ssh($|/)' + # Select the backend which will be used to source events. + # "fsnotify" doesn't have the ability to associate user data to file events. + # Valid values: auto, fsnotify, kprobes, ebpf. + # Default: fsnotify. + backend: fsnotify # Scan over the configured file paths at startup and send events for new or # modified files since the last time Auditbeat was running. @@ -1407,9 +1412,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/auditbeat/module/system/_meta/fields.yml b/x-pack/auditbeat/module/system/_meta/fields.yml index 61908a6ce29..43101839c0a 100644 --- a/x-pack/auditbeat/module/system/_meta/fields.yml +++ b/x-pack/auditbeat/module/system/_meta/fields.yml @@ -30,6 +30,25 @@ - name: process type: group fields: + - name: thread.capabilities.effective + level: extended + type: keyword + ignore_above: 1024 + description: This is the set of capabilities used by the kernel to perform permission + checks for the thread. + example: '["CAP_BPF", "CAP_SYS_ADMIN"]' + pattern: ^(CAP_[A-Z_]+|\d+)$ + default_field: false + - name: thread.capabilities.permitted + level: extended + type: keyword + ignore_above: 1024 + description: This is a limiting superset for the effective capabilities that + the thread may assume. + example: '["CAP_BPF", "CAP_SYS_ADMIN"]' + pattern: ^(CAP_[A-Z_]+|\d+)$ + default_field: false + - name: hash type: group description: > diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index 7711dffe2c0..4b0a95d23b0 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSystem returns asset data. // This is the base64 encoded zlib format compressed contents of module/system. func AssetSystem() string { - return "eJy0WV1v2zoSffevGPSlCeAqiNsEhR8WSJtiE2y7DdYp0DebEscSNxSpS1JJ1F9/QerDkk3ZlqMroEDNkOec+eBoSH2AJyzmoAttMJ0AGGY4zuHdwg28mwBQ1JFimWFSzOFfEwCAxwQ1AlEIJkFYM+RUQ4wCFTFIISzceIkJqaQ5x2ACoJAj0TiHEA2ZQLVwPpkAfABBUpwDPqMwjsMUGc4hVjLP3O96sv1/PVsqFjPhhuoFT1i8SEWrMY92+/x060CunU7HGcBjwjRERECIQGDNOEJGTAJnGMQBrC6eibrgMrb/gsvV+bRBk8rBWEk1ZGV6JNNMChQGTEIM6DzLOEPqplBiSI0t0HAmnlbnQdsXuUZ1tCtQGGaKJaPDvXF/C7lgf+XIC2DUAq0LJmKn0moAKYBAIrUJ4N6A9ZJMs9xGmmggsLi7+TC7uoaE6GTjlNIRdhXc305LIPsfImj5w+oOOjYYVCkThA834bFaWdNago4vMyUj1Ppod7Zs2Z7eK+KO6AR1k1WvGOWGhBxtaqG1Q7stQ3gsFTNJ6qi0c4hd8Ex4jm5Kg+g8iK+AIpIUKVAWozbVTGfftv6NBSEnTzgLl7Or6w2ex6Nb5nz5fvOfb7OwCajHnEkP08fPn05h+vj501Cmq8vZKUxXl7NjmXRCZrNB5izubmazoy3RCRnorsXdzQBPWfzlcAvcmmEcw9Kr5Dg+txzHCZ5aDvXVwJRyHMPy6epydkJEri5nF8Ni4ngGR8XxHB+X19fkepApv39f7zWiMcC9OQOSU+bvAzzFt1sAW0VcatMM+gp5D179rCzACiIpDGGi7nB4+VJjYi1VSuy6oLVqu8epn22Nrdd8ZliKHeJSKZci7gyXhHOguXK8nT8ykeVmWU8RREiNkRRUd2bJ3LSnEX1LCu+MTGHEtHPKZefve/xln1/OGmCiLSHwmB1KaXoMp8TgEM4vUhqwWD6eKnqo2B+kHrJQSo5EDOFboAG2rtLAdkANh0+AFfZHCgzsT4+A7W1zhID/tlrNGr7dcU3B9ZVfFo97Bcn1WqMJNEbHZN8BTY8bHRbVZsCe6FuV4/njrkLzMTFf0E/kgPtbHwVRUcIMRiZXIxrUga1OCq+fr5fXn859IlLii+IJ3D9uvgKhVKHW6I0dyzxEW4MHOO4f9lNI7aHYrtwHWFZSt2p3q1wDCWVu3GaRmT2y2sNO9d7p1tudmt0uKxR3Enif1w/65OeiAZ3a8kJEUUVdG4UmSs4Dr5KME2NtG1VJDVopiFAYqaeQh7kw+RRemKDyRfcoGt0v7mhdKvlBIjvyu4d6TVLGi1HJS8iKXiFNiJkCxZARMYW1Qgw1PeSRZ1R6+4X9Vl0Vpp/wCZVAPh7fo2ezvNcVzX4plnVUw+1qONOI8O3rAqQO7EDL8c3GINETifFNHWCFsbeQEAFMaEM4RwpSgcJUPiOt+d/WHW7f6xxy4F737bvpqdUevOKpGo2dyDRXPRUSlJWMiMYTvjzprRknmvjQIvfx+HbiG6n2WFXFe0y2CrKvDxmTqt2A+Pg4i1CMa10F6W07yj02ypmhpqswew8Pmv056mR2FJkF85LkaUpUcQJgudCHmSs+Zlh+/e/7bn1t7qfbFEOKqwU42KLZSbq8gt7t0Y6vp/9UdwLwq3uZveMlto34drbuMWTDFY/L9W8by14yytTYhr3XkMgULTRGRnZTu33JhXzE3gbgQclYkRSMBJULIAa4jFlPP2MTctnK1VE9Xt0wuQ8k7Rsm+CngOxP56xRMwrR9Q9vNEWMkdZntPRmxc2aqFcrw/xiZYQJXDu5AM1SUpHrz/YhpyIgytnE4C7GQ1QePvIx4ppitYuWqrf7Zv5Nh/24+FIWjIgFN/u9ubdi75Tb0TBiMcXuXDKTv234Z0dpjXN9R+XBsa8D94W2iVs2GMyFN1UBWI8xo5OvBkfScE2CsSN7syLawATxIrVnI2x/fYKUTQuXLsvFHD+ZZx2jXGduNKcoPwA7DfUU+n258u6RMk5AjXU17UFdCbpgtR7nZKRExKplr14+LQgp036q5jIGJc9dm9yFGqshMG/QlQdENmYuN1X6BJrpwwxQ0Yqp7QI2ss8Qef1A4DnfmKRF3ot/qGok2yyixBvVvnZ12rnyOCvaj+7pedGpMbegL0U4AVAKCyd8BAAD//yDbzZE=" + return "eJy8Wm1v27oV/u5fcRAMaILrKotvExT+MMBt7pZg7W0wp0C3uzubEo8lLhSpkVQSFfvxA6kXSzZlW4k7A0VjmXyec57z4kPJb+EBiynoQhtMRwCGGY5TOJm7CycjAIo6UiwzTIop/GkEAHCfoEYgCsEkCCuGnGqIUaAiBimEhbteYkIqac4xGAEo5Eg0TiFEQ0ZQbZyORgBvQZAUp4CPKIzjMEWGU4iVzDP3vl5s/65XS8ViJtylesMDFk9S0eqax3b7+uL2gVw5Ox1nAPcJ0xARASECgRXjCBkxCZxiEAewPH8k6pzL2P4LLpZn4wZNKgdjTaohK9cjmWZSoDBgEmJA51nGGVK3hBJDamyBhjPxsDwL2lrkGtXBUqAwzBQLRoercXsNuWD/yZEXwKgFWhVMxM5KawNIAQQSqU0AtwasSjLNchtpooHA/Gb2dnJ5BQnRyVqUUgi7C26vxyWQ/YMIWr6xdgcdHwyqlAnCh7twX+2saS1BR8tMyQi1PlhOkygkNIhIRkLGmWGoA1ytMDLsEStajo/Ip4DPBgXFXcKzWEiFCxLKR5zCxR8n73zuuARkukwgNNaXNr91qqmtB1QCORgJGaqVVKn9P2VaMykaVaIEowcNqypBK5+qj/GZpJkt9Te/nXyc3S0+3P35ZAzuz/nf54vZ9efbX09+f1OtzogxqMQU/nVqV/w2e/uPxe8//fef9KezPzS+rEjOzcLJOYUV4Rr3auqsNqZR70dpSoCzlBmb1jrPUFl9a12auHbltiXbSLnWD1JSANE6r7P3/yVlR8tWrW2mc2+R3BCdoG663jNGuSEhR9v6bEYV2rV0wmOpmElSR6VdwdoNj4Tn6JZ0VEnwGVBEkiIFymLUploZjKp17fpaexBy8oCTcDG5vFrjeeK84c6HT7O//jIJm4bjcWfUw/Tz+3cvYfr5/buhTJcXk5cwXV5MDmXSCZlMBrkzv5lNJgd7ohMyUK75zWyAUhZ/MdwDt2cYx7D0KjkOzy3H8QKlFkO1GphSjmNYPl1eTF4QkcuLyfmwmDiewVFxPIfH5fk5uRrkyrdvVzudaBxwk11Acsr8c6qn+XYbYKuJS73+hvE18h68+rW0AEuIpDCEiXoC5+XQxYQdC4jdF7R2bc7g9WvTxtYYmhmWYoe4tJRLEXcul4RToLlyvJ0Pmchys6iXCCKkxkgKqjurZG7ay4i+JoV3RaYwYtqJctH5fIde9vXVeQNMtE0IPG6HUpoexykxOITzg5QGLJaPp4oeKvYdqYcslJIjEUP45miArao0sLNPw+EzwBr2XQoM7FuPAZtlc4ABv7aOQjV8+0QwBnfu+TC/32mQXK00mkBjdEj27bHpfm2HRbUZsCP61srj6XFTofmYmC/oL+SA22sfBVFRwgxGJldHdKgDW51kn99fLa7enfmMSIkvii/g/jz7CIRShVqjN3Ys8xBtXNzDcXu3m0JqD8Vm597DspS61btb7RpIKHPjikVmaFupPbWU3zvdfrvVs9ttheJWAu9Sfa8mX+YN6Ni2FyKKKuraKDRRchZ4Lck4Mda3o1pSg1YWRCiM1GPIw1yYfAxPTFD5pHssOrou7tZPaclnEtkr33qoVyRlvDgqeQlZ0SukCTFjoBgyIsawUoihpvsUeUSlN7+wX2tXheknLO9fHI/v3lMsb3R9m2SnKZb1qI7b3XCqEeGXj3OQOrAXWsI3hUGiBxLjqybACmNnIyECmNCGcI4UpAKFqXxEWvO/bjrcvO+4T8Cd8u26E1lbu/cWZDVobEWmuRVZIUHZyYholPDlSW/PeKGLdy1yH4+vEl9JtcOrKt7HZKsg++aQY1K1BxAfH2cRiuN6V0F6x46yxo5yZqjpKszew4Nm3w86mR1EZsG8JHmaElW8ALDc6MPMFT9mWL7+7dN2f22en7QphjRXC7B3RLOLdPmIZHtGO7yf/qjpBOBr92HLlkpsE/H1bN1jyJorPi7XX2wse8koU8d27I2GRKZooTEyspva7ZtcyI842wDcKRkrkoKRoHIBxACXMeuZZ2xCLlq5elTFqztM7gFe+w4TfBHwiYn8eQym9RgrxkjqMtt7MmLrzFRbKMN/Y2SGGbh0cHuGoaIk1evnm0xDRpR74HYaYiGrBx55GfFMMdvFyl0b87O/kmF3Ne+LwkGRgCb/t0sbdpbcmp4JgzFuVslA+r7yy4jWHuf6jsr7Y1sD7g5vE7VqNZwKaaoBsrrCjEa+GhxJzzkBjhXJ2ZbZFjaAO6k1C3n74RssdUKofFo0evRgnnacdpOxLUxRPl92GO5XDmfjtbYLyjQJOdLluAd1KeSa2XKUxU6JiFHJXLt5XBRSoPstBZcxMHHmxuw+xEgVmWmDPiUouiFzsbG2n6OJzt1lChox1T2gRtZZYo8/KByHO/OUiFvRb02NRJtFlFiH+ktna5wrXwcF+979+qPo9Jja0SeinQFQGRCM/hcAAP//VUidAg==" } diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index d2dfae06598..08a72fe562e 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/auditbeat/helper/hasher" + "github.com/elastic/beats/v7/libbeat/common/capabilities" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/auditbeat/cache" @@ -101,12 +102,14 @@ type MetricSet struct { // Process represents information about a process. type Process struct { - Info types.ProcessInfo - UserInfo *types.UserInfo - User *user.User - Group *user.Group - Hashes map[hasher.HashType]hasher.Digest - Error error + Info types.ProcessInfo + UserInfo *types.UserInfo + User *user.User + Group *user.Group + CapEffective []string + CapPermitted []string + Hashes map[hasher.HashType]hasher.Digest + Error error } // Hash creates a hash for Process. @@ -376,6 +379,13 @@ func (ms *MetricSet) processEvent(process *Process, eventType string, action eve event.RootFields.Put("user.group.name", process.Group.Name) } + if len(process.CapEffective) > 0 { + event.RootFields.Put("process.thread.capabilities.effective", process.CapEffective) + } + if len(process.CapPermitted) > 0 { + event.RootFields.Put("process.thread.capabilities.permitted", process.CapPermitted) + } + if process.Hashes != nil { for hashType, digest := range process.Hashes { fieldName := "process.hash." + string(hashType) @@ -489,8 +499,20 @@ func (ms *MetricSet) getProcesses() ([]*Process, error) { } // Exclude Linux kernel processes, they are not very interesting. - if runtime.GOOS == "linux" && userInfo.UID == "0" && process.Info.Exe == "" { - continue + if runtime.GOOS == "linux" { + if userInfo.UID == "0" && process.Info.Exe == "" { + continue + } + + // Fetch Effective and Permitted capabilities + process.CapEffective, err = capabilities.FromPid(capabilities.Effective, pInfo.PID) + if err != nil && process.Error == nil { + process.Error = err + } + process.CapPermitted, err = capabilities.FromPid(capabilities.Permitted, pInfo.PID) + if err != nil && process.Error == nil { + process.Error = err + } } processes = append(processes, process) diff --git a/x-pack/dockerlogbeat/pipelinemock/pipelines.go b/x-pack/dockerlogbeat/pipelinemock/pipelines.go index d9054cf6eb4..04c7972e8ee 100644 --- a/x-pack/dockerlogbeat/pipelinemock/pipelines.go +++ b/x-pack/dockerlogbeat/pipelinemock/pipelines.go @@ -85,3 +85,11 @@ func (pc *MockPipelineConnector) ConnectWith(beat.ClientConfig) (beat.Client, er return c, nil } + +// HasConnectedClients returns true if there are clients connected. +func (pc *MockPipelineConnector) HasConnectedClients() bool { + pc.mtx.Lock() + defer pc.mtx.Unlock() + + return len(pc.clients) > 0 +} diff --git a/x-pack/dockerlogbeat/pipelinemock/reader.go b/x-pack/dockerlogbeat/pipelinemock/reader.go index 263e1ff2f78..ff51cc5fbb1 100644 --- a/x-pack/dockerlogbeat/pipelinemock/reader.go +++ b/x-pack/dockerlogbeat/pipelinemock/reader.go @@ -8,7 +8,6 @@ import ( "bytes" "encoding/binary" "io" - "io/ioutil" "testing" "github.com/docker/docker/api/types/plugins/logdriver" @@ -33,7 +32,7 @@ func CreateTestInputFromLine(t *testing.T, line string) io.ReadCloser { writer := new(bytes.Buffer) encodeLog(t, writer, exampleStruct) - return ioutil.NopCloser(writer) + return io.NopCloser(writer) } func encodeLog(t *testing.T, out io.Writer, entry *logdriver.LogEntry) { diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl index a35c0af5dc4..c5861174636 100644 --- a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl +++ b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl @@ -180,3 +180,44 @@ # This is used to shift collection start time and end time back in order to # collect logs when there is a delay in CloudWatch. #latency: 1m + +#------------------------------ ETW input -------------------------------- +# Beta: Config options for ETW (Event Trace for Windows) input (Only available for Windows) +#- type: etw + #enabled: false + #id: etw-dnsserver + + # Path to an .etl file to read from. + #file: "C:\Windows\System32\Winevt\Logs\Logfile.etl" + + # GUID of an ETW provider. + # Run 'logman query providers' to list the available providers. + #provider.guid: {EB79061A-A566-4698-9119-3ED2807060E7} + + # Name of an ETW provider. + # Run 'logman query providers' to list the available providers. + #provider.name: Microsoft-Windows-DNSServer + + # Tag to identify created sessions. + # If missing, its default value is the provider ID prefixed by 'Elastic-'. + #session_name: DNSServer-Analytical-Trace + + # Filter collected events with a level value that is less than or equal to this level. + # Allowed values are critical, error, warning, informational, and verbose. + #trace_level: verbose + + # 8-byte bitmask that enables the filtering of events from specific provider subcomponents. + # The provider will write a particular event if the event's keyword bits match any of the bits + # in this bitmask. + # Run 'logman query providers ""' to list available keywords. + #match_any_keyword: 0x8000000000000000 + + # 8-byte bitmask that enables the filtering of events from + # specific provider subcomponents. The provider will write a particular + # event if the event's keyword bits match all of the bits in this bitmask. + # Run 'logman query providers ""' to list available keywords. + #match_all_keyword: 0 + + # An existing session to read from. + # Run 'logman query -ets' to list existing sessions. + #session: UAL_Usermode_Provider diff --git a/x-pack/filebeat/docs/inputs/input-cel.asciidoc b/x-pack/filebeat/docs/inputs/input-cel.asciidoc index 837ea80ea1e..684ceb7aa18 100644 --- a/x-pack/filebeat/docs/inputs/input-cel.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cel.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] :type: cel -:mito_version: v1.8.0 +:mito_version: v1.9.0 :mito_docs: https://pkg.go.dev/github.com/elastic/mito@{mito_version} [id="{beatname_lc}-input-{type}"] @@ -160,8 +160,10 @@ As noted above the `cel` input provides functions, macros, and global variables ** {mito_docs}/lib#hdr-Drop[Drop] ** {mito_docs}/lib#hdr-Drop_Empty[Drop Empty] ** {mito_docs}/lib#hdr-Flatten[Flatten] +** {mito_docs}/lib#hdr-Keys[Keys] ** {mito_docs}/lib#hdr-Max[Max] ** {mito_docs}/lib#hdr-Min[Min] +** {mito_docs}/lib#hdr-Values[Values] ** {mito_docs}/lib#hdr-With[With] ** {mito_docs}/lib#hdr-With_Replace[With Replace] ** {mito_docs}/lib#hdr-With_Update[With Update] @@ -580,6 +582,13 @@ The RSA JWK Private Key JSON for your Okta Service App which is used for interac NOTE: Only one of the credentials settings can be set at once. For more information please refer to https://developer.okta.com/docs/guides/implement-oauth-for-okta-serviceapp/main/ +[float] +==== `auth.oauth2.okta.jwk_pem` + +The RSA JWK private key PEM block for your Okta Service App which is used for interacting with Okta Org Auth Server to mint tokens with okta.* scopes. + +NOTE: Only one of the credentials settings can be set at once. For more information please refer to https://developer.okta.com/docs/guides/implement-oauth-for-okta-serviceapp/main/ + [[resource-parameters]] [float] ==== `resource.url` diff --git a/x-pack/filebeat/docs/inputs/input-etw.asciidoc b/x-pack/filebeat/docs/inputs/input-etw.asciidoc new file mode 100644 index 00000000000..9ace3fdcc1b --- /dev/null +++ b/x-pack/filebeat/docs/inputs/input-etw.asciidoc @@ -0,0 +1,144 @@ +[role="xpack"] + +:type: etw + +[id="{beatname_lc}-input-{type}"] +=== ETW input + +++++ +ETW +++++ + +beta[] + +https://learn.microsoft.com/en-us/windows/win32/etw/event-tracing-portal[Event Tracing for Windows] is a powerful logging and tracing mechanism built into the Windows operating system. It provides a detailed view of application and system behavior, performance issues, and runtime diagnostics. Trace events contain an event header and provider-defined data that describes the current state of an application or operation. You can use the events to debug an application and perform capacity and performance analysis. + +The ETW input can interact with ETW in three distinct ways: it can create a new session to capture events from user-mode providers, attach to an already existing session to collect ongoing event data, or read events from a pre-recorded .etl file. This functionality enables the module to adapt to different scenarios, such as real-time event monitoring or analyzing historical data. + +This input currently supports manifest-based, MOF (classic) and TraceLogging providers while WPP providers are not supported. https://learn.microsoft.com/en-us/windows/win32/etw/about-event-tracing#types-of-providers[Here] you can find more information about the available types of providers. + +It has been tested in every Windows versions supported by Filebeat, starting from Windows 8.1 and Windows Server 2016. In addition, administrative privileges are required in order to control event tracing sessions. + +Example configurations: + +Read from a provider by name: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + id: etw-dnsserver + enabled: true + provider.name: Microsoft-Windows-DNSServer + session_name: DNSServer-Analytical + trace_level: verbose + match_any_keyword: 0x8000000000000000 + match_all_keyword: 0 +---- + +Same provider can be defined by its GUID: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + id: etw-dnsserver + enabled: true + provider.guid: {EB79061A-A566-4698-9119-3ED2807060E7} + session_name: DNSServer-Analytical + trace_level: verbose + match_any_keyword: 0x8000000000000000 + match_all_keyword: 0 +---- + +Read from a current session: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + enabled: true + id: etw-dnsserver-session + session: UAL_Usermode_Provider +---- + +Read from a .etl file: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + enabled: true + id: etw-dnsserver-session + file: "C\Windows\System32\Winevt\Logs\Logfile.etl" +---- + +NOTE: Examples shown above are mutually exclusive, since the options `provider.name`, `provider.guid`, `session` and `file` cannot be present at the same time. Nevertheless, it is a requirement that one of them appears. + +Multiple providers example: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + id: etw-dnsserver + enabled: true + provider.name: Microsoft-Windows-DNSServer + session_name: DNSServer-Analytical + trace_level: verbose + match_any_keyword: 0xfffffffffffffffff + match_all_keyword: 0 +- type: etw + id: etw-security + enabled: true + provider.name: Microsoft-Windows-Security-Auditing + session_name: Security-Auditing + trace_level: warning + match_any_keyword: 0xffffffffffffffff + match_all_keyword: 0 +---- + +==== Configuration options + +The `ETW` input supports the following configuration options. + +[float] +==== `file` + +Specifies the path to an .etl file for reading ETW events. This file format is commonly used for storing ETW event logs. + +[float] +==== `provider.guid` + +Identifies the GUID of an ETW provider. To see available providers, use the command `logman query providers`. + +[float] +==== `provider.name` + +Specifies the name of the ETW provider. Available providers can be listed using `logman query providers`. + +[float] +==== `session_name` + +When specified a provider, a new session is created. It sets the name for a new ETW session associated with the provider. If not provided, the default is the provider ID prefixed with 'Elastic-'. + +[float] +==== `trace_level` + +Defines the filtering level for events based on severity. Valid options include critical, error, warning, informational, and verbose. + +[float] +==== `match_any_keyword` + +An 8-byte bitmask used for filtering events from specific provider subcomponents based on keyword matching. Any matching keyword will enable the event to be written. Default value is `0xfffffffffffffffff` so it matches every available keyword. + +Run `logman query providers ""` to list the available keywords for a specific provider. + +[float] +==== `match_all_keyword` + +Similar to MatchAnyKeyword, this 8-byte bitmask filters events that match all specified keyword bits. Default value is `0` to let every event pass. + +Run `logman query providers ""` to list the available keywords for a specific provider. + +[float] +==== `session` + +Names an existing ETW session to read from. Existing sessions can be listed using `logman query -ets`. + +:type!: diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 410edf9f948..5fbd5dc15a5 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -11,6 +11,8 @@ Use the `httpjson` input to read messages from an HTTP API with JSON payloads. +If you are starting development of a new custom HTTP API input, we recommend that you use the <> which provides greater flexibility and an improved developer experience. + This input supports: * Auth @@ -218,7 +220,8 @@ Some built-in helper functions are provided to work with the input state inside - `min`: returns the minimum of two values. - `mul`: multiplies two integers. - `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now. -- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`. +- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Note: Parsing timezone abbreviations may cause ambiguities. Prefer `parseDateInTZ` for explicit timezone handling. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`. +- `parseDateInTZ`: parses a date string within a specified timezone (TZ), returning a `time.Time` in UTC. Specified timezone overwrites implicit timezone from the input date. Accepts timezone offsets ("-07:00", "-0700", "-07") or IANA Time Zone names ("America/New_York"). If TZ is invalid, defaults to UTC. Optional layout argument as in parseDate. Example: `[[ parseDateInTZ "2020-11-05T12:25:32" "America/New_York" ]]`, `[[ parseDateInTZ "2020-11-05T12:25:32" "-07:00" "RFC3339" ]]`. - `parseDuration`: parses duration strings and returns `time.Duration`. Example: `[[parseDuration "1h"]]`. - `parseTimestampMilli`: parses a timestamp in milliseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000]]` returns `2020-11-05 13:25:32 +0000 UTC`. - `parseTimestampNano`: parses a timestamp in nanoseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000000000]]` returns `2020-11-05 13:25:32 +0000 UTC`. @@ -399,8 +402,12 @@ NOTE: Only one of the credentials settings can be set at once. For more informat The RSA JWK Private Key JSON for your Okta Service App which is used for interacting with Okta Org Auth Server to mint tokens with okta.* scopes. -NOTE: Only one of the credentials settings can be set at once. For more information please refer to https://developer.okta.com/docs/guides/implement-oauth-for-okta-serviceapp/main/ +[float] +==== `auth.oauth2.okta.jwk_pem` +The RSA JWK private key PEM block for your Okta Service App which is used for interacting with Okta Org Auth Server to mint tokens with okta.* scopes. + +NOTE: Only one of the credentials settings can be set at once. For more information please refer to https://developer.okta.com/docs/guides/implement-oauth-for-okta-serviceapp/main/ [float] ==== `auth.oauth2.google.delegated_account` diff --git a/x-pack/filebeat/docs/inputs/input-websocket.asciidoc b/x-pack/filebeat/docs/inputs/input-websocket.asciidoc new file mode 100644 index 00000000000..8ee2da2b42a --- /dev/null +++ b/x-pack/filebeat/docs/inputs/input-websocket.asciidoc @@ -0,0 +1,272 @@ +[role="xpack"] + +:type: websocket +:mito_version: v1.8.0 +:mito_docs: https://pkg.go.dev/github.com/elastic/mito@{mito_version} + +[id="{beatname_lc}-input-{type}"] +=== Websocket Input +experimental[] + +The `websocket` input reads messages from a websocket server or api endpoint. This input uses the `CEL engine` and the `mito` library interally to parse and process the messages. Having support for `CEL` allows you to parse and process the messages in a more flexible way. It has many similarities with the `cel` input as to how the `CEL` programs are written but deviates in the way the messages are read and processed. The `websocket` input is a `streaming` input and can only be used to read messages from a websocket server or api endpoint. + +This input supports: + +* Auth +** Basic +** Bearer +** Custom + +NOTE: The `websocket` input as of now does not support XML messages. Auto-reconnects are also not supported at the moment so reconnection will occur on input restart. + +==== Execution + +The execution environment provided for the input includes includes the functions, macros, and global variables provided by the mito library. +A single JSON object is provided as an input accessible through a `state` variable. `state` contains a `response` map field and may contain arbitrary other fields configured via the input's `state` configuration. If the CEL program saves cursor states between executions of the program, the configured `state.cursor` value will be replaced by the saved cursor prior to execution. + +On start the `state` will be something like this: + +["source","json",subs="attributes"] +---- +{ + "response": { ... }, + "cursor": { ... }, + ... +} +---- +The `websocket` input creates a `response` field in the state map and attaches the websocket message to this field. All `CEL` programs written should act on this `response` field. Additional fields may be present at the root of the object and if the program tolerates it, the cursor value may be absent. Only the cursor is persisted over restarts, but all fields in state are retained between iterations of the processing loop except for the produced events array, see below. + +If the cursor is present the program should process or filter out responses based on its value. If cursor is not present all responses should be processed as per the program's logic. + +After completion of a program's execution it should return a single object with a structure looking like this: + +["source","json",subs="attributes"] +---- +{ + "events": [ <1> + {...}, + ... + ], + "cursor": [ <2> + {...}, + ... + ] +} +---- + +<1> The `events` field must be present, but may be empty or null. If it is not empty, it must only have objects as elements. +The field could be an array or a single object that will be treated as an array with a single element. This depends completely on the websocket server or api endpoint. The `events` field is the array of events to be published to the output. Each event must be a JSON object. + +<2> If `cursor` is present it must be either be a single object or an array with the same length as events; each element _i_ of the `cursor` will be the details for obtaining the events at and beyond event _i_ in the `events` array. If the `cursor` is a single object, it will be the details for obtaining events after the last event in the `events` array and will only be retained on successful publication of all the events in the `events` array. + + +Example configuration: + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +# Read and process simple websocket messages from a local websocket server +- type: websocket + url: ws://localhost:443/v1/stream + program: | + bytes(state.response).decode_json().as(inner_body,{ + "events": { + "message": inner_body.encode_json(), + } + }) +---- + +==== Debug state logging + +The Websocket input will log the complete state when logging at the DEBUG level before and after CEL evaluation. +This will include any sensitive or secret information kept in the `state` object, and so DEBUG level logging should not be used in production when sensitive information is retained in the `state` object. See <> configuration parameters for settings to exclude sensitive fields from DEBUG logs. + +==== Authentication +The Websocket input supports authentication via Basic token authentication, Bearer token authentication and authentication via a custom auth config. Unlike REST inputs Basic Authentication contains a basic auth token, Bearer Authentication contains a bearer token and custom auth contains any combination of custom header and value. These token/key values are are added to the request headers and are not exposed to the `state` object. The custom auth configuration is useful for constructing requests that require custom headers and values for authentication. The basic and bearer token configurations will always use the `Authorization` header and prepend the token with `Basic` or `Bearer` respectively. + +Example configurations with authentication: + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: websocket + auth.basic_token: "dXNlcjpwYXNzd29yZA==" + url: wss://localhost:443/_stream +---- + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: websocket + auth.bearer_token: "dXNlcjpwYXNzd29yZA==" + url: wss://localhost:443/_stream +---- + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: websocket + auth.custom: + header: "x-api-key" + value: "dXNlcjpwYXNzd29yZA==" + url: wss://localhost:443/_stream +---- + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: websocket + auth.custom: + header: "Auth" + value: "Bearer dXNlcjpwYXNzd29yZA==" + url: wss://localhost:443/_stream +---- + +[[input-state-websocket]] +==== Input state + +The `websocket` input keeps a runtime state between every message received. This state can be accessed by the CEL program and may contain arbitrary objects. +The state must contain a `response` map and may contain any object the user wishes to store in it. All objects are stored at runtime, except `cursor`, which has values that are persisted between restarts. + +==== Configuration options + +The `websocket` input supports the following configuration options plus the +<<{beatname_lc}-input-{type}-common-options>> described later. + +[[program-websocket]] +[float] +==== `program` + +The CEL program that is executed on each message received. This field should ideally be present but if not the default program given below is used. + +["source","yaml",subs="attributes"] +---- +program: | + bytes(state.response).decode_json().as(inner_body,{ + "events": { + "message": inner_body.encode_json(), + } + }) +---- + +[[state-websocket]] +[float] +==== `state` + +`state` is an optional object that is passed to the CEL program on the first execution. It is available to the executing program as the `state` variable. Except for the `state.cursor` field, `state` does not persist over restarts. + +[[cursor-websocket]] +[float] +==== `state.cursor` + +The cursor is an object available as `state.cursor` where arbitrary values may be stored. Cursor state is kept between input restarts and updated after each event of a request has been published. When a cursor is used the CEL program must either create a cursor state for each event that is returned by the program, or a single cursor that reflects the cursor for completion of the full set of events. + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +# Read and process simple websocket messages from a local websocket server +- type: websocket + url: ws://localhost:443/v1/stream + program: | + bytes(state.response).as(body, { + "events": [body.decode_json().with({ + "last_requested_at": has(state.cursor) && has(state.cursor.last_requested_at) ? + state.cursor.last_requested_at + : + now + })], + "cursor": {"last_requested_at": now} + }) +---- + +[[regexp-websocket]] +[float] +==== `regexp` + +A set of named regular expressions that may be used during a CEL program's execution using the `regexp` extension library. The syntax used for the regular expressions is https://github.com/google/re2/wiki/Syntax[RE2]. + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: websocket + # Define two regular expressions, 'products' and 'solutions' for use during CEL program execution. + regexp: + products: '(?i)(Elasticsearch|Beats|Logstash|Kibana)' + solutions: '(?i)(Search|Observability|Security)' +---- + +[[websocket-state-redact]] +[float] +==== `redact` + +During debug level logging, the `state` object and the resulting evaluation result are included in logs. This may result in leaking of secrets. In order to prevent this, fields may be redacted or deleted from the logged `state`. The `redact` configuration allows users to configure this field redaction behaviour. For safety reasons if the `redact` configuration is missing a warning is logged. + +In the case of no-required redaction an empty `redact.fields` configuration should be used to silence the logged warning. + +["source","yaml",subs="attributes"] +---- +- type: websocket + redact: + fields: ~ +---- + +As an example, if a user-constructed Basic Authentication request is used in a CEL program the password can be redacted like so + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: websocket + url: ws://localhost:443/_stream + state: + user: user@domain.tld + password: P@$$W0₹D + redact: + fields: + - password + delete: true +---- + +Note that fields under the `auth` configuration hierarchy are not exposed to the `state` and so do not need to be redacted. For this reason it is preferable to use these for authentication over the request construction shown above where possible. + +[float] +==== `redact.fields` + +This specifies fields in the `state` to be redacted prior to debug logging. Fields listed in this array will be either replaced with a `*` or deleted entirely from messages sent to debug logs. + +[float] +==== `redact.delete` + +This specifies whether fields should be replaced with a `*` or deleted entirely from messages sent to debug logs. If delete is `true`, fields will be deleted rather than replaced. + +[float] +=== Metrics + +This input exposes metrics under the <>. +These metrics are exposed under the `/inputs` path. They can be used to +observe the activity of the input. + +[options="header"] +|======= +| Metric | Description +| `url` | URL of the input resource. +| `cel_eval_errors` | Number of errors encountered during cel program evaluation. +| `errors_total` | Number of errors encountered over the life cycle of the input. +| `batches_received_total` | Number of event arrays received. +| `batches_published_total` | Number of event arrays published. +| `received_bytes_total` | Number of bytes received over the life cycle of the input. +| `events_received_total` | Number of events received. +| `events_published_total` | Number of events published. +| `cel_processing_time` | Histogram of the elapsed successful CEL program processing times in nanoseconds. +| `batch_processing_time` | Histogram of the elapsed successful batch processing times in nanoseconds (time of receipt to time of ACK for non-empty batches). +|======= + +==== Developer tools + +A stand-alone CEL environment that implements the majority of the websocket input's Comment Expression Language functionality is available in the https://github.com/elastic/mito[Elastic Mito] repository. This tool may be used to help develop CEL programs to be used by the input. Installation is available from source by running `go install github.com/elastic/mito/cmd/mito@latest` and requires a Go toolchain. + +[id="{beatname_lc}-input-{type}-common-options"] +include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] + +NOTE: The `websocket` input is currently tagged as experimental and might have bugs and other issues. Please report any issues on the https://github.com/elastic/beats[Github] repository. + +:type!: \ No newline at end of file diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 14308c2cce1..ab9bafd0ed5 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3579,6 +3579,47 @@ filebeat.inputs: # collect logs when there is a delay in CloudWatch. #latency: 1m +#------------------------------ ETW input -------------------------------- +# Beta: Config options for ETW (Event Trace for Windows) input (Only available for Windows) +#- type: etw + #enabled: false + #id: etw-dnsserver + + # Path to an .etl file to read from. + #file: "C:\Windows\System32\Winevt\Logs\Logfile.etl" + + # GUID of an ETW provider. + # Run 'logman query providers' to list the available providers. + #provider.guid: {EB79061A-A566-4698-9119-3ED2807060E7} + + # Name of an ETW provider. + # Run 'logman query providers' to list the available providers. + #provider.name: Microsoft-Windows-DNSServer + + # Tag to identify created sessions. + # If missing, its default value is the provider ID prefixed by 'Elastic-'. + #session_name: DNSServer-Analytical-Trace + + # Filter collected events with a level value that is less than or equal to this level. + # Allowed values are critical, error, warning, informational, and verbose. + #trace_level: verbose + + # 8-byte bitmask that enables the filtering of events from specific provider subcomponents. + # The provider will write a particular event if the event's keyword bits match any of the bits + # in this bitmask. + # Run 'logman query providers ""' to list available keywords. + #match_any_keyword: 0x8000000000000000 + + # 8-byte bitmask that enables the filtering of events from + # specific provider subcomponents. The provider will write a particular + # event if the event's keyword bits match all of the bits in this bitmask. + # Run 'logman query providers ""' to list available keywords. + #match_all_keyword: 0 + + # An existing session to read from. + # Run 'logman query -ets' to list existing sessions. + #session: UAL_Usermode_Provider + # =========================== Filebeat autodiscover ============================ # Autodiscover allows you to detect changes in the system and spawn new modules @@ -4823,9 +4864,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index f7c35308ed3..43b6758766e 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -12,6 +12,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/cometd" + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/etw" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/gcppubsub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" diff --git a/x-pack/filebeat/input/cel/config_auth.go b/x-pack/filebeat/input/cel/config_auth.go index e550a9635d5..d6b35d633e6 100644 --- a/x-pack/filebeat/input/cel/config_auth.go +++ b/x-pack/filebeat/input/cel/config_auth.go @@ -6,6 +6,7 @@ package cel import ( "context" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -141,6 +142,7 @@ type oAuth2Config struct { // okta specific RSA JWK private key OktaJWKFile string `config:"okta.jwk_file"` OktaJWKJSON common.JSONBlob `config:"okta.jwk_json"` + OktaJWKPEM string `config:"okta.jwk_pem"` } // isEnabled returns true if the `enable` field is set to true in the yaml. @@ -321,8 +323,26 @@ func (o *oAuth2Config) validateGoogleProvider() error { } func (o *oAuth2Config) validateOktaProvider() error { - if o.TokenURL == "" || o.ClientID == "" || len(o.Scopes) == 0 || (o.OktaJWKJSON == nil && o.OktaJWKFile == "") { - return errors.New("okta validation error: token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file must be provided") + if o.TokenURL == "" || o.ClientID == "" || len(o.Scopes) == 0 { + return errors.New("okta validation error: token_url, client_id, scopes must be provided") + } + var n int + if o.OktaJWKJSON != nil { + n++ + } + if o.OktaJWKFile != "" { + n++ + } + if o.OktaJWKPEM != "" { + n++ + } + if n != 1 { + return errors.New("okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided") + } + // jwk_pem + if o.OktaJWKPEM != "" { + _, err := x509.ParsePKCS1PrivateKey([]byte(o.OktaJWKPEM)) + return err } // jwk_file if o.OktaJWKFile != "" { diff --git a/x-pack/filebeat/input/cel/config_okta_auth.go b/x-pack/filebeat/input/cel/config_okta_auth.go index cf9003dee8a..74366afd3d5 100644 --- a/x-pack/filebeat/input/cel/config_okta_auth.go +++ b/x-pack/filebeat/input/cel/config_okta_auth.go @@ -5,10 +5,13 @@ package cel import ( + "bytes" "context" "crypto/rsa" + "crypto/x509" "encoding/base64" "encoding/json" + "encoding/pem" "fmt" "math/big" "net/http" @@ -43,9 +46,20 @@ func (o *oAuth2Config) fetchOktaOauthClient(ctx context.Context, _ *http.Client) }, } - oktaJWT, err := generateOktaJWT(o.OktaJWKJSON, conf) - if err != nil { - return nil, fmt.Errorf("oauth2 client: error generating Okta JWT: %w", err) + var ( + oktaJWT string + err error + ) + if len(o.OktaJWKPEM) != 0 { + oktaJWT, err = generateOktaJWTPEM(o.OktaJWKPEM, conf) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error generating Okta JWT PEM: %w", err) + } + } else { + oktaJWT, err = generateOktaJWT(o.OktaJWKJSON, conf) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error generating Okta JWT: %w", err) + } } token, err := exchangeForBearerToken(ctx, oktaJWT, conf) @@ -59,14 +73,16 @@ func (o *oAuth2Config) fetchOktaOauthClient(ctx context.Context, _ *http.Client) oktaJWK: o.OktaJWKJSON, token: token, } - // reuse the tokenSource to refresh the token (automatically calls the custom Token() method when token is no longer valid). + // reuse the tokenSource to refresh the token (automatically calls + // the custom Token() method when token is no longer valid). client := oauth2.NewClient(ctx, oauth2.ReuseTokenSource(token, tokenSource)) return client, nil } -// Token implements the oauth2.TokenSource interface and helps to implement custom token refresh logic. -// Parent context is passed via the customTokenSource struct since we cannot modify the function signature here. +// Token implements the oauth2.TokenSource interface and helps to implement +// custom token refresh logic. The parent context is passed via the +// customTokenSource struct since we cannot modify the function signature here. func (ts *oktaTokenSource) Token() (*oauth2.Token, error) { ts.mu.Lock() defer ts.mu.Unlock() @@ -85,70 +101,79 @@ func (ts *oktaTokenSource) Token() (*oauth2.Token, error) { } func generateOktaJWT(oktaJWK []byte, cnf *oauth2.Config) (string, error) { - // unmarshal the JWK into a map - var jwkData map[string]string + // Unmarshal the JWK into big ints. + var jwkData struct { + N base64int `json:"n"` + E base64int `json:"e"` + D base64int `json:"d"` + P base64int `json:"p"` + Q base64int `json:"q"` + Dp base64int `json:"dp"` + Dq base64int `json:"dq"` + Qinv base64int `json:"qi"` + } err := json.Unmarshal(oktaJWK, &jwkData) if err != nil { return "", fmt.Errorf("error decoding JWK: %w", err) } - // create an RSA private key from JWK components - decodeBase64 := func(key string) (*big.Int, error) { - data, err := base64.RawURLEncoding.DecodeString(jwkData[key]) - if err != nil { - return nil, fmt.Errorf("error decoding RSA JWK component %s: %w", key, err) - } - return new(big.Int).SetBytes(data), nil + // Create an RSA private key from JWK components. + key := &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: &jwkData.N.Int, + E: int(jwkData.E.Int64()), + }, + D: &jwkData.D.Int, + Primes: []*big.Int{&jwkData.P.Int, &jwkData.Q.Int}, + Precomputed: rsa.PrecomputedValues{ + Dp: &jwkData.Dp.Int, + Dq: &jwkData.Dq.Int, + Qinv: &jwkData.Qinv.Int, + }, } - n, err := decodeBase64("n") - if err != nil { - return "", err - } - e, err := decodeBase64("e") - if err != nil { - return "", err - } - d, err := decodeBase64("d") - if err != nil { - return "", err - } - p, err := decodeBase64("p") - if err != nil { - return "", err + return signJWT(cnf, key) + +} + +// base64int is a JSON decoding shim for base64-encoded big.Int. +type base64int struct { + big.Int +} + +func (i *base64int) UnmarshalJSON(b []byte) error { + src, ok := bytes.CutPrefix(b, []byte{'"'}) + if !ok { + return fmt.Errorf("invalid JSON type: %s", b) } - q, err := decodeBase64("q") - if err != nil { - return "", err + src, ok = bytes.CutSuffix(src, []byte{'"'}) + if !ok { + return fmt.Errorf("invalid JSON type: %s", b) } - dp, err := decodeBase64("dp") + dst := make([]byte, base64.RawURLEncoding.DecodedLen(len(src))) + _, err := base64.RawURLEncoding.Decode(dst, src) if err != nil { - return "", err + return err } - dq, err := decodeBase64("dq") - if err != nil { - return "", err + i.SetBytes(dst) + return nil +} + +func generateOktaJWTPEM(pemdata string, cnf *oauth2.Config) (string, error) { + blk, rest := pem.Decode([]byte(pemdata)) + if rest := bytes.TrimSpace(rest); len(rest) != 0 { + return "", fmt.Errorf("PEM text has trailing data: %s", rest) } - qi, err := decodeBase64("qi") + key, err := x509.ParsePKCS8PrivateKey(blk.Bytes) if err != nil { return "", err } + return signJWT(cnf, key) +} - privateKeyRSA := &rsa.PrivateKey{ - PublicKey: rsa.PublicKey{ - N: n, - E: int(e.Int64()), - }, - D: d, - Primes: []*big.Int{p, q}, - Precomputed: rsa.PrecomputedValues{ - Dp: dp, - Dq: dq, - Qinv: qi, - }, - } - - // create a JWT token using required claims and sign it with the private key +// signJWT creates a JWT token using required claims and sign it with the +// private key. +func signJWT(cnf *oauth2.Config, key any) (string, error) { now := time.Now() tok, err := jwt.NewBuilder().Audience([]string{cnf.Endpoint.TokenURL}). Issuer(cnf.ClientID). @@ -159,11 +184,10 @@ func generateOktaJWT(oktaJWK []byte, cnf *oauth2.Config) (string, error) { if err != nil { return "", err } - signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, privateKeyRSA)) + signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, key)) if err != nil { return "", fmt.Errorf("failed to sign token: %w", err) } - return string(signedToken), nil } diff --git a/x-pack/filebeat/input/cel/config_okta_auth_test.go b/x-pack/filebeat/input/cel/config_okta_auth_test.go new file mode 100644 index 00000000000..fc02a2ec9e7 --- /dev/null +++ b/x-pack/filebeat/input/cel/config_okta_auth_test.go @@ -0,0 +1,88 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cel + +import ( + "testing" + + "github.com/lestrrat-go/jwx/v2/jwt" + "golang.org/x/oauth2" +) + +func TestGenerateOktaJWT(t *testing.T) { + // jwt is a JWT obtained from the Okta integration. + const jwtText = `{ "d": "Cmhokw2MnZfX6da36nnsnQ7IPX9vE6se8_D1NgyL9j9rarYpexhlp45hswcAIFNgWA03NV848Gc0e84AW6wMbyD2E8LPI0Bd8lhdmzRE6L4or2Rxqqjk2Pr2aqGnqs4A0uTijAA7MfPF1zFFdR3EOVx499fEeTiMcLjO83IJCoNiOySDoQgt3KofX5bCbaDy2eiB83rzf0fEcWrWfTY65_Hc2c5lek-1uuF7NpELVzX80p5H-b9MOfLn0BdOGe-mJ2j5bXi-UCQ45Wxj2jdkoA_Qwb4MEtXZjp5LjcM75SrlGfVd99acML2wGZgYLGweJ0sAPDlKzGvj4ve-JT8nNw", "p": "8-UBb4psN0wRPktkh3S48L3ng4T5zR08t7nwXDYNajROrS2j7oq60dtlGY4IwgwcC0c9GDQP7NiN2IpU2uahYkGQ7lDyM_h7UfQWL5fMrsYiKgn2pUgSy5TTT8smkSLbJAD35nAH6PknsQ2PuvOlb4laiC0MXw1Rw4vT9HAEB9M", "q": "0DJkPEN0bECG_6lorlNJgIfoNahVevGKK-Yti1YZ5K-nQCuffPCwPG0oZZo_55y5LODe9W7psxnAt7wxkpAY4lK2hpHTWJSkPjqXWFYIP8trn4RZDShnJXli0i1XqPOqkiVzBZGx5nLtj2bUtmXfIU7-kneHGvLQ5EXcyQW1ISM", "dp": "Ye1PWEPSE5ndSo_m-2RoZXE6pdocmrjkijiEQ-IIHN6HwI0Ux1C4lk5rF4mqBo_qKrUd2Lv-sPB6c7mHPKVhoxwEX0vtE-TvTwacadufeYVgblS1zcNUmJ1XAzDkeV3vc1NYNhRBeM-hmjuBvGTbxh72VLsRvpCQhd186yaW17U", "dq": "jvSK7vZCUrJb_-CLCGgX6DFpuK5FQ43mmg4K58nPLb-Oz_kkId4CpPsu6dToXFi4raAad9wYi-n68i4-u6xF6eFxgyVOQVyPCkug7_7i2ysKUxXFL8u2R3z55edMca4eSQt91y0bQmlXxUeOd0-rzms3UcrQ8igYVyXBXCaXIJE", "qi": "iIY1Y4bzMYIFG7XH7gNP7C-mWi6QH4l9aGRTzPB_gPaFThvc0XKW0S0l82bfp_PPPWg4D4QpDCp7rZ6KhEA8BlNi86Vt3V6F3Hz5XiDa4ikgQNsAXiXLqf83R-y1-cwHjW70PP3U89hmalCRRFfVXcLHV77AVHqbrp9rAIo-X-I", "kty": "RSA", "e": "AQAB", "kid": "koeFQjkyiav_3Qwr3aRinCqCD2LaEHOjFnje7XlkbdI", "n": "xloTY8bAuI5AEo8JursCd7w0LmELCae7JOFaVo9njGrG8tRNqgIdjPyoGY_ABwKkmjcCMLGMA29llFDbry8rB4LTWai-h_jX4_uUUnl52mLX-lO6merL5HEPZF438Ql9Hrxs5yGzT8n865-E_3uwYSBrhTjvlZJeXYUeVHfKo8pJSSsw3RZEjBW4Tt0eFmCZnFErtTyk3oUPaYVP-8YLLAenhUDV4Lm1dC4dxqUj0Oh6XrWgIb-eYHGolMY9g9xbgyd4ir39RodA_1DOjzHWpNfCM-J5ZOtfpuKCAe5__u7L8FT0m56XOxcDoVVsz1J1VNrACWAGbhDWNjyHfL5E2Q" }` + cnf := &oauth2.Config{ + ClientID: "0oaajljpeokFZLyKU5d7", + Scopes: []string{"okta.logs.read"}, + } + got, err := generateOktaJWT([]byte(jwtText), cnf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tok, err := jwt.Parse([]byte(got), jwt.WithVerify(false)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tok.Issuer() != cnf.ClientID { + t.Errorf("unexpected issuer: got:%s want:%s", tok.Issuer(), cnf.ClientID) + } + if tok.Subject() != cnf.ClientID { + t.Errorf("unexpected subject: got:%s want:%s", tok.Subject(), cnf.ClientID) + } +} + +func TestGenerateOktaJWTPEM(t *testing.T) { + // jwtText is generated by https://mkjwk.org/ using the instructions at + // https://developer.okta.com/docs/guides/dpop/nonoktaresourceserver/main/#create-the-json-web-token + const jwtText = ` +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCOuef3HMRhohVT +5kSoAJgV+atpDjkwTwkOq+ImnbBlv75GaApG90w8VpjXjhqN/1KJmwfyrKiquiMq +OPu+o/672Dys5rUAaWSbT7wRF1GjLDDZrM0GHRdV4DGxM/LKI8I5yE1Mx3EzV+D5 +ZLmcRc5U4oEoMwtGpr0zRZ7uUr6a28UQwcUsVIPItc1/9rERlo1WTv8dcaj4ECC3 +2Sc0y/F+9XqwJvLd4Uv6ckzP0Sv4tbDA+7jpD9MneAIUiZ4LVj2cwbBd+YRY6jXx +MkevcCSmSX60clBY1cIFkw1DYHqtdHEwAQcQHLGMoi72xRP2qrdzIPsaTKVYoHVo +WA9vADdHAgMBAAECggEAIlx7jjCsztyYyeQsL05FTzUWoWo9NnYwtgmHnshkCXsK +MiUmJEOxZO1sSqj5l6oakupyFWigCspZYPbrFNCiqVK7+NxqQzkccY/WtT6p9uDS +ufUyPwCN96zMCd952lSVlBe3FH8Hr9a+YQxw60CbFjCZ67WuR0opTsi6JKJjJSDb +TQQZ4qJR97D05I1TgfmO+VO7G/0/dDaNHnnlYz0AnOgZPSyvrU2G5cYye4842EMB +ng81xjHD+xp55JNui/xYkhmYspYhrB2KlEjkKb08OInUjBeaLEAgA1r9yOHsfV/3 +DQzDPRO9iuqx5BfJhdIqUB1aifrye+sbxt9uMBtUgQKBgQDVdfO3GYT+ZycOQG9P +QtdMn6uiSddchVCGFpk331u6M6yafCKjI/MlJDl29B+8R5sVsttwo8/qnV/xd3cn +pY14HpKAsE4l6/Ciagzoj+0NqfPEDhEzbo8CyArcd7pSxt3XxECAfZe2+xivEPHe +gFO60vSFjFtvlLRMDMOmqX3kYQKBgQCrK1DISyQTnD6/axsgh2/ESOmT7n+JRMx/ +YzA7Lxu3zGzUC8/sRDa1C41t054nf5ZXJueYLDSc4kEAPddzISuCLxFiTD2FQ75P +lHWMgsEzQObDm4GPE9cdKOjoAvtAJwbvZcjDa029CDx7aCaDzbNvdmplZ7EUrznR +55U8Wsm8pwKBgBytxTmzZwfbCgdDJvFKNKzpwuCB9TpL+v6Y6Kr2Clfg+26iAPFU +MiWqUUInGGBuamqm5g6jI5sM28gQWeTsvC4IRXyes1Eq+uCHSQax15J/Y+3SSgNT +9kjUYYkvWMwoRcPobRYWSZze7XkP2L8hFJ7EGvAaZGqAWxzgliS9HtnhAoGAONZ/ +UqMw7Zoac/Ga5mhSwrj7ZvXxP6Gqzjofj+eKqrOlB5yMhIX6LJATfH6iq7cAMxxm +Fu/G4Ll4oB3o5wACtI3wldV/MDtYfJBtoCTjBqPsfNOsZ9hMvBATlsc2qwzKjsAb +tFhzTevoOYpSD75EcSS/G8Ec2iN9bagatBnpl00CgYBVqAOFZelNfP7dj//lpk8y +EUAw7ABOq0S9wkpFWTXIVPoBQUipm3iAUqGNPmvr/9ShdZC9xeu5AwKram4caMWJ +ExRhcDP1hFM6CdmSkIYEgBKvN9N0O4Lx1ba34gk74Hm65KXxokjJHOC0plO7c7ok +LNV/bIgMHOMoxiGrwyjAhg== +-----END PRIVATE KEY----- +` + cnf := &oauth2.Config{ + ClientID: "0oaajljpeokFZLyKU5d7", + Scopes: []string{"okta.logs.read"}, + } + got, err := generateOktaJWTPEM(jwtText, cnf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tok, err := jwt.Parse([]byte(got), jwt.WithVerify(false)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tok.Issuer() != cnf.ClientID { + t.Errorf("unexpected issuer: got:%s want:%s", tok.Issuer(), cnf.ClientID) + } + if tok.Subject() != cnf.ClientID { + t.Errorf("unexpected subject: got:%s want:%s", tok.Subject(), cnf.ClientID) + } +} diff --git a/x-pack/filebeat/input/cel/config_test.go b/x-pack/filebeat/input/cel/config_test.go index 0cd404705e2..e4c98b78dc5 100644 --- a/x-pack/filebeat/input/cel/config_test.go +++ b/x-pack/filebeat/input/cel/config_test.go @@ -489,8 +489,8 @@ var oAuth2ValidationTests = []struct { }, }, { - name: "okta requires token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file to be provided", - wantErr: errors.New("okta validation error: token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file must be provided accessing 'auth.oauth2'"), + name: "unique_okta_jwk_token", + wantErr: errors.New("okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided accessing 'auth.oauth2'"), input: map[string]interface{}{ "auth.oauth2": map[string]interface{}{ "provider": "okta", @@ -501,7 +501,7 @@ var oAuth2ValidationTests = []struct { }, }, { - name: "okta oauth2 validation fails if jwk_json is not a valid JSON", + name: "invalid_okta_jwk_json", wantErr: errors.New("the field can't be converted to valid JSON accessing 'auth.oauth2.okta.jwk_json'"), input: map[string]interface{}{ "auth.oauth2": map[string]interface{}{ @@ -514,7 +514,7 @@ var oAuth2ValidationTests = []struct { }, }, { - name: "okta successful oauth2 validation", + name: "okta_successful_oauth2_validation", input: map[string]interface{}{ "auth.oauth2": map[string]interface{}{ "provider": "okta", diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 420c61a1e64..12dd4c4dcec 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -723,7 +723,9 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger) (*http.Client, ) traceLogger := zap.New(core) - trace = httplog.NewLoggingRoundTripper(c.Transport, traceLogger) + const margin = 1e3 // 1OkB ought to be enough room for all the remainder of the trace details. + maxSize := cfg.Resource.Tracer.MaxSize * 1e6 + trace = httplog.NewLoggingRoundTripper(c.Transport, traceLogger, max(0, maxSize-margin)) c.Transport = trace } diff --git a/x-pack/filebeat/input/cometd/input_test.go b/x-pack/filebeat/input/cometd/input_test.go index 101847ebae9..7774ce7c54d 100644 --- a/x-pack/filebeat/input/cometd/input_test.go +++ b/x-pack/filebeat/input/cometd/input_test.go @@ -117,6 +117,8 @@ func TestSingleInput(t *testing.T) { } func TestInputStop_Wait(t *testing.T) { + t.Skip("Flaky test https://github.com/elastic/beats/issues/37987") + expectedHTTPEventCount = 1 defer atomic.StoreUint64(&called, 0) eventsCh := make(chan beat.Event) diff --git a/x-pack/filebeat/input/default-inputs/inputs_other.go b/x-pack/filebeat/input/default-inputs/inputs_other.go index d396d4635a1..91d5917f261 100644 --- a/x-pack/filebeat/input/default-inputs/inputs_other.go +++ b/x-pack/filebeat/input/default-inputs/inputs_other.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -//go:build !aix +//go:build !aix && !windows package inputs @@ -20,8 +20,10 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" + "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" "github.com/elastic/beats/v7/x-pack/filebeat/input/shipper" + "github.com/elastic/beats/v7/x-pack/filebeat/input/websocket" "github.com/elastic/elastic-agent-libs/logp" ) @@ -39,5 +41,7 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 awscloudwatch.Plugin(), lumberjack.Plugin(), shipper.Plugin(log, store), + websocket.Plugin(log, store), + netflow.Plugin(log), } } diff --git a/x-pack/filebeat/input/default-inputs/inputs_windows.go b/x-pack/filebeat/input/default-inputs/inputs_windows.go new file mode 100644 index 00000000000..361883f39ad --- /dev/null +++ b/x-pack/filebeat/input/default-inputs/inputs_windows.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package inputs + +import ( + "github.com/elastic/beats/v7/filebeat/beater" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" + "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" + "github.com/elastic/beats/v7/x-pack/filebeat/input/azureblobstorage" + "github.com/elastic/beats/v7/x-pack/filebeat/input/cel" + "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" + "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics" + "github.com/elastic/beats/v7/x-pack/filebeat/input/etw" + "github.com/elastic/beats/v7/x-pack/filebeat/input/gcs" + "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" + "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" + "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" + "github.com/elastic/beats/v7/x-pack/filebeat/input/shipper" + "github.com/elastic/elastic-agent-libs/logp" +) + +func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2.Plugin { + return []v2.Plugin{ + azureblobstorage.Plugin(log, store), + cel.Plugin(log, store), + cloudfoundry.Plugin(), + entityanalytics.Plugin(log), + gcs.Plugin(log, store), + http_endpoint.Plugin(), + httpjson.Plugin(log, store), + o365audit.Plugin(log, store), + awss3.Plugin(store), + awscloudwatch.Plugin(), + lumberjack.Plugin(), + shipper.Plugin(log, store), + etw.Plugin(), + } +} diff --git a/x-pack/filebeat/input/entityanalytics/provider/activedirectory/internal/activedirectory/activedirectory.go b/x-pack/filebeat/input/entityanalytics/provider/activedirectory/internal/activedirectory/activedirectory.go new file mode 100644 index 00000000000..f2b8d09fb3d --- /dev/null +++ b/x-pack/filebeat/input/entityanalytics/provider/activedirectory/internal/activedirectory/activedirectory.go @@ -0,0 +1,346 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Package activedirectory provides Active Directory user and group query support. +package activedirectory + +import ( + "crypto/tls" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/go-ldap/ldap/v3" +) + +var ( + ErrInvalidDistinguishedName = errors.New("invalid base distinguished name") + ErrGroups = errors.New("failed to get group details") + ErrUsers = errors.New("failed to get user details") +) + +var cnUsers = &ldap.RelativeDN{Attributes: []*ldap.AttributeTypeAndValue{{Type: "CN", Value: "Users"}}} + +// Entry is an Active Directory user entry with associated group membership. +type Entry struct { + ID string `json:"id"` + User map[string]any `json:"user"` + Groups []any `json:"groups,omitempty"` + WhenChanged time.Time `json:"whenChanged"` +} + +// GetDetails returns all the users in the Active directory with the provided base +// on the host with the given ldap url (ldap://, ldaps://, ldapi:// or cldap://). +// Group membership details are collected and added to the returned documents. If +// the group query fails, the user details query will still be attempted, but +// a non-nil error indicating the failure will be returned. If since is non-zero +// only records with whenChanged since that time will be returned. since is +// expected to be configured in a time zone the Active Directory server will +// understand, most likely UTC. +func GetDetails(url, user, pass string, base *ldap.DN, since time.Time, pagingSize uint32, dialer *net.Dialer, tlsconfig *tls.Config) ([]Entry, error) { + if base == nil || len(base.RDNs) == 0 { + return nil, fmt.Errorf("%w: no path", ErrInvalidDistinguishedName) + } + baseDN := base.String() + if !base.RDNs[0].Equal(cnUsers) { + return nil, fmt.Errorf("%w: %s does not have %s", ErrInvalidDistinguishedName, baseDN, cnUsers) + } + + var opts []ldap.DialOpt + if dialer != nil { + opts = append(opts, ldap.DialWithDialer(dialer)) + } + if tlsconfig != nil { + opts = append(opts, ldap.DialWithTLSConfig(tlsconfig)) + } + conn, err := ldap.DialURL(url, opts...) + if err != nil { + return nil, err + } + + err = conn.Bind(user, pass) + if err != nil { + return nil, err + } + defer conn.Unbind() + + var errs []error + + // Format update epoch moment. + var sinceFmtd string + if !since.IsZero() { + const denseTimeLayout = "20060102150405.0Z" // Differs from the const below in resolution and behaviour. + sinceFmtd = since.Format(denseTimeLayout) + } + + // Get groups in the directory. Get all groups independent of the + // since parameter as they may not have changed for changed users. + var groups directory + grps, err := search(conn, baseDN, "(objectClass=group)", pagingSize) + if err != nil { + // Allow continuation if groups query fails, but warn. + errs = []error{fmt.Errorf("%w: %w", ErrGroups, err)} + groups.Entries = entries{} + } else { + groups = collate(grps, nil) + } + + // Get users in the directory... + userFilter := "(objectClass=user)" + if sinceFmtd != "" { + userFilter = "(&(objectClass=user)(whenChanged>=" + sinceFmtd + "))" + } + usrs, err := search(conn, baseDN, userFilter, pagingSize) + if err != nil { + errs = append(errs, fmt.Errorf("%w: %w", ErrUsers, err)) + return nil, errors.Join(errs...) + } + // ...and apply group membership. + users := collate(usrs, groups.Entries) + + // Also collect users that are members of groups that have changed. + if sinceFmtd != "" { + grps, err := search(conn, baseDN, "(&(objectClass=groups)(whenChanged>="+sinceFmtd+"))", pagingSize) + if err != nil { + // Allow continuation if groups query fails, but warn. + errs = append(errs, fmt.Errorf("failed to collect changed groups: %w: %w", ErrGroups, err)) + } else { + groups := collate(grps, nil) + + // Get users of the changed groups + var modGrps []string + for _, e := range groups.Entries { + dn, ok := e["distinguishedName"].(string) + if !ok { + continue + } + modGrps = append(modGrps, dn) + } + if len(modGrps) != 0 { + for i, u := range modGrps { + modGrps[i] = "(memberOf=" + u + ")" + } + query := "(&(objectClass=user)(|" + strings.Join(modGrps, "") + ")" + usrs, err := search(conn, baseDN, query, pagingSize) + if err != nil { + errs = append(errs, fmt.Errorf("failed to collect users of changed groups%w: %w", ErrUsers, err)) + } else { + // ...and apply group membership, inserting into users + // if not present. + for dn, u := range collate(usrs, groups.Entries).Entries { + _, ok := users.Entries[dn] + if ok { + continue + } + users.Entries[dn] = u + } + } + } + } + } + + // Assemble into a set of documents. + docs := make([]Entry, 0, len(users.Entries)) + for id, u := range users.Entries { + user := u["user"].(map[string]any) + var groups []any + switch g := u["groups"].(type) { + case nil: + case []any: + // Do not bother concretising these. + groups = g + } + docs = append(docs, Entry{ID: id, User: user, Groups: groups, WhenChanged: whenChanged(user, groups)}) + } + return docs, errors.Join(errs...) +} + +func whenChanged(user map[string]any, groups []any) time.Time { + l, _ := user["whenChanged"].(time.Time) + for _, g := range groups { + g, ok := g.(map[string]any) + if !ok { + continue + } + gl, ok := g["whenChanged"].(time.Time) + if !ok { + continue + } + if gl.After(l) { + l = gl + } + } + return l +} + +// search performs an LDAP filter search on conn at the LDAP base. If paging +// is non-zero, page sizing will be used. See [ldap.Conn.SearchWithPaging] for +// details. +func search(conn *ldap.Conn, base, filter string, pagingSize uint32) (*ldap.SearchResult, error) { + srch := &ldap.SearchRequest{ + BaseDN: base, + Scope: ldap.ScopeWholeSubtree, + DerefAliases: ldap.NeverDerefAliases, + SizeLimit: 0, + TimeLimit: 0, + TypesOnly: false, + Filter: filter, + Attributes: nil, + Controls: nil, + } + if pagingSize != 0 { + return conn.SearchWithPaging(srch, pagingSize) + } + return conn.Search(srch) +} + +// entries is a set of LDAP entries keyed on the entities distinguished name +// and then the name of the attribute. +type entries map[string]map[string]any + +type directory struct { + Entries entries `json:"entries"` + Referrals []string `json:"referrals"` + Controls []string `json:"controls"` +} + +// collate renders an LDAP search result in to a map[string]any, annotating with +// group information if it is available. Fields with known types will be converted +// from strings to the known type. +// Also included in the returned map is the sets of referrals and controls. +func collate(resp *ldap.SearchResult, groups entries) directory { + dir := directory{ + Entries: make(entries), + } + for _, e := range resp.Entries { + u := make(map[string]any) + m := u + if groups != nil { + m = map[string]any{"user": u} + } + for _, attr := range e.Attributes { + val := entype(attr) + u[attr.Name] = val + if groups != nil && attr.Name == "memberOf" { + switch val := val.(type) { + case []string: + if len(val) != 0 { + grps := make([]any, 0, len(val)) + for _, n := range val { + g, ok := groups[n] + if !ok { + continue + } + grps = append(grps, g) + } + if len(grps) != 0 { + m["groups"] = grps + } + } + + case string: + g, ok := groups[val] + if ok { + m["groups"] = []any{g} + } + } + } + } + dir.Entries[e.DN] = m + } + + // Do we want this information? If not, remove this stanza. If we + // do, we should include the information in the Entries returned + // by the exposed API. + if len(resp.Referrals) != 0 { + dir.Referrals = resp.Referrals + } + if len(resp.Controls) != 0 { + dir.Controls = make([]string, 0, len(resp.Controls)) + for _, e := range resp.Controls { + if e == nil { + continue + } + dir.Controls = append(dir.Controls, e.String()) + } + } + + return dir +} + +// entype converts LDAP attributes with known types to their known type if +// possible, falling back to the string if not. +func entype(attr *ldap.EntryAttribute) any { + if len(attr.Values) == 0 { + return attr.Values + } + switch attr.Name { + case "isCriticalSystemObject", "showInAdvancedViewOnly": + if len(attr.Values) != 1 { + return attr.Values + } + switch { + case strings.EqualFold(attr.Values[0], "true"): + return true + case strings.EqualFold(attr.Values[0], "false"): + return false + default: + return attr.Values[0] + } + case "whenCreated", "whenChanged", "dSCorePropagationData": + var times []time.Time + if len(attr.Values) > 1 { + times = make([]time.Time, 0, len(attr.Values)) + } + for _, v := range attr.Values { + const denseTimeLayout = "20060102150405.999999999Z" + t, err := time.Parse(denseTimeLayout, v) + if err != nil { + return attr.Values + } + if len(attr.Values) == 1 { + return t + } + times = append(times, t) + } + return times + case "accountExpires", "lastLogon", "lastLogonTimestamp", "pwdLastSet": + var times []time.Time + if len(attr.Values) > 1 { + times = make([]time.Time, 0, len(attr.Values)) + } + for _, v := range attr.Values { + ts, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return attr.Values + } + if len(attr.Values) == 1 { + return fromWindowsNT(ts) + } + times = append(times, fromWindowsNT(ts)) + } + return times + case "objectGUID", "objectSid": + if len(attr.ByteValues) == 1 { + return attr.ByteValues[0] + } + return attr.ByteValues + } + if len(attr.Values) == 1 { + return attr.Values[0] + } + return attr.Values +} + +// epochDelta is the unix epoch in ldap time. +const epochDelta = 116444736000000000 + +var unixEpoch = time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) + +func fromWindowsNT(ts int64) time.Time { + return unixEpoch.Add(time.Duration(ts-epochDelta) * 100) +} diff --git a/x-pack/filebeat/input/entityanalytics/provider/activedirectory/internal/activedirectory/activedirectory_test.go b/x-pack/filebeat/input/entityanalytics/provider/activedirectory/internal/activedirectory/activedirectory_test.go new file mode 100644 index 00000000000..80f3d79efa8 --- /dev/null +++ b/x-pack/filebeat/input/entityanalytics/provider/activedirectory/internal/activedirectory/activedirectory_test.go @@ -0,0 +1,112 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package activedirectory + +import ( + "encoding/json" + "flag" + "os" + "sort" + "testing" + "time" + + "github.com/go-ldap/ldap/v3" +) + +var logResponses = flag.Bool("log_response", false, "use to log users/groups returned from the API") + +// Invoke test with something like this: +// +// AD_BASE=CN=Users,DC=,DC=local AD_URL=ldap:// AD_USER=CN=Administrator,CN=Users,DC=,DC=local AD_PASS= go test -v -log_response +func Test(t *testing.T) { + url, ok := os.LookupEnv("AD_URL") + if !ok { + t.Skip("activedirectory tests require ${AD_URL} to be set") + } + baseDN, ok := os.LookupEnv("AD_BASE") + if !ok { + t.Skip("activedirectory tests require ${AD_BASE} to be set") + } + user, ok := os.LookupEnv("AD_USER") + if !ok { + t.Skip("activedirectory tests require ${AD_USER} to be set") + } + pass, ok := os.LookupEnv("AD_PASS") + if !ok { + t.Skip("activedirectory tests require ${AD_PASS} to be set") + } + + base, err := ldap.ParseDN(baseDN) + if err != nil { + t.Fatalf("invalid base distinguished name: %v", err) + } + + var times []time.Time + t.Run("full", func(t *testing.T) { + users, err := GetDetails(url, user, pass, base, time.Time{}, 0, nil, nil) + if err != nil { + t.Fatalf("unexpected error from GetDetails: %v", err) + } + + if len(users) == 0 { + t.Error("expected non-empty result from query") + } + found := false + var gotUsers []any + for _, e := range users { + dn := e.User["distinguishedName"] + gotUsers = append(gotUsers, dn) + if dn == user { + found = true + } + + when, ok := e.User["whenChanged"].(time.Time) + if ok { + times = append(times, when) + } + } + if !found { + t.Errorf("expected login user to be found in directory: got:%q", gotUsers) + } + + if !*logResponses { + return + } + b, err := json.MarshalIndent(users, "", "\t") + if err != nil { + t.Errorf("failed to marshal users for logging: %v", err) + } + t.Logf("user: %s", b) + }) + + t.Run("update", func(t *testing.T) { + sort.Slice(times, func(i, j int) bool { return times[i].Before(times[j]) }) + since := times[0].Add(time.Second) // Step past first entry by a small amount within LDAP resolution. + var want int + // ... and count all entries since then. + for _, when := range times[1:] { + if !since.After(when) { + want++ + } + } + users, err := GetDetails(url, user, pass, base, since, 0, nil, nil) + if err != nil { + t.Fatalf("unexpected error from GetDetails: %v", err) + } + + if len(users) != want { + t.Errorf("unexpected number of results from query since %v: got:%d want:%d", since, len(users), want) + } + + if !*logResponses && !t.Failed() { + return + } + b, err := json.MarshalIndent(users, "", "\t") + if err != nil { + t.Errorf("failed to marshal users for logging: %v", err) + } + t.Logf("user: %s", b) + }) +} diff --git a/x-pack/filebeat/input/etw/_meta/fields.yml b/x-pack/filebeat/input/etw/_meta/fields.yml new file mode 100644 index 00000000000..9a732f73e3b --- /dev/null +++ b/x-pack/filebeat/input/etw/_meta/fields.yml @@ -0,0 +1,111 @@ +- key: winlog + title: "Windows ETW" + description: > + Fields from the ETW input (Event Tracing for Windows). + fields: + + - name: winlog + type: group + description: > + All fields specific to the Windows Event Tracing are defined here. + fields: + + - name: activity_id + type: keyword + required: false + description: > + A globally unique identifier that identifies the current activity. The + events that are published with this identifier are part of the same + activity. + + - name: channel + type: keyword + required: false + description: > + Used to enable special event processing. Channel values below 16 are reserved for use by Microsoft to enable special treatment by the ETW runtime. Channel values 16 and above will be ignored by the ETW runtime (treated the same as channel 0) and can be given user-defined semantics. + + - name: event_data + type: object + object_type: keyword + required: false + description: > + The event-specific data. The content of this object is specific to + any provider and event. + + - name: flags + type: keyword + required: false + description: > + Flags that provide information about the event such as the type of session it was logged to and if the event contains extended data. + + - name: keywords + type: keyword + required: false + description: > + The keywords are used to indicate an event's membership in a set of event categories. + + - name: level + type: keyword + required: false + description: > + Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider. + + - name: opcode + type: keyword + required: false + description: > + The opcode defined in the event. Task and opcode are typically used to + identify the location in the application from where the event was + logged. + + - name: process_id + type: keyword + required: false + description: > + Identifies the process that generated the event. + + - name: provider_guid + type: keyword + required: false + description: > + A globally unique identifier that identifies the provider that logged + the event. + + - name: provider_name + type: keyword + required: false + description: > + The source of the event log record (the application or service that + logged the record). + + - name: session + type: keyword + required: false + description: > + Configured session to forward ETW events from providers to consumers. + + - name: severity + type: keyword + required: false + description: > + Human-readable level of severity. + + - name: task + type: keyword + required: false + description: > + The task defined in the event. Task and opcode are typically used to + identify the location in the application from where the event was + logged. + + - name: thread_id + type: keyword + required: false + description: > + Identifies the thread that generated the event. + + - name: version + type: long + required: false + description: > + Specify the version of a manifest-based event. diff --git a/x-pack/filebeat/input/etw/config.go b/x-pack/filebeat/input/etw/config.go new file mode 100644 index 00000000000..2f3925884f3 --- /dev/null +++ b/x-pack/filebeat/input/etw/config.go @@ -0,0 +1,114 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "fmt" + + "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" +) + +var validTraceLevel = map[string]bool{ + "critical": true, + "error": true, + "warning": true, + "information": true, + "verbose": true, +} + +type config struct { + // Logfile is the path to an .etl file to read from. + Logfile string `config:"file"` + // ProviderGUID is the GUID of an ETW provider. + // Run 'logman query providers' to list the available providers. + ProviderGUID string `config:"provider.guid"` + // ProviderName is the name of an ETW provider. + // Run 'logman query providers' to list the available providers. + ProviderName string `config:"provider.name"` + // SessionName is the name used to create a new session for the + // defined provider. If missing, its default value is the provider ID + // prefixed by 'Elastic-' + SessionName string `config:"session_name"` + // TraceLevel filters all provider events with a level value + // that is less than or equal to this level. + // Allowed values are critical, error, warning, informational, and verbose. + TraceLevel string `config:"trace_level"` + // MatchAnyKeyword is an 8-byte bitmask that enables the filtering of + // events from specific provider subcomponents. The provider will write + // a particular event if the event's keyword bits match any of the bits + // in this bitmask. + // See https://learn.microsoft.com/en-us/message-analyzer/system-etw-provider-event-keyword-level-settings for more details. + // Use logman query providers "" to list the available keywords. + MatchAnyKeyword uint64 `config:"match_any_keyword"` + // An 8-byte bitmask that enables the filtering of events from + // specific provider subcomponents. The provider will write a particular + // event if the event's keyword bits match all of the bits in this bitmask. + // See https://learn.microsoft.com/en-us/message-analyzer/system-etw-provider-event-keyword-level-settings for more details. + MatchAllKeyword uint64 `config:"match_all_keyword"` + // Session is the name of an existing session to read from. + // Run 'logman query -ets' to list existing sessions. + Session string `config:"session"` +} + +func convertConfig(cfg config) etw.Config { + return etw.Config{ + Logfile: cfg.Logfile, + ProviderGUID: cfg.ProviderGUID, + ProviderName: cfg.ProviderName, + SessionName: cfg.SessionName, + TraceLevel: cfg.TraceLevel, + MatchAnyKeyword: cfg.MatchAnyKeyword, + MatchAllKeyword: cfg.MatchAllKeyword, + Session: cfg.Session, + } +} + +func defaultConfig() config { + return config{ + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + } +} + +func (c *config) Validate() error { + if c.ProviderName == "" && c.ProviderGUID == "" && c.Logfile == "" && c.Session == "" { + return fmt.Errorf("provider, existing logfile or running session must be set") + } + + if !validTraceLevel[c.TraceLevel] { + return fmt.Errorf("invalid Trace Level value '%s'", c.TraceLevel) + } + + if c.ProviderGUID != "" { + if c.ProviderName != "" { + return fmt.Errorf("configuration constraint error: provider GUID and provider name cannot be defined together") + } + if c.Logfile != "" { + return fmt.Errorf("configuration constraint error: provider GUID and file cannot be defined together") + } + if c.Session != "" { + return fmt.Errorf("configuration constraint error: provider GUID and existing session cannot be defined together") + } + } + + if c.ProviderName != "" { + if c.Logfile != "" { + return fmt.Errorf("configuration constraint error: provider name and file cannot be defined together") + } + if c.Session != "" { + return fmt.Errorf("configuration constraint error: provider name and existing session cannot be defined together") + } + } + + if c.Logfile != "" { + if c.Session != "" { + return fmt.Errorf("configuration constraint error: file and existing session cannot be defined together") + } + } + + return nil +} diff --git a/x-pack/filebeat/input/etw/config_test.go b/x-pack/filebeat/input/etw/config_test.go new file mode 100644 index 00000000000..d18af17bd64 --- /dev/null +++ b/x-pack/filebeat/input/etw/config_test.go @@ -0,0 +1,140 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + confpkg "github.com/elastic/elastic-agent-libs/config" +) + +func Test_validateConfig(t *testing.T) { + testCases := []struct { + name string // Sub-test name. + config config // Load config parameters. + wantError string // Expected error + }{ + { + name: "valid config", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + SessionName: "MySession-DNSServer", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + }, + { + name: "minimal config", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + }, + { + name: "missing source config", + config: config{ + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "provider, existing logfile or running session must be set", + }, + { + name: "invalid trace level", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + TraceLevel: "failed", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "invalid Trace Level value 'failed'", + }, + { + name: "conflict provider GUID and name", + config: config{ + ProviderGUID: "{eb79061a-a566-4698-1234-3ed2807033a0}", + ProviderName: "Microsoft-Windows-DNSServer", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider GUID and provider name cannot be defined together", + }, + { + name: "conflict provider GUID and logfile", + config: config{ + ProviderGUID: "{eb79061a-a566-4698-1234-3ed2807033a0}", + Logfile: "C:\\Windows\\System32\\winevt\\File.etl", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider GUID and file cannot be defined together", + }, + { + name: "conflict provider GUID and session", + config: config{ + ProviderGUID: "{eb79061a-a566-4698-1234-3ed2807033a0}", + Session: "EventLog-Application", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider GUID and existing session cannot be defined together", + }, + { + name: "conflict provider name and logfile", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + Logfile: "C:\\Windows\\System32\\winevt\\File.etl", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider name and file cannot be defined together", + }, + { + name: "conflict provider name and session", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + Session: "EventLog-Application", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider name and existing session cannot be defined together", + }, + { + name: "conflict logfile and session", + config: config{ + Logfile: "C:\\Windows\\System32\\winevt\\File.etl", + Session: "EventLog-Application", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: file and existing session cannot be defined together", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + c := confpkg.MustNewConfigFrom(tc.config) + config := defaultConfig() + err := c.Unpack(&config) + + // Validate responses + if tc.wantError != "" { + if assert.Error(t, err) { + assert.Contains(t, err.Error(), tc.wantError) + } else { + t.Fatalf("Configuration validation failed. No returned error while expecting '%s'", tc.wantError) + } + } else { + if err != nil { + t.Fatalf("Configuration validation failed. No error expected but got '%v'", err) + } + } + }) + } +} diff --git a/x-pack/filebeat/input/etw/fields.go b/x-pack/filebeat/input/etw/fields.go new file mode 100644 index 00000000000..4ae281b363b --- /dev/null +++ b/x-pack/filebeat/input/etw/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package etw + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "etw", asset.ModuleFieldsPri, AssetEtw); err != nil { + panic(err) + } +} + +// AssetEtw returns asset data. +// This is the base64 encoded zlib format compressed contents of input/etw. +func AssetEtw() string { + return "eJzUVk2P2zYQvftXDHLp7sFGtsDm4EOBIkjQAu2pLnJcjMiRNF1qqJCUVP/7gh/ySmsFaNG4aHwzRb55b+bNkHt4pvMRJhZjmx1A4GDoCG8+sWg7efhw+vRmB6DJK8d9YCtH+GEHAPCRyWgPtbMdhJbiTmDphwB3H0aSACeHiqWB2joocPeHHUCdDh53CWUPgh0tCMRfOPd0hMbZoS8rG/Hj70djChz4nhTXrCDYROciYMUFHYGmmoU0tOToUKBWnJa8UAUeOZyfWF++zQSf6TxZt1x39HlgR/oINRpPiy9fEJBEQGNshcacYRD+PBCwJglcMzkILYaX/z5JU4NzUdTM7QCnllaQFEX7fDhK7ofKsG9Jw8ShhdCyXwZJW9AFsHUK4LFb410iXSdItShC5lbJ+d2TjiUlwcpQrjKaLBB6ZxV5z9Ic4H3mASOagTxUZOwED++SNkee3Eg6WXHwBNUZfmXlrLd12EAPjjB0MUJ1vnjbDRK4o6tIMYZowMqOBBMbAxUBN2Id6Y3zcJfQo6qSaUA/ZxHe3icwhRJRGh5JImG3n03rqUMJrPxGKVJSnjQGvKqGrf4gFRbLeeHpq9bq1FLmsL80YyST7AnKSogZTRZjXwgArzp3bTo5xwqPrKNFRWfsDd21wcbfyoAfI3jupEIGWGrrOoy7Y9mHkEqZLekH1caCxpXII+r10aNWgANM6MHYpsmmjqK4XpyOSUIWD/RnINGkcwKvJRdxN1MdKzbHSC00lD5k0awwEKBkzt956KiryPmWe2ABBE+pzEUSBmqsY9pyrKHxdqPjlwie8z+SS3MyL5XGfQuhdXZoWnhc3QvL4fDqyLvLkYfH1WR5te/hZeP3j49zOy8CvJR8dvhGdmyvrKZbljhHuBBjeSF2gBP652TRsivqDeeeVb6qsiFWkOVKyfKMVblFCir2veGylB4NU7yAF5mY0K/Qcp9s5KWM/RteyT+vL9wSME+BhoTcZYB/aSbNZX1qhv/Ry+EyTtPHnOAV4t+SJOvnwVd3pbeDUzQ/RrI5jG3AkbJOw91rN1kHsQtZUdK1YaIElI/fbygrA/pWmt5bqbkZXLq+81UQbHyLTOh0ehuU91pqiznNPm5SVvzQkdsanvNYuxXtn4YOZe8IdXocmatxek0poH++pTMi/rc2rUIbU/jfDasc75/MqpHcpv2NlebfkvstPe5yjkuc6CGEDoVr8mFfYaxN4fZXAAAA///9F1n8" +} diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go new file mode 100644 index 00000000000..b5b331b3c92 --- /dev/null +++ b/x-pack/filebeat/input/etw/input.go @@ -0,0 +1,266 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "fmt" + "math" + "strconv" + "sync" + "time" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/feature" + "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + + "golang.org/x/sync/errgroup" + "golang.org/x/sys/windows" +) + +const ( + inputName = "etw" +) + +// It abstracts the underlying operations needed to work with ETW, allowing for easier +// testing and decoupling from the Windows-specific ETW API. +type sessionOperator interface { + newSession(config config) (*etw.Session, error) + attachToExistingSession(session *etw.Session) error + createRealtimeSession(session *etw.Session) error + startConsumer(session *etw.Session) error + stopSession(session *etw.Session) error +} + +type realSessionOperator struct{} + +func (op *realSessionOperator) newSession(config config) (*etw.Session, error) { + return etw.NewSession(convertConfig(config)) +} + +func (op *realSessionOperator) attachToExistingSession(session *etw.Session) error { + return session.AttachToExistingSession() +} + +func (op *realSessionOperator) createRealtimeSession(session *etw.Session) error { + return session.CreateRealtimeSession() +} + +func (op *realSessionOperator) startConsumer(session *etw.Session) error { + return session.StartConsumer() +} + +func (op *realSessionOperator) stopSession(session *etw.Session) error { + return session.StopSession() +} + +// etwInput struct holds the configuration and state for the ETW input +type etwInput struct { + log *logp.Logger + config config + etwSession *etw.Session + publisher stateless.Publisher + operator sessionOperator +} + +func Plugin() input.Plugin { + return input.Plugin{ + Name: inputName, + Stability: feature.Beta, + Info: "Collect ETW logs.", + Manager: stateless.NewInputManager(configure), + } +} + +func configure(cfg *conf.C) (stateless.Input, error) { + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + return nil, err + } + + return &etwInput{ + config: conf, + operator: &realSessionOperator{}, + }, nil +} + +func (e *etwInput) Name() string { return inputName } + +func (e *etwInput) Test(_ input.TestContext) error { + return nil +} + +// Run starts the ETW session and processes incoming events. +func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { + var err error + + // Initialize a new ETW session with the provided configuration + e.etwSession, err = e.operator.newSession(e.config) + if err != nil { + return fmt.Errorf("error initializing ETW session: %w", err) + } + e.etwSession.Callback = e.consumeEvent + e.publisher = publisher + + // Set up logger with session information + e.log = ctx.Logger.With("session", e.etwSession.Name) + e.log.Info("Starting " + inputName + " input") + defer e.log.Info(inputName + " input stopped") + + // Handle realtime session creation or attachment + if e.etwSession.Realtime { + if !e.etwSession.NewSession { + // Attach to an existing session + err = e.operator.attachToExistingSession(e.etwSession) + if err != nil { + return fmt.Errorf("unable to retrieve handler: %w", err) + } + e.log.Debug("attached to existing session") + } else { + // Create a new realtime session + err = e.operator.createRealtimeSession(e.etwSession) + if err != nil { + return fmt.Errorf("realtime session could not be created: %w", err) + } + e.log.Debug("created new session") + } + } + + stopConsumer := sync.OnceFunc(e.Close) + defer stopConsumer() + + // Stop the consumer upon input cancellation (shutdown). + go func() { + <-ctx.Cancelation.Done() + stopConsumer() + }() + + // Start a goroutine to consume ETW events + g := new(errgroup.Group) + g.Go(func() error { + e.log.Debug("starting ETW consumer") + defer e.log.Debug("stopped ETW consumer") + if err = e.operator.startConsumer(e.etwSession); err != nil { + return fmt.Errorf("failed running ETW consumer: %w", err) + } + return nil + }) + + return g.Wait() +} + +var ( + // levelToSeverity maps ETW trace levels to names for use in ECS log.level. + levelToSeverity = map[uint8]string{ + 1: "critical", // Abnormal exit or termination events + 2: "error", // Severe error events + 3: "warning", // Warning events such as allocation failures + 4: "information", // Non-error events such as entry or exit events + 5: "verbose", // Detailed trace events + } + + // zeroGUID is the zero-value for a windows.GUID. + zeroGUID = windows.GUID{} +) + +// buildEvent builds the final beat.Event emitted by this input. +func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cfg config) beat.Event { + winlog := map[string]any{ + "activity_guid": h.ActivityId.String(), + "channel": strconv.FormatUint(uint64(h.EventDescriptor.Channel), 10), + "event_data": data, + "flags": strconv.FormatUint(uint64(h.Flags), 10), + "keywords": strconv.FormatUint(h.EventDescriptor.Keyword, 10), + "opcode": strconv.FormatUint(uint64(h.EventDescriptor.Opcode), 10), + "process_id": strconv.FormatUint(uint64(h.ProcessId), 10), + "provider_guid": h.ProviderId.String(), + "session": session.Name, + "task": strconv.FormatUint(uint64(h.EventDescriptor.Task), 10), + "thread_id": strconv.FormatUint(uint64(h.ThreadId), 10), + "version": h.EventDescriptor.Version, + } + // Fallback to the session GUID if there is no provider GUID. + if h.ProviderId == zeroGUID { + winlog["provider_guid"] = session.GUID.String() + } + + event := mapstr.M{ + "code": strconv.FormatUint(uint64(h.EventDescriptor.Id), 10), + "created": time.Now().UTC(), + "kind": "event", + "severity": h.EventDescriptor.Level, + } + if cfg.ProviderName != "" { + event["provider"] = cfg.ProviderName + } + + fields := mapstr.M{ + "event": event, + "winlog": winlog, + } + if level, found := levelToSeverity[h.EventDescriptor.Level]; found { + fields.Put("log.level", level) + } + if cfg.Logfile != "" { + fields.Put("log.file.path", cfg.Logfile) + } + + return beat.Event{ + Timestamp: convertFileTimeToGoTime(uint64(h.TimeStamp)), + Fields: fields, + } +} + +// convertFileTimeToGoTime converts a Windows FileTime to a Go time.Time structure. +func convertFileTimeToGoTime(fileTime64 uint64) time.Time { + // Define the offset between Windows epoch (1601) and Unix epoch (1970) + const epochDifference = 116444736000000000 + if fileTime64 < epochDifference { + // Time is before the Unix epoch, adjust accordingly + return time.Time{} + } + + fileTime := windows.Filetime{ + HighDateTime: uint32(fileTime64 >> 32), + LowDateTime: uint32(fileTime64 & math.MaxUint32), + } + + return time.Unix(0, fileTime.Nanoseconds()).UTC() +} + +func (e *etwInput) consumeEvent(record *etw.EventRecord) uintptr { + if record == nil { + e.log.Error("received null event record") + return 1 + } + + e.log.Debugf("received event with ID %d and user-data length %d", record.EventHeader.EventDescriptor.Id, record.UserDataLength) + + data, err := etw.GetEventProperties(record) + if err != nil { + e.log.Errorw("failed to read event properties", "error", err) + return 1 + } + + evt := buildEvent(data, record.EventHeader, e.etwSession, e.config) + e.publisher.Publish(evt) + + return 0 +} + +// Close stops the ETW session and logs the outcome. +func (e *etwInput) Close() { + if err := e.operator.stopSession(e.etwSession); err != nil { + e.log.Error("failed to shutdown ETW session") + return + } + e.log.Info("successfully shutdown") +} diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go new file mode 100644 index 00000000000..fd2673278d3 --- /dev/null +++ b/x-pack/filebeat/input/etw/input_test.go @@ -0,0 +1,513 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "context" + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + + "golang.org/x/sys/windows" +) + +type mockSessionOperator struct { + // Fields to store function implementations that tests can customize + newSessionFunc func(config config) (*etw.Session, error) + attachToExistingSessionFunc func(session *etw.Session) error + createRealtimeSessionFunc func(session *etw.Session) error + startConsumerFunc func(session *etw.Session) error + stopSessionFunc func(session *etw.Session) error +} + +func (m *mockSessionOperator) newSession(config config) (*etw.Session, error) { + if m.newSessionFunc != nil { + return m.newSessionFunc(config) + } + return nil, nil +} + +func (m *mockSessionOperator) attachToExistingSession(session *etw.Session) error { + if m.attachToExistingSessionFunc != nil { + return m.attachToExistingSessionFunc(session) + } + return nil +} + +func (m *mockSessionOperator) createRealtimeSession(session *etw.Session) error { + if m.createRealtimeSessionFunc != nil { + return m.createRealtimeSessionFunc(session) + } + return nil +} + +func (m *mockSessionOperator) startConsumer(session *etw.Session) error { + if m.startConsumerFunc != nil { + return m.startConsumerFunc(session) + } + return nil +} + +func (m *mockSessionOperator) stopSession(session *etw.Session) error { + if m.stopSessionFunc != nil { + return m.stopSessionFunc(session) + } + return nil +} + +func Test_RunEtwInput_NewSessionError(t *testing.T) { + // Mocks + mockOperator := &mockSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { + return nil, fmt.Errorf("failed creating session '%s'", config.SessionName) + } + + // Setup input + inputCtx := input.Context{ + Cancelation: nil, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + err := etwInput.Run(inputCtx, nil) + assert.EqualError(t, err, "error initializing ETW session: failed creating session 'MySession'") +} + +func Test_RunEtwInput_AttachToExistingSessionError(t *testing.T) { + // Mocks + mockOperator := &mockSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { + mockSession := &etw.Session{ + Name: "MySession", + Realtime: true, + NewSession: false, + } + return mockSession, nil + } + // Setup the mock behavior for AttachToExistingSession + mockOperator.attachToExistingSessionFunc = func(session *etw.Session) error { + return fmt.Errorf("mock error") + } + + // Setup input + inputCtx := input.Context{ + Cancelation: nil, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + err := etwInput.Run(inputCtx, nil) + assert.EqualError(t, err, "unable to retrieve handler: mock error") +} + +func Test_RunEtwInput_CreateRealtimeSessionError(t *testing.T) { + // Mocks + mockOperator := &mockSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { + mockSession := &etw.Session{ + Name: "MySession", + Realtime: true, + NewSession: true, + } + return mockSession, nil + } + // Setup the mock behavior for AttachToExistingSession + mockOperator.attachToExistingSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for CreateRealtimeSession + mockOperator.createRealtimeSessionFunc = func(session *etw.Session) error { + return fmt.Errorf("mock error") + } + + // Setup input + inputCtx := input.Context{ + Cancelation: nil, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + err := etwInput.Run(inputCtx, nil) + assert.EqualError(t, err, "realtime session could not be created: mock error") +} + +func Test_RunEtwInput_StartConsumerError(t *testing.T) { + // Mocks + mockOperator := &mockSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { + mockSession := &etw.Session{ + Name: "MySession", + Realtime: true, + NewSession: true, + } + return mockSession, nil + } + // Setup the mock behavior for AttachToExistingSession + mockOperator.attachToExistingSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for CreateRealtimeSession + mockOperator.createRealtimeSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for StartConsumer + mockOperator.startConsumerFunc = func(session *etw.Session) error { + return fmt.Errorf("mock error") + } + // Setup the mock behavior for StopSession + mockOperator.stopSessionFunc = func(session *etw.Session) error { + return nil + } + + // Setup cancellation + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + // Setup input + inputCtx := input.Context{ + Cancelation: ctx, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + err := etwInput.Run(inputCtx, nil) + assert.EqualError(t, err, "failed running ETW consumer: mock error") +} + +func Test_RunEtwInput_Success(t *testing.T) { + // Mocks + mockOperator := &mockSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { + mockSession := &etw.Session{ + Name: "MySession", + Realtime: true, + NewSession: true, + } + return mockSession, nil + } + // Setup the mock behavior for AttachToExistingSession + mockOperator.attachToExistingSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for CreateRealtimeSession + mockOperator.createRealtimeSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for StartConsumer + mockOperator.startConsumerFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for StopSession + mockOperator.stopSessionFunc = func(session *etw.Session) error { + return nil + } + + // Setup cancellation + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + // Setup input + inputCtx := input.Context{ + Cancelation: ctx, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + go func() { + err := etwInput.Run(inputCtx, nil) + if err != nil { + t.Errorf("Run() error = %v, wantErr %v", err, false) + } + }() + + // Simulate waiting for a condition + time.Sleep(time.Millisecond * 100) + cancelFunc() // Trigger cancellation to test cleanup and goroutine exit +} + +func Test_buildEvent(t *testing.T) { + tests := []struct { + name string + data map[string]any + header etw.EventHeader + session *etw.Session + cfg config + expected mapstr.M + }{ + { + name: "TestStandardData", + data: map[string]any{ + "key": "value", + }, + header: etw.EventHeader{ + Size: 0, + HeaderType: 0, + Flags: 30, + EventProperty: 30, + ThreadId: 80, + ProcessId: 60, + TimeStamp: 133516441890350000, + ProviderId: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + EventDescriptor: etw.EventDescriptor{ + Id: 20, + Version: 90, + Channel: 10, + Level: 1, // Critical + Opcode: 50, + Task: 70, + Keyword: 40, + }, + Time: 0, + ActivityId: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + }, + session: &etw.Session{ + GUID: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + Name: "Elastic-TestProvider", + }, + cfg: config{ + ProviderName: "TestProvider", + }, + + expected: mapstr.M{ + "winlog": map[string]any{ + "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", + "channel": "10", + "event_data": map[string]any{ + "key": "value", + }, + "flags": "30", + "keywords": "40", + "opcode": "50", + "process_id": "60", + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "session": "Elastic-TestProvider", + "task": "70", + "thread_id": "80", + "version": "90", + }, + "event.code": "20", + "event.provider": "TestProvider", + "event.severity": uint8(1), + "log.level": "critical", + }, + }, + { + // This case tests an unmapped severity, empty provider GUID and including logfile + name: "TestAlternativeMetadata", + data: map[string]any{ + "key": "value", + }, + header: etw.EventHeader{ + Size: 0, + HeaderType: 0, + Flags: 30, + EventProperty: 30, + ThreadId: 80, + ProcessId: 60, + TimeStamp: 133516441890350000, + EventDescriptor: etw.EventDescriptor{ + Id: 20, + Version: 90, + Channel: 10, + Level: 17, // Unknown + Opcode: 50, + Task: 70, + Keyword: 40, + }, + Time: 0, + ActivityId: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + }, + session: &etw.Session{ + GUID: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + Name: "Elastic-TestProvider", + }, + cfg: config{ + ProviderName: "TestProvider", + Logfile: "C:\\TestFile", + }, + + expected: mapstr.M{ + "winlog": map[string]any{ + "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", + "channel": "10", + "event_data": map[string]any{ + "key": "value", + }, + "flags": "30", + "keywords": "40", + "opcode": "50", + "process_id": "60", + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "session": "Elastic-TestProvider", + "task": "70", + "thread_id": "80", + "version": "90", + }, + "event.code": "20", + "event.provider": "TestProvider", + "event.severity": uint8(17), + "log.file.path": "C:\\TestFile", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + evt := buildEvent(tt.data, tt.header, tt.session, tt.cfg) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["activity_guid"], evt.Fields["winlog"].(map[string]any)["activity_guid"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["channel"], evt.Fields["winlog"].(map[string]any)["channel"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["event_data"], evt.Fields["winlog"].(map[string]any)["event_data"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["flags"], evt.Fields["winlog"].(map[string]any)["flags"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["keywords"], evt.Fields["winlog"].(map[string]any)["keywords"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["opcode"], evt.Fields["winlog"].(map[string]any)["opcode"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["process_id"], evt.Fields["winlog"].(map[string]any)["process_id"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["provider_guid"], evt.Fields["winlog"].(map[string]any)["provider_guid"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["session"], evt.Fields["winlog"].(map[string]any)["session"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["task"], evt.Fields["winlog"].(map[string]any)["task"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["thread_id"], evt.Fields["winlog"].(map[string]any)["thread_id"]) + mapEv := evt.Fields.Flatten() + + assert.Equal(t, tt.expected["winlog"].(map[string]any)["version"], strconv.Itoa(int(mapEv["winlog.version"].(uint8)))) + assert.Equal(t, tt.expected["event.code"], mapEv["event.code"]) + assert.Equal(t, tt.expected["event.provider"], mapEv["event.provider"]) + assert.Equal(t, tt.expected["event.severity"], mapEv["event.severity"]) + assert.Equal(t, tt.expected["log.file.path"], mapEv["log.file.path"]) + assert.Equal(t, tt.expected["log.level"], mapEv["log.level"]) + }) + } +} + +func Test_convertFileTimeToGoTime(t *testing.T) { + tests := []struct { + name string + fileTime uint64 + want time.Time + }{ + { + name: "TestZeroValue", + fileTime: 0, + want: time.Time{}, + }, + { + name: "TestUnixEpoch", + fileTime: 116444736000000000, // January 1, 1970 (Unix epoch) + want: time.Unix(0, 0), + }, + { + name: "TestActualDate", + fileTime: 133515900000000000, // February 05, 2024, 7:00:00 AM + want: time.Date(2024, 0o2, 0o5, 7, 0, 0, 0, time.UTC), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := convertFileTimeToGoTime(tt.fileTime) + if !got.Equal(tt.want) { + t.Errorf("convertFileTimeToGoTime() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/x-pack/filebeat/input/http_endpoint/handler.go b/x-pack/filebeat/input/http_endpoint/handler.go index 75e34c0928e..0e2620b5b65 100644 --- a/x-pack/filebeat/input/http_endpoint/handler.go +++ b/x-pack/filebeat/input/http_endpoint/handler.go @@ -177,7 +177,9 @@ func (h *handler) logRequest(r *http.Request, status int, respBody []byte) { zap.ByteString("http.response.body.content", respBody), ) } - httplog.LogRequest(h.reqLogger, r, extra...) + // Limit request logging body size to 10kiB. + const maxBodyLen = 10 * (1 << 10) + httplog.LogRequest(h.reqLogger, r, maxBodyLen, extra...) if scheme != "" { r.URL.Scheme = scheme } diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index 94894803777..d05592dfa50 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -6,6 +6,7 @@ package httpjson import ( "context" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -104,6 +105,7 @@ type oAuth2Config struct { // okta specific RSA JWK private key OktaJWKFile string `config:"okta.jwk_file"` OktaJWKJSON common.JSONBlob `config:"okta.jwk_json"` + OktaJWKPEM string `config:"okta.jwk_pem"` } // IsEnabled returns true if the `enable` field is set to true in the yaml. @@ -289,8 +291,26 @@ func (o *oAuth2Config) validateGoogleProvider() error { } func (o *oAuth2Config) validateOktaProvider() error { - if o.TokenURL == "" || o.ClientID == "" || len(o.Scopes) == 0 || (o.OktaJWKJSON == nil && o.OktaJWKFile == "") { - return errors.New("okta validation error: token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file must be provided") + if o.TokenURL == "" || o.ClientID == "" || len(o.Scopes) == 0 { + return errors.New("okta validation error: token_url, client_id, scopes must be provided") + } + var n int + if o.OktaJWKJSON != nil { + n++ + } + if o.OktaJWKFile != "" { + n++ + } + if o.OktaJWKPEM != "" { + n++ + } + if n != 1 { + return errors.New("okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided") + } + // jwk_pem + if o.OktaJWKPEM != "" { + _, err := x509.ParsePKCS1PrivateKey([]byte(o.OktaJWKPEM)) + return err } // jwk_file if o.OktaJWKFile != "" { diff --git a/x-pack/filebeat/input/httpjson/config_okta_auth.go b/x-pack/filebeat/input/httpjson/config_okta_auth.go index 8bf2995d746..c2b4289d9c9 100644 --- a/x-pack/filebeat/input/httpjson/config_okta_auth.go +++ b/x-pack/filebeat/input/httpjson/config_okta_auth.go @@ -5,10 +5,13 @@ package httpjson import ( + "bytes" "context" "crypto/rsa" + "crypto/x509" "encoding/base64" "encoding/json" + "encoding/pem" "fmt" "math/big" "net/http" @@ -43,9 +46,20 @@ func (o *oAuth2Config) fetchOktaOauthClient(ctx context.Context, _ *http.Client) }, } - oktaJWT, err := generateOktaJWT(o.OktaJWKJSON, conf) - if err != nil { - return nil, fmt.Errorf("oauth2 client: error generating Okta JWT: %w", err) + var ( + oktaJWT string + err error + ) + if len(o.OktaJWKPEM) != 0 { + oktaJWT, err = generateOktaJWTPEM(o.OktaJWKPEM, conf) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error generating Okta JWT PEM: %w", err) + } + } else { + oktaJWT, err = generateOktaJWT(o.OktaJWKJSON, conf) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error generating Okta JWT: %w", err) + } } token, err := exchangeForBearerToken(ctx, oktaJWT, conf) @@ -85,70 +99,78 @@ func (ts *oktaTokenSource) Token() (*oauth2.Token, error) { } func generateOktaJWT(oktaJWK []byte, cnf *oauth2.Config) (string, error) { - // unmarshal the JWK into a map - var jwkData map[string]string + // Unmarshal the JWK into big ints. + var jwkData struct { + N base64int `json:"n"` + E base64int `json:"e"` + D base64int `json:"d"` + P base64int `json:"p"` + Q base64int `json:"q"` + Dp base64int `json:"dp"` + Dq base64int `json:"dq"` + Qinv base64int `json:"qi"` + } err := json.Unmarshal(oktaJWK, &jwkData) if err != nil { return "", fmt.Errorf("error decoding JWK: %w", err) } - // create an RSA private key from JWK components - decodeBase64 := func(key string) (*big.Int, error) { - data, err := base64.RawURLEncoding.DecodeString(jwkData[key]) - if err != nil { - return nil, fmt.Errorf("error decoding RSA JWK component %s: %w", key, err) - } - return new(big.Int).SetBytes(data), nil + // Create an RSA private key from JWK components. + key := &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: &jwkData.N.Int, + E: int(jwkData.E.Int64()), + }, + D: &jwkData.D.Int, + Primes: []*big.Int{&jwkData.P.Int, &jwkData.Q.Int}, + Precomputed: rsa.PrecomputedValues{ + Dp: &jwkData.Dp.Int, + Dq: &jwkData.Dq.Int, + Qinv: &jwkData.Qinv.Int, + }, } - n, err := decodeBase64("n") - if err != nil { - return "", err - } - e, err := decodeBase64("e") - if err != nil { - return "", err - } - d, err := decodeBase64("d") - if err != nil { - return "", err - } - p, err := decodeBase64("p") - if err != nil { - return "", err + return signJWT(cnf, key) + +} + +// base64int is a JSON decoding shim for base64-encoded big.Int. +type base64int struct { + big.Int +} + +func (i *base64int) UnmarshalJSON(b []byte) error { + src, ok := bytes.CutPrefix(b, []byte{'"'}) + if !ok { + return fmt.Errorf("invalid JSON type: %s", b) } - q, err := decodeBase64("q") - if err != nil { - return "", err + src, ok = bytes.CutSuffix(src, []byte{'"'}) + if !ok { + return fmt.Errorf("invalid JSON type: %s", b) } - dp, err := decodeBase64("dp") + dst := make([]byte, base64.RawURLEncoding.DecodedLen(len(src))) + _, err := base64.RawURLEncoding.Decode(dst, src) if err != nil { - return "", err + return err } - dq, err := decodeBase64("dq") - if err != nil { - return "", err + i.SetBytes(dst) + return nil +} + +func generateOktaJWTPEM(pemdata string, cnf *oauth2.Config) (string, error) { + blk, rest := pem.Decode([]byte(pemdata)) + if rest := bytes.TrimSpace(rest); len(rest) != 0 { + return "", fmt.Errorf("PEM text has trailing data: %s", rest) } - qi, err := decodeBase64("qi") + key, err := x509.ParsePKCS8PrivateKey(blk.Bytes) if err != nil { return "", err } + return signJWT(cnf, key) +} - privateKeyRSA := &rsa.PrivateKey{ - PublicKey: rsa.PublicKey{ - N: n, - E: int(e.Int64()), - }, - D: d, - Primes: []*big.Int{p, q}, - Precomputed: rsa.PrecomputedValues{ - Dp: dp, - Dq: dq, - Qinv: qi, - }, - } - - // create a JWT token using required claims and sign it with the private key +// signJWT creates a JWT token using required claims and sign it with the private key. +func signJWT(cnf *oauth2.Config, key any) (string, error) { now := time.Now() tok, err := jwt.NewBuilder().Audience([]string{cnf.Endpoint.TokenURL}). Issuer(cnf.ClientID). @@ -159,11 +181,10 @@ func generateOktaJWT(oktaJWK []byte, cnf *oauth2.Config) (string, error) { if err != nil { return "", err } - signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, privateKeyRSA)) + signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, key)) if err != nil { return "", fmt.Errorf("failed to sign token: %w", err) } - return string(signedToken), nil } diff --git a/x-pack/filebeat/input/httpjson/config_okta_auth_test.go b/x-pack/filebeat/input/httpjson/config_okta_auth_test.go new file mode 100644 index 00000000000..2f686af0437 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/config_okta_auth_test.go @@ -0,0 +1,88 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + "testing" + + "github.com/lestrrat-go/jwx/v2/jwt" + "golang.org/x/oauth2" +) + +func TestGenerateOktaJWT(t *testing.T) { + // jwt is a JWT obtained from the Okta integration. + const jwtText = `{ "d": "Cmhokw2MnZfX6da36nnsnQ7IPX9vE6se8_D1NgyL9j9rarYpexhlp45hswcAIFNgWA03NV848Gc0e84AW6wMbyD2E8LPI0Bd8lhdmzRE6L4or2Rxqqjk2Pr2aqGnqs4A0uTijAA7MfPF1zFFdR3EOVx499fEeTiMcLjO83IJCoNiOySDoQgt3KofX5bCbaDy2eiB83rzf0fEcWrWfTY65_Hc2c5lek-1uuF7NpELVzX80p5H-b9MOfLn0BdOGe-mJ2j5bXi-UCQ45Wxj2jdkoA_Qwb4MEtXZjp5LjcM75SrlGfVd99acML2wGZgYLGweJ0sAPDlKzGvj4ve-JT8nNw", "p": "8-UBb4psN0wRPktkh3S48L3ng4T5zR08t7nwXDYNajROrS2j7oq60dtlGY4IwgwcC0c9GDQP7NiN2IpU2uahYkGQ7lDyM_h7UfQWL5fMrsYiKgn2pUgSy5TTT8smkSLbJAD35nAH6PknsQ2PuvOlb4laiC0MXw1Rw4vT9HAEB9M", "q": "0DJkPEN0bECG_6lorlNJgIfoNahVevGKK-Yti1YZ5K-nQCuffPCwPG0oZZo_55y5LODe9W7psxnAt7wxkpAY4lK2hpHTWJSkPjqXWFYIP8trn4RZDShnJXli0i1XqPOqkiVzBZGx5nLtj2bUtmXfIU7-kneHGvLQ5EXcyQW1ISM", "dp": "Ye1PWEPSE5ndSo_m-2RoZXE6pdocmrjkijiEQ-IIHN6HwI0Ux1C4lk5rF4mqBo_qKrUd2Lv-sPB6c7mHPKVhoxwEX0vtE-TvTwacadufeYVgblS1zcNUmJ1XAzDkeV3vc1NYNhRBeM-hmjuBvGTbxh72VLsRvpCQhd186yaW17U", "dq": "jvSK7vZCUrJb_-CLCGgX6DFpuK5FQ43mmg4K58nPLb-Oz_kkId4CpPsu6dToXFi4raAad9wYi-n68i4-u6xF6eFxgyVOQVyPCkug7_7i2ysKUxXFL8u2R3z55edMca4eSQt91y0bQmlXxUeOd0-rzms3UcrQ8igYVyXBXCaXIJE", "qi": "iIY1Y4bzMYIFG7XH7gNP7C-mWi6QH4l9aGRTzPB_gPaFThvc0XKW0S0l82bfp_PPPWg4D4QpDCp7rZ6KhEA8BlNi86Vt3V6F3Hz5XiDa4ikgQNsAXiXLqf83R-y1-cwHjW70PP3U89hmalCRRFfVXcLHV77AVHqbrp9rAIo-X-I", "kty": "RSA", "e": "AQAB", "kid": "koeFQjkyiav_3Qwr3aRinCqCD2LaEHOjFnje7XlkbdI", "n": "xloTY8bAuI5AEo8JursCd7w0LmELCae7JOFaVo9njGrG8tRNqgIdjPyoGY_ABwKkmjcCMLGMA29llFDbry8rB4LTWai-h_jX4_uUUnl52mLX-lO6merL5HEPZF438Ql9Hrxs5yGzT8n865-E_3uwYSBrhTjvlZJeXYUeVHfKo8pJSSsw3RZEjBW4Tt0eFmCZnFErtTyk3oUPaYVP-8YLLAenhUDV4Lm1dC4dxqUj0Oh6XrWgIb-eYHGolMY9g9xbgyd4ir39RodA_1DOjzHWpNfCM-J5ZOtfpuKCAe5__u7L8FT0m56XOxcDoVVsz1J1VNrACWAGbhDWNjyHfL5E2Q" }` + cnf := &oauth2.Config{ + ClientID: "0oaajljpeokFZLyKU5d7", + Scopes: []string{"okta.logs.read"}, + } + got, err := generateOktaJWT([]byte(jwtText), cnf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tok, err := jwt.Parse([]byte(got), jwt.WithVerify(false)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tok.Issuer() != cnf.ClientID { + t.Errorf("unexpected issuer: got:%s want:%s", tok.Issuer(), cnf.ClientID) + } + if tok.Subject() != cnf.ClientID { + t.Errorf("unexpected subject: got:%s want:%s", tok.Subject(), cnf.ClientID) + } +} + +func TestGenerateOktaJWTPEM(t *testing.T) { + // jwtText is generated by https://mkjwk.org/ using the instructions at + // https://developer.okta.com/docs/guides/dpop/nonoktaresourceserver/main/#create-the-json-web-token + const jwtText = ` +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCOuef3HMRhohVT +5kSoAJgV+atpDjkwTwkOq+ImnbBlv75GaApG90w8VpjXjhqN/1KJmwfyrKiquiMq +OPu+o/672Dys5rUAaWSbT7wRF1GjLDDZrM0GHRdV4DGxM/LKI8I5yE1Mx3EzV+D5 +ZLmcRc5U4oEoMwtGpr0zRZ7uUr6a28UQwcUsVIPItc1/9rERlo1WTv8dcaj4ECC3 +2Sc0y/F+9XqwJvLd4Uv6ckzP0Sv4tbDA+7jpD9MneAIUiZ4LVj2cwbBd+YRY6jXx +MkevcCSmSX60clBY1cIFkw1DYHqtdHEwAQcQHLGMoi72xRP2qrdzIPsaTKVYoHVo +WA9vADdHAgMBAAECggEAIlx7jjCsztyYyeQsL05FTzUWoWo9NnYwtgmHnshkCXsK +MiUmJEOxZO1sSqj5l6oakupyFWigCspZYPbrFNCiqVK7+NxqQzkccY/WtT6p9uDS +ufUyPwCN96zMCd952lSVlBe3FH8Hr9a+YQxw60CbFjCZ67WuR0opTsi6JKJjJSDb +TQQZ4qJR97D05I1TgfmO+VO7G/0/dDaNHnnlYz0AnOgZPSyvrU2G5cYye4842EMB +ng81xjHD+xp55JNui/xYkhmYspYhrB2KlEjkKb08OInUjBeaLEAgA1r9yOHsfV/3 +DQzDPRO9iuqx5BfJhdIqUB1aifrye+sbxt9uMBtUgQKBgQDVdfO3GYT+ZycOQG9P +QtdMn6uiSddchVCGFpk331u6M6yafCKjI/MlJDl29B+8R5sVsttwo8/qnV/xd3cn +pY14HpKAsE4l6/Ciagzoj+0NqfPEDhEzbo8CyArcd7pSxt3XxECAfZe2+xivEPHe +gFO60vSFjFtvlLRMDMOmqX3kYQKBgQCrK1DISyQTnD6/axsgh2/ESOmT7n+JRMx/ +YzA7Lxu3zGzUC8/sRDa1C41t054nf5ZXJueYLDSc4kEAPddzISuCLxFiTD2FQ75P +lHWMgsEzQObDm4GPE9cdKOjoAvtAJwbvZcjDa029CDx7aCaDzbNvdmplZ7EUrznR +55U8Wsm8pwKBgBytxTmzZwfbCgdDJvFKNKzpwuCB9TpL+v6Y6Kr2Clfg+26iAPFU +MiWqUUInGGBuamqm5g6jI5sM28gQWeTsvC4IRXyes1Eq+uCHSQax15J/Y+3SSgNT +9kjUYYkvWMwoRcPobRYWSZze7XkP2L8hFJ7EGvAaZGqAWxzgliS9HtnhAoGAONZ/ +UqMw7Zoac/Ga5mhSwrj7ZvXxP6Gqzjofj+eKqrOlB5yMhIX6LJATfH6iq7cAMxxm +Fu/G4Ll4oB3o5wACtI3wldV/MDtYfJBtoCTjBqPsfNOsZ9hMvBATlsc2qwzKjsAb +tFhzTevoOYpSD75EcSS/G8Ec2iN9bagatBnpl00CgYBVqAOFZelNfP7dj//lpk8y +EUAw7ABOq0S9wkpFWTXIVPoBQUipm3iAUqGNPmvr/9ShdZC9xeu5AwKram4caMWJ +ExRhcDP1hFM6CdmSkIYEgBKvN9N0O4Lx1ba34gk74Hm65KXxokjJHOC0plO7c7ok +LNV/bIgMHOMoxiGrwyjAhg== +-----END PRIVATE KEY----- +` + cnf := &oauth2.Config{ + ClientID: "0oaajljpeokFZLyKU5d7", + Scopes: []string{"okta.logs.read"}, + } + got, err := generateOktaJWTPEM(jwtText, cnf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tok, err := jwt.Parse([]byte(got), jwt.WithVerify(false)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tok.Issuer() != cnf.ClientID { + t.Errorf("unexpected issuer: got:%s want:%s", tok.Issuer(), cnf.ClientID) + } + if tok.Subject() != cnf.ClientID { + t.Errorf("unexpected subject: got:%s want:%s", tok.Subject(), cnf.ClientID) + } +} diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 74e72ded332..d88c6ac4a62 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -464,7 +464,7 @@ func TestConfigOauth2Validation(t *testing.T) { }, { name: "okta requires token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file to be provided", - expectedErr: "okta validation error: token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file must be provided accessing 'auth.oauth2'", + expectedErr: "okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided accessing 'auth.oauth2'", input: map[string]interface{}{ "auth.oauth2": map[string]interface{}{ "provider": "okta", diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 17877b60701..50a4f7f20a6 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -253,7 +253,12 @@ func newNetHTTPClient(ctx context.Context, cfg *requestConfig, log *logp.Logger, ) traceLogger := zap.New(core) - netHTTPClient.Transport = httplog.NewLoggingRoundTripper(netHTTPClient.Transport, traceLogger) + const margin = 1e3 // 1OkB ought to be enough room for all the remainder of the trace details. + maxSize := cfg.Tracer.MaxSize*1e6 - margin + if maxSize < 0 { + maxSize = 0 + } + netHTTPClient.Transport = httplog.NewLoggingRoundTripper(netHTTPClient.Transport, traceLogger, maxSize) } if reg != nil { diff --git a/x-pack/filebeat/input/httpjson/value_tpl.go b/x-pack/filebeat/input/httpjson/value_tpl.go index 97bc75a62d9..cf7e43cf8e4 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/value_tpl.go @@ -71,6 +71,7 @@ func (t *valueTpl) Unpack(in string) error { "mul": mul, "now": now, "parseDate": parseDate, + "parseDateInTZ": parseDateInTZ, "parseDuration": parseDuration, "parseTimestamp": parseTimestamp, "parseTimestampMilli": parseTimestampMilli, @@ -194,6 +195,58 @@ func parseDate(date string, layout ...string) time.Time { return t.UTC() } +// parseDateInTZ parses a date string within a specified timezone, returning a time.Time +// 'tz' is the timezone (offset or IANA name) for parsing +func parseDateInTZ(date string, tz string, layout ...string) time.Time { + var ly string + if len(layout) == 0 { + ly = defaultTimeLayout + } else { + ly = layout[0] + } + if found := predefinedLayouts[ly]; found != "" { + ly = found + } + + var loc *time.Location + // Attempt to parse timezone as offset in various formats + for _, format := range []string{"-07", "-0700", "-07:00"} { + t, err := time.Parse(format, tz) + if err != nil { + continue + } + name, offset := t.Zone() + loc = time.FixedZone(name, offset) + break + } + + // If parsing tz as offset fails, try loading location by name + if loc == nil { + var err error + loc, err = time.LoadLocation(tz) + if err != nil { + loc = time.UTC // Default to UTC on error + } + } + + // Using Parse allows us not to worry about the timezone + // as the predefined timezone is applied afterwards + t, err := time.Parse(ly, date) + if err != nil { + return time.Time{} + } + + // Manually create a new time object with the parsed date components and the desired location + // It allows interpreting the parsed time in the specified timezone + year, month, day := t.Date() + hour, min, sec := t.Clock() + nanosec := t.Nanosecond() + localTime := time.Date(year, month, day, hour, min, sec, nanosec, loc) + + // Convert the time to UTC to standardize the output + return localTime.UTC() +} + func formatDate(date time.Time, layouttz ...string) string { var layout, tz string switch { diff --git a/x-pack/filebeat/input/httpjson/value_tpl_test.go b/x-pack/filebeat/input/httpjson/value_tpl_test.go index 487451099ad..4b642a16973 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/value_tpl_test.go @@ -142,6 +142,41 @@ func TestValueTpl(t *testing.T) { paramTr: transformable{}, expectedVal: "2020-11-05 12:25:32 +0000 UTC", }, + { + name: "func parseDateInTZ with RFC3339Nano and timezone offset", + value: `[[ parseDateInTZ "2020-11-05T12:25:32.1234567Z" "-0700" "RFC3339Nano" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 19:25:32.1234567 +0000 UTC", + }, + { + name: "func parseDateInTZ defaults to RFC3339 with implicit offset and timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32+04:00" "-0700" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 19:25:32 +0000 UTC", + }, + { + name: "func parseDateInTZ defaults to RFC3339 with IANA timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32Z" "America/New_York" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 17:25:32 +0000 UTC", + }, + { + name: "func parseDateInTZ with custom layout and timezone name", + value: `[[ parseDateInTZ "Thu Nov 5 12:25:32 2020" "Europe/Paris" "Mon Jan _2 15:04:05 2006" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 11:25:32 +0000 UTC", + }, + { + name: "func parseDateInTZ with invalid timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32Z" "Invalid/Timezone" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 12:25:32 +0000 UTC", + }, { name: "func formatDate", setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index 4f0eb9eb670..eac54d7378f 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -32,9 +32,10 @@ type contextKey string // NewLoggingRoundTripper returns a LoggingRoundTripper that logs requests and // responses to the provided logger. -func NewLoggingRoundTripper(next http.RoundTripper, logger *zap.Logger) *LoggingRoundTripper { +func NewLoggingRoundTripper(next http.RoundTripper, logger *zap.Logger, maxBodyLen int) *LoggingRoundTripper { return &LoggingRoundTripper{ transport: next, + maxBodyLen: maxBodyLen, logger: logger, txBaseID: newID(), txIDCounter: atomic.NewUint64(0), @@ -44,6 +45,7 @@ func NewLoggingRoundTripper(next http.RoundTripper, logger *zap.Logger) *Logging // LoggingRoundTripper is an http.RoundTripper that logs requests and responses. type LoggingRoundTripper struct { transport http.RoundTripper + maxBodyLen int // The maximum length of a body. Longer bodies will be truncated. logger *zap.Logger // Destination logger. txBaseID string // Random value to make transaction IDs unique. txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. @@ -63,6 +65,7 @@ type LoggingRoundTripper struct { // http.request // user_agent.original // http.request.body.content +// http.request.body.truncated // http.request.body.bytes // http.request.mime_type // event.original (the request without body from httputil.DumpRequestOut) @@ -71,6 +74,7 @@ type LoggingRoundTripper struct { // // http.response.status_code // http.response.body.content +// http.response.body.truncated // http.response.body.bytes // http.response.mime_type // event.original (the response without body from httputil.DumpResponse) @@ -86,7 +90,7 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err } } - req, respParts, errorsMessages := logRequest(log, req) + req, respParts, errorsMessages := logRequest(log, req, rt.maxBodyLen) resp, err := rt.transport.RoundTrip(req) if err != nil { @@ -107,7 +111,8 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read response body: %s", err)) } else { respParts = append(respParts, - zap.ByteString("http.response.body.content", body), + zap.ByteString("http.response.body.content", body[:min(len(body), rt.maxBodyLen)]), + zap.Bool("http.response.body.truncated", rt.maxBodyLen < len(body)), zap.Int("http.response.body.bytes", len(body)), zap.String("http.response.mime_type", resp.Header.Get("Content-Type")), ) @@ -143,17 +148,18 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err // http.request // user_agent.original // http.request.body.content +// http.request.body.truncated // http.request.body.bytes // http.request.mime_type // event.original (the request without body from httputil.DumpRequestOut) // // Additional fields in extra will also be logged. -func LogRequest(log *zap.Logger, req *http.Request, extra ...zapcore.Field) *http.Request { - req, _, _ = logRequest(log, req, extra...) +func LogRequest(log *zap.Logger, req *http.Request, maxBodyLen int, extra ...zapcore.Field) *http.Request { + req, _, _ = logRequest(log, req, maxBodyLen, extra...) return req } -func logRequest(log *zap.Logger, req *http.Request, extra ...zapcore.Field) (_ *http.Request, parts []zapcore.Field, errorsMessages []string) { +func logRequest(log *zap.Logger, req *http.Request, maxBodyLen int, extra ...zapcore.Field) (_ *http.Request, parts []zapcore.Field, errorsMessages []string) { reqParts := append([]zapcore.Field{ zap.String("url.original", req.URL.String()), zap.String("url.scheme", req.URL.Scheme), @@ -174,7 +180,8 @@ func logRequest(log *zap.Logger, req *http.Request, extra ...zapcore.Field) (_ * errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read request body: %s", err)) } else { reqParts = append(reqParts, - zap.ByteString("http.request.body.content", body), + zap.ByteString("http.request.body.content", body[:min(len(body), maxBodyLen)]), + zap.Bool("http.request.body.truncated", maxBodyLen < len(body)), zap.Int("http.request.body.bytes", len(body)), zap.String("http.request.mime_type", req.Header.Get("Content-Type")), ) diff --git a/x-pack/filebeat/input/netflow/case.go b/x-pack/filebeat/input/netflow/case.go index 62e2a9aeeff..6e2f74f2c4c 100644 --- a/x-pack/filebeat/input/netflow/case.go +++ b/x-pack/filebeat/input/netflow/case.go @@ -60,9 +60,8 @@ func CamelCaseToSnakeCase(in string) string { } out := make([]rune, 0, len(in)+4) - runes := []rune(in) upperCount := 1 - for _, r := range runes { + for _, r := range in { lr := unicode.ToLower(r) isUpper := lr != r if isUpper { diff --git a/x-pack/filebeat/input/netflow/definitions.go b/x-pack/filebeat/input/netflow/definitions.go index 62e6dc3f1b0..bc00ed5d2fc 100644 --- a/x-pack/filebeat/input/netflow/definitions.go +++ b/x-pack/filebeat/input/netflow/definitions.go @@ -7,7 +7,7 @@ package netflow import ( "errors" "fmt" - "io/ioutil" + "io" "math" "os" "strconv" @@ -95,7 +95,7 @@ func LoadFieldDefinitionsFromFile(path string) (defs fields.FieldDict, err error return nil, err } defer file.Close() - contents, err := ioutil.ReadAll(file) + contents, err := io.ReadAll(file) if err != nil { return nil, err } @@ -169,7 +169,7 @@ func loadFields(def map[interface{}]interface{}, pem uint32, dest fields.FieldDi return fmt.Errorf("bad field ID %d: should have two items (type, name) or one (:skip) (Got %+v)", fieldID, list) } key := fields.Key{ - EnterpriseID: uint32(pem), + EnterpriseID: pem, FieldID: uint16(fieldID), } if _, exists := dest[key]; exists { diff --git a/x-pack/filebeat/input/netflow/input.go b/x-pack/filebeat/input/netflow/input.go index 97f9931f325..addd3d39c25 100644 --- a/x-pack/filebeat/input/netflow/input.go +++ b/x-pack/filebeat/input/netflow/input.go @@ -6,23 +6,26 @@ package netflow import ( "bytes" + "context" "fmt" "net" "sync" "time" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/harvester" - "github.com/elastic/beats/v7/filebeat/input" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/filebeat/inputsource" "github.com/elastic/beats/v7/filebeat/inputsource/udp" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/fields" + conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/go-concert/unison" ) const ( @@ -34,21 +37,76 @@ var ( numDropped = monitoring.NewUint(nil, "filebeat.input.netflow.packets.dropped") numFlows = monitoring.NewUint(nil, "filebeat.input.netflow.flows") aliveInputs atomic.Int - logger *logp.Logger - initLogger sync.Once ) +func Plugin(log *logp.Logger) v2.Plugin { + return v2.Plugin{ + Name: inputName, + Stability: feature.Stable, + Deprecated: false, + Info: "collect and decode packets of netflow protocol", + Manager: &netflowInputManager{ + log: log.Named(inputName), + }, + } +} + +type netflowInputManager struct { + log *logp.Logger +} + +func (im *netflowInputManager) Init(_ unison.Group, _ v2.Mode) error { + return nil +} + +func (im *netflowInputManager) Create(cfg *conf.C) (v2.Input, error) { + inputCfg := defaultConfig + if err := cfg.Unpack(&inputCfg); err != nil { + return nil, err + } + + customFields := make([]fields.FieldDict, len(inputCfg.CustomDefinitions)) + for idx, yamlPath := range inputCfg.CustomDefinitions { + f, err := LoadFieldDefinitionsFromFile(yamlPath) + if err != nil { + return nil, fmt.Errorf("failed parsing custom field definitions from file '%s': %w", yamlPath, err) + } + customFields[idx] = f + } + + dec, err := decoder.NewDecoder(decoder.NewConfig(). + WithProtocols(inputCfg.Protocols...). + WithExpiration(inputCfg.ExpirationTimeout). + WithLogOutput(&logDebugWrapper{Logger: im.log}). + WithCustomFields(customFields...). + WithSequenceResetEnabled(inputCfg.DetectSequenceReset). + WithSharedTemplates(inputCfg.ShareTemplates)) + if err != nil { + return nil, fmt.Errorf("error initializing netflow decoder: %w", err) + } + + input := &netflowInput{ + decoder: dec, + internalNetworks: inputCfg.InternalNetworks, + logger: im.log, + queueSize: inputCfg.PacketQueueSize, + } + + input.udp = udp.New(&inputCfg.Config, input.packetDispatch) + + return input, nil +} + type packet struct { data []byte source net.Addr } type netflowInput struct { - mutex sync.Mutex + mtx sync.Mutex udp *udp.Server decoder *decoder.Decoder - outlet channel.Outleter - forwarder *harvester.Forwarder + client beat.Client internalNetworks []string logger *logp.Logger queueC chan packet @@ -56,11 +114,99 @@ type netflowInput struct { started bool } -func init() { - err := input.Register(inputName, NewInput) +func (n *netflowInput) Name() string { + return inputName +} + +func (n *netflowInput) Test(_ v2.TestContext) error { + return nil +} + +func (n *netflowInput) packetDispatch(data []byte, metadata inputsource.NetworkMetadata) { + select { + case n.queueC <- packet{data, metadata.RemoteAddr}: + numPackets.Inc() + default: + numDropped.Inc() + } +} + +func (n *netflowInput) Run(context v2.Context, connector beat.PipelineConnector) error { + n.mtx.Lock() + if n.started { + n.mtx.Unlock() + return nil + } + + n.started = true + n.mtx.Unlock() + + n.logger.Info("Starting netflow input") + + n.logger.Info("Connecting to beat event publishing") + client, err := connector.ConnectWith(beat.ClientConfig{ + PublishMode: beat.DefaultGuarantees, + Processing: beat.ProcessingConfig{ + // This input only produces events with basic types so normalization + // is not required. + EventNormalization: boolPtr(false), + }, + CloseRef: context.Cancelation, + EventListener: nil, + }) + if err != nil { + n.logger.Errorw("Failed connecting to beat event publishing", "error", err) + n.stop() + return err + } + + n.logger.Info("Starting netflow decoder") + if err := n.decoder.Start(); err != nil { + n.logger.Errorw("Failed to start netflow decoder", "error", err) + n.stop() + return err + } + + n.queueC = make(chan packet, n.queueSize) + + n.logger.Info("Starting udp server") + err = n.udp.Start() if err != nil { - panic(err) + n.logger.Errorf("Failed to start udp server: %v", err) + n.stop() + return err + } + + if aliveInputs.Inc() == 1 && n.logger.IsDebug() { + go n.statsLoop(ctxtool.FromCanceller(context.Cancelation)) } + defer aliveInputs.Dec() + + go func() { + <-context.Cancelation.Done() + n.stop() + }() + + for packet := range n.queueC { + flows, err := n.decoder.Read(bytes.NewBuffer(packet.data), packet.source) + if err != nil { + n.logger.Warnf("Error parsing NetFlow packet of length %d from %s: %v", len(packet.data), packet.source, err) + continue + } + + fLen := len(flows) + if fLen == 0 { + continue + } + evs := make([]beat.Event, fLen) + numFlows.Add(uint64(fLen)) + for i, flow := range flows { + evs[i] = toBeatEvent(flow, n.internalNetworks) + } + client.PublishAll(evs) + } + + return nil } // An adapter so that logp.Logger can be used as a log.Logger. @@ -83,171 +229,72 @@ func (w *logDebugWrapper) Write(p []byte) (n int, err error) { return n, nil } -// NewInput creates a new Netflow input -func NewInput( - cfg *conf.C, - connector channel.Connector, - context input.Context, -) (input.Input, error) { - initLogger.Do(func() { - logger = logp.NewLogger(inputName) - }) - out, err := connector.Connect(cfg) - if err != nil { - return nil, err - } +// stop stops the netflow input +func (n *netflowInput) stop() { + n.mtx.Lock() + defer n.mtx.Unlock() - config := defaultConfig - if err = cfg.Unpack(&config); err != nil { - out.Close() - return nil, err + if !n.started { + return } - var customFields []fields.FieldDict - for _, yamlPath := range config.CustomDefinitions { - f, err := LoadFieldDefinitionsFromFile(yamlPath) - if err != nil { - return nil, fmt.Errorf("failed parsing custom field definitions from file '%s': %w", yamlPath, err) - } - customFields = append(customFields, f) + if n.udp != nil { + n.udp.Stop() } - decoder, err := decoder.NewDecoder(decoder.NewConfig(). - WithProtocols(config.Protocols...). - WithExpiration(config.ExpirationTimeout). - WithLogOutput(&logDebugWrapper{Logger: logger}). - WithCustomFields(customFields...). - WithSequenceResetEnabled(config.DetectSequenceReset). - WithSharedTemplates(config.ShareTemplates)) - if err != nil { - return nil, fmt.Errorf("error initializing netflow decoder: %w", err) - } - - input := &netflowInput{ - outlet: out, - internalNetworks: config.InternalNetworks, - forwarder: harvester.NewForwarder(out), - decoder: decoder, - logger: logger, - queueSize: config.PacketQueueSize, - } - - input.udp = udp.New(&config.Config, input.packetDispatch) - return input, nil -} - -func (p *netflowInput) Publish(events []beat.Event) error { - for _, evt := range events { - p.forwarder.Send(evt) - } - return nil -} -// Run starts listening for NetFlow events over the network. -func (p *netflowInput) Run() { - p.mutex.Lock() - defer p.mutex.Unlock() - - if !p.started { - logger.Info("Starting UDP input") - - if err := p.decoder.Start(); err != nil { - logger.Errorw("Failed to start netflow decoder", "error", err) - p.outlet.Close() - return - } - - p.queueC = make(chan packet, p.queueSize) - err := p.udp.Start() - if err != nil { - logger.Errorf("Error running harvester: %v", err) - p.outlet.Close() - p.decoder.Stop() - close(p.queueC) - return + if n.decoder != nil { + if err := n.decoder.Stop(); err != nil { + n.logger.Errorw("Error stopping decoder", "error", err) } + } - go p.recvRoutine() - // Only the first active input launches the stats thread - if aliveInputs.Inc() == 1 && logger.IsDebug() { - go p.statsLoop() + if n.client != nil { + if err := n.client.Close(); err != nil { + n.logger.Errorw("Error closing beat client", "error", err) } - p.started = true } -} - -// Stop stops the UDP input -func (p *netflowInput) Stop() { - p.mutex.Lock() - defer p.mutex.Unlock() - if p.started { - aliveInputs.Dec() - defer p.outlet.Close() - defer close(p.queueC) - logger.Info("Stopping UDP input") - p.udp.Stop() - p.started = false - } -} + close(n.queueC) -// Wait suspends the UDP input -func (p *netflowInput) Wait() { - p.Stop() + n.started = false } -func (p *netflowInput) statsLoop() { +func (n *netflowInput) statsLoop(ctx context.Context) { prevPackets := numPackets.Get() prevFlows := numFlows.Get() prevDropped := numDropped.Get() // The stats thread only monitors queue length for the first input - prevQueue := len(p.queueC) + prevQueue := len(n.queueC) t := time.NewTicker(time.Second) defer t.Stop() - for range t.C { - packets := numPackets.Get() - flows := numFlows.Get() - dropped := numDropped.Get() - queue := len(p.queueC) - if packets > prevPackets || flows > prevFlows || dropped > prevDropped || queue > prevQueue { - logger.Debugf("Stats total:[ packets=%d dropped=%d flows=%d queue_len=%d ] delta:[ packets/s=%d dropped/s=%d flows/s=%d queue_len/s=%+d ]", - packets, dropped, flows, queue, packets-prevPackets, dropped-prevDropped, flows-prevFlows, queue-prevQueue) - prevFlows = flows - prevPackets = packets - prevQueue = queue - prevDropped = dropped - } else { - p.mutex.Lock() + for { + select { + case <-t.C: + packets := numPackets.Get() + flows := numFlows.Get() + dropped := numDropped.Get() + queue := len(n.queueC) + if packets > prevPackets || flows > prevFlows || dropped > prevDropped || queue > prevQueue { + n.logger.Debugf("Stats total:[ packets=%d dropped=%d flows=%d queue_len=%d ] delta:[ packets/s=%d dropped/s=%d flows/s=%d queue_len/s=%+d ]", + packets, dropped, flows, queue, packets-prevPackets, dropped-prevDropped, flows-prevFlows, queue-prevQueue) + prevFlows = flows + prevPackets = packets + prevQueue = queue + prevDropped = dropped + continue + } + + n.mtx.Lock() count := aliveInputs.Load() - p.mutex.Unlock() + n.mtx.Unlock() if count == 0 { - break + return } - } - } -} - -func (p *netflowInput) packetDispatch(data []byte, metadata inputsource.NetworkMetadata) { - select { - case p.queueC <- packet{data, metadata.RemoteAddr}: - numPackets.Inc() - default: - numDropped.Inc() - } -} -func (p *netflowInput) recvRoutine() { - for packet := range p.queueC { - flows, err := p.decoder.Read(bytes.NewBuffer(packet.data), packet.source) - if err != nil { - p.logger.Warnf("Error parsing NetFlow packet of length %d from %s: %v", len(packet.data), packet.source, err) - } - if n := len(flows); n > 0 { - evs := make([]beat.Event, n) - numFlows.Add(uint64(n)) - for i, flow := range flows { - evs[i] = toBeatEvent(flow, p.internalNetworks) - } - p.Publish(evs) + case <-ctx.Done(): + return } } } + +func boolPtr(b bool) *bool { return &b } diff --git a/x-pack/filebeat/input/netflow/input_test.go b/x-pack/filebeat/input/netflow/input_test.go index 506cf48db16..2f78cf83c7e 100644 --- a/x-pack/filebeat/input/netflow/input_test.go +++ b/x-pack/filebeat/input/netflow/input_test.go @@ -9,11 +9,23 @@ package netflow import ( "testing" - "github.com/elastic/beats/v7/filebeat/input/inputtest" + "github.com/elastic/beats/v7/libbeat/tests/resources" + + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/stretchr/testify/require" ) func TestNewInputDone(t *testing.T) { - config := mapstr.M{} - inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) + + goroutines := resources.NewGoroutinesChecker() + defer goroutines.Check(t) + + config, err := conf.NewConfigFrom(mapstr.M{}) + require.NoError(t, err) + + _, err = Plugin(logp.NewLogger("netflow_test")).Manager.Create(config) + require.NoError(t, err) } diff --git a/x-pack/filebeat/input/netflow/netflow_test.go b/x-pack/filebeat/input/netflow/netflow_test.go index b567d67bc43..a7cf3038173 100644 --- a/x-pack/filebeat/input/netflow/netflow_test.go +++ b/x-pack/filebeat/input/netflow/netflow_test.go @@ -6,26 +6,33 @@ package netflow import ( "bytes" + "context" "encoding/binary" "encoding/json" "flag" - "io/ioutil" "net" "os" "path/filepath" "strings" "testing" + "time" "github.com/google/gopacket" "github.com/google/gopacket/pcap" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/dockerlogbeat/pipelinemock" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/protocol" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/record" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/test" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) var ( @@ -60,6 +67,108 @@ type TestResult struct { Flows []beat.Event `json:"events,omitempty"` } +func newV2Context() (v2.Context, func()) { + ctx, cancel := context.WithCancel(context.Background()) + return v2.Context{ + Logger: logp.NewLogger("netflow_test"), + ID: "test_id", + Cancelation: ctx, + }, cancel +} + +func TestNetFlow(t *testing.T) { + pcaps, err := filepath.Glob(filepath.Join(pcapDir, "*.pcap")) + if err != nil { + t.Fatal(err) + } + + for _, file := range pcaps { + testName := strings.TrimSuffix(filepath.Base(file), ".pcap") + + t.Run(testName, func(t *testing.T) { + + pluginCfg, err := conf.NewConfigFrom(mapstr.M{}) + require.NoError(t, err) + + netflowPlugin, err := Plugin(logp.NewLogger("netflow_test")).Manager.Create(pluginCfg) + require.NoError(t, err) + + mockPipeline := &pipelinemock.MockPipelineConnector{} + + ctx, cancelFn := newV2Context() + errChan := make(chan error) + go func() { + defer close(errChan) + errChan <- netflowPlugin.Run(ctx, mockPipeline) + }() + + defer cancelFn() + + require.Eventually(t, mockPipeline.HasConnectedClients, 5*time.Second, 100*time.Millisecond, + "no client has connected to the pipeline") + + udpAddr, err := net.ResolveUDPAddr("udp", defaultConfig.Config.Host) + require.NoError(t, err) + + conn, err := net.DialUDP("udp", nil, udpAddr) + require.NoError(t, err) + + f, err := pcap.OpenOffline(file) + require.NoError(t, err) + defer f.Close() + + goldenData := readGoldenFile(t, filepath.Join(goldenDir, testName+".pcap.golden.json")) + + // Process packets in PCAP and get flow records. + var totalBytes, totalPackets int + packetSource := gopacket.NewPacketSource(f, f.LinkType()) + for pkt := range packetSource.Packets() { + payloadData := pkt.TransportLayer().LayerPayload() + + n, err := conn.Write(payloadData) + require.NoError(t, err) + totalBytes += n + totalPackets++ + } + + require.Eventually(t, func() bool { + return len(mockPipeline.GetAllEvents()) == len(goldenData.Flows) + }, 5*time.Second, 100*time.Millisecond, + "got a different number of events than expected") + + for _, event := range goldenData.Flows { + // fields that cannot be matched at runtime + _ = event.Delete("netflow.exporter.address") + _ = event.Delete("event.created") + _ = event.Delete("observer.ip") + } + + publishedEvents := mockPipeline.GetAllEvents() + for _, event := range publishedEvents { + // fields that cannot be matched at runtime + _ = event.Delete("netflow.exporter.address") + _ = event.Delete("event.created") + _ = event.Delete("observer.ip") + } + + require.EqualValues(t, goldenData, normalize(t, TestResult{ + Name: goldenData.Name, + Error: "", + Flows: publishedEvents, + })) + + cancelFn() + select { + case err := <-errChan: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("netflow plugin did not stop") + } + + }) + } +} + func TestPCAPFiles(t *testing.T) { pcaps, err := filepath.Glob(filepath.Join(pcapDir, "*.pcap")) if err != nil { @@ -83,7 +192,7 @@ func TestPCAPFiles(t *testing.T) { t.Fatal(err) } - err = ioutil.WriteFile(goldenName, data, 0o644) + err = os.WriteFile(goldenName, data, 0o644) if err != nil { t.Fatal(err) } @@ -115,7 +224,7 @@ func TestDatFiles(t *testing.T) { t.Fatal(err) } - err = ioutil.WriteFile(goldenName, data, 0o644) + err = os.WriteFile(goldenName, data, 0o644) if err != nil { t.Fatal(err) } @@ -141,7 +250,7 @@ func TestDatFiles(t *testing.T) { } func readDatTests(t testing.TB) *DatTests { - data, err := ioutil.ReadFile("testdata/dat_tests.yaml") + data, err := os.ReadFile("testdata/dat_tests.yaml") if err != nil { t.Fatal(err) } @@ -179,7 +288,7 @@ func getFlowsFromDat(t testing.TB, name string, testCase TestCase) TestResult { source := test.MakeAddress(t, datSourceIP+":4444") var events []beat.Event for _, f := range testCase.Files { - dat, err := ioutil.ReadFile(filepath.Join(datDir, f)) + dat, err := os.ReadFile(filepath.Join(datDir, f)) if err != nil { t.Fatal(err) } @@ -268,7 +377,7 @@ func normalize(t testing.TB, result TestResult) TestResult { } func readGoldenFile(t testing.TB, file string) TestResult { - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { t.Fatal(err) } @@ -343,7 +452,7 @@ func TestReverseFlows(t *testing.T) { }, } - var evs []beat.Event + evs := make([]beat.Event, 0, len(flows)) for _, f := range flows { evs = append(evs, toBeatEvent(f, []string{"private"})) } diff --git a/x-pack/filebeat/input/websocket/cel.go b/x-pack/filebeat/input/websocket/cel.go new file mode 100644 index 00000000000..11c2e7ad8f1 --- /dev/null +++ b/x-pack/filebeat/input/websocket/cel.go @@ -0,0 +1,99 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "regexp" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/checker/decls" + + "github.com/elastic/beats/v7/libbeat/version" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/useragent" + "github.com/elastic/mito/lib" +) + +var ( + // mimetypes holds supported MIME type mappings. + mimetypes = map[string]interface{}{ + "application/gzip": func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) }, + "application/x-ndjson": lib.NDJSON, + "application/zip": lib.Zip, + "text/csv; header=absent": lib.CSVNoHeader, + "text/csv; header=present": lib.CSVHeader, + "text/csv;header=absent": lib.CSVNoHeader, + "text/csv;header=present": lib.CSVHeader, + } +) + +func regexpsFromConfig(cfg config) (map[string]*regexp.Regexp, error) { + if len(cfg.Regexps) == 0 { + return nil, nil + } + patterns := make(map[string]*regexp.Regexp) + for name, expr := range cfg.Regexps { + var err error + patterns[name], err = regexp.Compile(expr) + if err != nil { + return nil, err + } + } + return patterns, nil +} + +// The Filebeat user-agent is provided to the program as useragent. +var userAgent = useragent.UserAgent("Filebeat", version.GetDefaultVersion(), version.Commit(), version.BuildTime().String()) + +func newProgram(ctx context.Context, src, root string, patterns map[string]*regexp.Regexp, log *logp.Logger) (cel.Program, *cel.Ast, error) { + opts := []cel.EnvOption{ + cel.Declarations(decls.NewVar(root, decls.Dyn)), + cel.OptionalTypes(cel.OptionalTypesVersion(lib.OptionalTypesVersion)), + lib.Collections(), + lib.Crypto(), + lib.JSON(nil), + lib.Strings(), + lib.Time(), + lib.Try(), + lib.Debug(debug(log)), + lib.MIME(mimetypes), + lib.Regexp(patterns), + lib.Globals(map[string]interface{}{ + "useragent": userAgent, + }), + } + + env, err := cel.NewEnv(opts...) + if err != nil { + return nil, nil, fmt.Errorf("failed to create env: %w", err) + } + + ast, iss := env.Compile(src) + if iss.Err() != nil { + return nil, nil, fmt.Errorf("failed compilation: %w", iss.Err()) + } + + prg, err := env.Program(ast) + if err != nil { + return nil, nil, fmt.Errorf("failed program instantiation: %w", err) + } + return prg, ast, nil +} + +func debug(log *logp.Logger) func(string, any) { + log = log.Named("websocket_debug") + return func(tag string, value any) { + level := "DEBUG" + if _, ok := value.(error); ok { + level = "ERROR" + } + + log.Debugw(level, "tag", tag, "value", value) + } +} diff --git a/x-pack/filebeat/input/websocket/config.go b/x-pack/filebeat/input/websocket/config.go new file mode 100644 index 00000000000..1a961f3c162 --- /dev/null +++ b/x-pack/filebeat/input/websocket/config.go @@ -0,0 +1,105 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "context" + "fmt" + "net/url" + "regexp" + + "github.com/elastic/elastic-agent-libs/logp" +) + +type config struct { + // Program is the CEL program to be run for each polling. + Program string `config:"program"` + // Regexps is the set of regular expression to be made + // available to the program. + Regexps map[string]string `config:"regexp"` + // State is the initial state to be provided to the + // program. If it has a cursor field, that field will + // be overwritten by any stored cursor, but will be + // available if no stored cursor exists. + State map[string]interface{} `config:"state"` + // Auth is the authentication config for connection. + Auth authConfig `config:"auth"` + // URL is the websocket url to connect to. + URL *urlConfig `config:"url" validate:"required"` + // Redact is the debug log state redaction configuration. + Redact *redact `config:"redact"` +} + +type redact struct { + // Fields indicates which fields to apply redaction to prior + // to logging. + Fields []string `config:"fields"` + // Delete indicates that fields should be completely deleted + // before logging rather than redaction with a "*". + Delete bool `config:"delete"` +} + +type authConfig struct { + // Custom auth config to use for authentication. + CustomAuth *customAuthConfig `config:"custom"` + // Baerer token to use for authentication. + BearerToken string `config:"bearer_token"` + // Basic auth token to use for authentication. + BasicToken string `config:"basic_token"` +} + +type customAuthConfig struct { + // Custom auth config to use for authentication. + Header string `config:"header"` + Value string `config:"value"` +} +type urlConfig struct { + *url.URL +} + +func (u *urlConfig) Unpack(in string) error { + parsed, err := url.Parse(in) + if err != nil { + return err + } + u.URL = parsed + return nil +} + +func (c config) Validate() error { + if c.Redact == nil { + logp.L().Named("input.websocket").Warn("missing recommended 'redact' configuration: " + + "see documentation for details: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-websocket.html#_redact") + } + _, err := regexpsFromConfig(c) + if err != nil { + return fmt.Errorf("failed to check regular expressions: %w", err) + } + + var patterns map[string]*regexp.Regexp + if len(c.Regexps) != 0 { + patterns = map[string]*regexp.Regexp{".": nil} + } + if c.Program != "" { + _, _, err = newProgram(context.Background(), c.Program, root, patterns, logp.L().Named("input.websocket")) + if err != nil { + return fmt.Errorf("failed to check program: %w", err) + } + } + err = checkURLScheme(c.URL) + if err != nil { + return err + } + return nil +} + +func checkURLScheme(url *urlConfig) error { + switch url.Scheme { + case "ws", "wss": + return nil + default: + return fmt.Errorf("unsupported scheme: %s", url.Scheme) + } +} diff --git a/x-pack/filebeat/input/websocket/config_test.go b/x-pack/filebeat/input/websocket/config_test.go new file mode 100644 index 00000000000..021bf89056f --- /dev/null +++ b/x-pack/filebeat/input/websocket/config_test.go @@ -0,0 +1,121 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +var configTests = []struct { + name string + config map[string]interface{} + wantErr error +}{ + { + name: "invalid_url_scheme", + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + "url": "http://localhost:8080", + }, + wantErr: fmt.Errorf("unsupported scheme: http accessing config"), + }, + { + name: "missing_url", + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + }, + wantErr: fmt.Errorf("missing required field accessing 'url'"), + }, + { + name: "invalid_program", + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": has(state.cursor) && inner_body.ts > state.cursor.last_updated ? + [inner_body] + : + null, + })`, + "url": "wss://localhost:443/v1/stream", + }, + wantErr: fmt.Errorf("failed to check program: failed compilation: ERROR: :3:79: found no matching overload for '_?_:_' applied to '(bool, list(dyn), null)'\n | \"events\": has(state.cursor) && inner_body.ts > state.cursor.last_updated ? \n | ..............................................................................^ accessing config"), + }, + { + name: "invalid_regexps", + config: map[string]interface{}{ + "regexp": map[string]interface{}{ + "products": "(?i)(xq>)d+)", + "solutions": "(?i)(Search|Observability|Security)", + }, + "url": "wss://localhost:443/v1/stream", + }, + wantErr: fmt.Errorf("failed to check regular expressions: error parsing regexp: unexpected ): `(?i)(xq>)d+)` accessing config"), + }, + { + name: "valid_regexps", + config: map[string]interface{}{ + "regexp": map[string]interface{}{ + "products": "(?i)(Elasticsearch|Beats|Logstash|Kibana)", + "solutions": "(?i)(Search|Observability|Security)", + }, + "url": "wss://localhost:443/v1/stream", + }, + }, + { + name: "valid_config", + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + "url": "wss://localhost:443/v1/stream", + "regexp": map[string]interface{}{ + "products": "(?i)(Elasticsearch|Beats|Logstash|Kibana)", + "solutions": "(?i)(Search|Observability|Security)", + }, + "state": map[string]interface{}{ + "cursor": map[string]int{ + "last_updated": 1502908200, + }, + }, + }, + }, +} + +func TestConfig(t *testing.T) { + logp.TestingSetup() + for _, test := range configTests { + t.Run(test.name, func(t *testing.T) { + cfg := conf.MustNewConfigFrom(test.config) + conf := config{} + // Make sure we pass the redact requirement. + conf.Redact = &redact{} + err := cfg.Unpack(&conf) + + switch { + case err == nil && test.wantErr != nil: + t.Fatalf("expected error unpacking config: %v", test.wantErr) + case err != nil && test.wantErr == nil: + t.Fatalf("unexpected error unpacking config: %v", err) + case err != nil && test.wantErr != nil: + assert.EqualError(t, err, test.wantErr.Error()) + default: + // no error + } + }) + } +} diff --git a/x-pack/filebeat/input/websocket/input.go b/x-pack/filebeat/input/websocket/input.go new file mode 100644 index 00000000000..c48ce177931 --- /dev/null +++ b/x-pack/filebeat/input/websocket/input.go @@ -0,0 +1,379 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "context" + "errors" + "fmt" + "reflect" + "time" + + "github.com/google/cel-go/cel" + "github.com/gorilla/websocket" + "google.golang.org/protobuf/types/known/structpb" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/feature" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/mito/lib" +) + +type input struct { + time func() time.Time + cfg config +} + +const ( + inputName string = "websocket" + root string = "state" +) + +func Plugin(log *logp.Logger, store inputcursor.StateStore) v2.Plugin { + return v2.Plugin{ + Name: inputName, + Stability: feature.Experimental, + Deprecated: false, + Info: "Websocket Input", + Doc: "Collect data from websocket api endpoints", + Manager: NewInputManager(log, store), + } +} + +func (input) Name() string { return inputName } + +func (input) Test(src inputcursor.Source, _ v2.TestContext) error { + return nil +} + +// Run starts the input and blocks as long as websocket connections are alive. It will return on +// context cancellation or type invalidity errors, any other error will be retried. +func (input) Run(env v2.Context, src inputcursor.Source, crsr inputcursor.Cursor, pub inputcursor.Publisher) error { + var cursor map[string]interface{} + if !crsr.IsNew() { // Allow the user to bootstrap the program if needed. + err := crsr.Unpack(&cursor) + if err != nil { + return err + } + } + return input{}.run(env, src.(*source), cursor, pub) +} + +func (i input) run(env v2.Context, src *source, cursor map[string]interface{}, pub inputcursor.Publisher) error { + cfg := src.cfg + i.cfg = cfg + log := env.Logger.With("input_url", cfg.URL) + + metrics := newInputMetrics(env.ID) + defer metrics.Close() + metrics.url.Set(cfg.URL.String()) + metrics.errorsTotal.Set(0) + + ctx := ctxtool.FromCanceller(env.Cancelation) + + patterns, err := regexpsFromConfig(cfg) + if err != nil { + metrics.errorsTotal.Inc() + return err + } + + prg, ast, err := newProgram(ctx, cfg.Program, root, patterns, log) + if err != nil { + metrics.errorsTotal.Inc() + return err + } + var state map[string]interface{} + if cfg.State == nil { + state = make(map[string]interface{}) + } else { + state = cfg.State + } + if cursor != nil { + state["cursor"] = cursor + } + + // websocket client + headers := formHeader(cfg) + url := cfg.URL.String() + c, resp, err := websocket.DefaultDialer.DialContext(ctx, url, headers) + if resp != nil && resp.Body != nil { + log.Debugw("websocket connection response", "body", resp.Body) + resp.Body.Close() + } + if err != nil { + metrics.errorsTotal.Inc() + log.Errorw("failed to establish websocket connection", "error", err) + return err + } + defer c.Close() + + done := make(chan error) + + go func() { + defer close(done) + for { + _, message, err := c.ReadMessage() + if err != nil { + metrics.errorsTotal.Inc() + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + log.Errorw("websocket connection closed", "error", err) + } else { + log.Errorw("failed to read websocket data", "error", err) + } + done <- err + return + } + metrics.receivedBytesTotal.Add(uint64(len(message))) + state["response"] = message + log.Debugw("received websocket message", logp.Namespace("websocket"), string(message)) + err = i.processAndPublishData(ctx, metrics, prg, ast, state, cursor, pub, log) + if err != nil { + metrics.errorsTotal.Inc() + log.Errorw("failed to process and publish data", "error", err) + done <- err + return + } + } + }() + + // blocks until done is closed, context is cancelled or an error is received + select { + case err := <-done: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// processAndPublishData processes the data in state, updates the cursor and publishes it to the publisher. +// the CEL program here only executes a single time, since the websocket connection is persistent and events are received and processed in real time. +func (i *input) processAndPublishData(ctx context.Context, metrics *inputMetrics, prg cel.Program, ast *cel.Ast, + state map[string]interface{}, cursor map[string]interface{}, pub inputcursor.Publisher, log *logp.Logger) error { + goodCursor := cursor + log.Debugw("cel engine state before eval", logp.Namespace("websocket"), "state", redactor{state: state, cfg: i.cfg.Redact}) + start := i.now().In(time.UTC) + state, err := evalWith(ctx, prg, ast, state, start) + log.Debugw("cel engine state after eval", logp.Namespace("websocket"), "state", redactor{state: state, cfg: i.cfg.Redact}) + if err != nil { + metrics.celEvalErrors.Add(1) + switch { + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return err + default: + metrics.errorsTotal.Inc() + } + log.Errorw("failed evaluation", "error", err) + } + metrics.celProcessingTime.Update(time.Since(start).Nanoseconds()) + + e, ok := state["events"] + if !ok { + log.Errorw("unexpected missing events from evaluation") + } + var events []interface{} + switch e := e.(type) { + case []interface{}: + if len(e) == 0 { + return nil + } + events = e + case map[string]interface{}: + if e == nil { + return nil + } + log.Debugw("single event object returned by evaluation", "event", e) + events = []interface{}{e} + default: + return fmt.Errorf("unexpected type returned for evaluation events: %T", e) + } + + // We have a non-empty batch of events to process. + metrics.batchesReceived.Add(1) + metrics.eventsReceived.Add(uint64(len(events))) + + // Drop events from state. If we fail during the publication, + // we will reprocess these events. + delete(state, "events") + + // Get cursors if they exist. + var ( + cursors []interface{} + singleCursor bool + ) + if c, ok := state["cursor"]; ok { + cursors, ok = c.([]interface{}) + if ok { + if len(cursors) != len(events) { + log.Errorw("unexpected cursor list length", "cursors", len(cursors), "events", len(events)) + // But try to continue. + if len(cursors) < len(events) { + cursors = nil + } + } + } else { + cursors = []interface{}{c} + singleCursor = true + } + } + // Drop old cursor from state. This will be replaced with + // the current cursor object below; it is an array now. + delete(state, "cursor") + + start = time.Now() + var hadPublicationError bool + for i, e := range events { + event, ok := e.(map[string]interface{}) + if !ok { + return fmt.Errorf("unexpected type returned for evaluation events: %T", e) + } + var pubCursor interface{} + if cursors != nil { + if singleCursor { + // Only set the cursor for publication at the last event + // when a single cursor object has been provided. + if i == len(events)-1 { + goodCursor = cursor + cursor, ok = cursors[0].(map[string]interface{}) + if !ok { + return fmt.Errorf("unexpected type returned for evaluation cursor element: %T", cursors[0]) + } + pubCursor = cursor + } + } else { + goodCursor = cursor + cursor, ok = cursors[i].(map[string]interface{}) + if !ok { + return fmt.Errorf("unexpected type returned for evaluation cursor element: %T", cursors[i]) + } + pubCursor = cursor + } + } + // Publish the event. + err = pub.Publish(beat.Event{ + Timestamp: time.Now(), + Fields: event, + }, pubCursor) + if err != nil { + hadPublicationError = true + metrics.errorsTotal.Inc() + log.Errorw("error publishing event", "error", err) + cursors = nil // We are lost, so retry with this event's cursor, + continue // but continue with the events that we have without + // advancing the cursor. This allows us to potentially publish the + // events we have now, with a fallback to the last guaranteed + // correctly published cursor. + } + if i == 0 { + metrics.batchesPublished.Add(1) + } + metrics.eventsPublished.Add(1) + + err = ctx.Err() + if err != nil { + return err + } + } + // calculate batch processing time + metrics.batchProcessingTime.Update(time.Since(start).Nanoseconds()) + + // Advance the cursor to the final state if there was no error during + // publications. This is needed to transition to the next set of events. + if !hadPublicationError { + goodCursor = cursor + } + + // Replace the last known good cursor. + state["cursor"] = goodCursor + + switch { + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + metrics.errorsTotal.Inc() + log.Infof("input stopped because context was cancelled with: %v", err) + err = nil + } + return err +} + +func evalWith(ctx context.Context, prg cel.Program, ast *cel.Ast, state map[string]interface{}, now time.Time) (map[string]interface{}, error) { + out, _, err := prg.ContextEval(ctx, map[string]interface{}{ + // Replace global program "now" with current time. This is necessary + // as the lib.Time now global is static at program instantiation time + // which will persist over multiple evaluations. The lib.Time behaviour + // is correct for mito where CEL program instances live for only a + // single evaluation. Rather than incurring the cost of creating a new + // cel.Program for each evaluation, shadow lib.Time's now with a new + // value for each eval. We retain the lib.Time now global for + // compatibility between CEL programs developed in mito with programs + // run in the input. + "now": now, + root: state, + }) + if err != nil { + err = lib.DecoratedError{AST: ast, Err: err} + } + if e := ctx.Err(); e != nil { + err = e + } + if err != nil { + state["events"] = errorMessage(fmt.Sprintf("failed eval: %v", err)) + clearWantMore(state) + return state, fmt.Errorf("failed eval: %w", err) + } + + v, err := out.ConvertToNative(reflect.TypeOf((*structpb.Struct)(nil))) + if err != nil { + state["events"] = errorMessage(fmt.Sprintf("failed proto conversion: %v", err)) + clearWantMore(state) + return state, fmt.Errorf("failed proto conversion: %w", err) + } + switch v := v.(type) { + case *structpb.Struct: + return v.AsMap(), nil + default: + // This should never happen. + errMsg := fmt.Sprintf("unexpected native conversion type: %T", v) + state["events"] = errorMessage(errMsg) + clearWantMore(state) + return state, errors.New(errMsg) + } +} + +// now is time.Now with a modifiable time source. +func (i input) now() time.Time { + if i.time == nil { + return time.Now() + } + return i.time() +} + +// clearWantMore sets the state to not request additional work in a periodic evaluation. +// It leaves state intact if there is no "want_more" element, and sets the element to false +// if there is. This is necessary instead of just doing delete(state, "want_more") as +// client CEL code may expect the want_more field to be present. +func clearWantMore(state map[string]interface{}) { + if _, ok := state["want_more"]; ok { + state["want_more"] = false + } +} + +func errorMessage(msg string) map[string]interface{} { + return map[string]interface{}{"error": map[string]interface{}{"message": msg}} +} + +func formHeader(cfg config) map[string][]string { + header := make(map[string][]string) + switch { + case cfg.Auth.CustomAuth != nil: + header[cfg.Auth.CustomAuth.Header] = []string{cfg.Auth.CustomAuth.Value} + case cfg.Auth.BearerToken != "": + header["Authorization"] = []string{"Bearer " + cfg.Auth.BearerToken} + case cfg.Auth.BasicToken != "": + header["Authorization"] = []string{"Basic " + cfg.Auth.BasicToken} + } + return header +} diff --git a/x-pack/filebeat/input/websocket/input_manager.go b/x-pack/filebeat/input/websocket/input_manager.go new file mode 100644 index 00000000000..49fca0b0a82 --- /dev/null +++ b/x-pack/filebeat/input/websocket/input_manager.go @@ -0,0 +1,71 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "github.com/elastic/go-concert/unison" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +// inputManager wraps one stateless input manager +// and one cursor input manager. It will create one or the other +// based on the config that is passed. +type InputManager struct { + cursor *inputcursor.InputManager +} + +var _ v2.InputManager = InputManager{} + +func NewInputManager(log *logp.Logger, store inputcursor.StateStore) InputManager { + return InputManager{ + cursor: &inputcursor.InputManager{ + Logger: log, + StateStore: store, + Type: inputName, + Configure: cursorConfigure, + }, + } +} + +func cursorConfigure(cfg *conf.C) ([]inputcursor.Source, inputcursor.Input, error) { + src := &source{cfg: config{}} + if err := cfg.Unpack(&src.cfg); err != nil { + return nil, nil, err + } + + if src.cfg.Program == "" { + // set default program + src.cfg.Program = ` + bytes(state.response).decode_json().as(inner_body,{ + "events": { + "message": inner_body.encode_json(), + } + }) + ` + } + return []inputcursor.Source{src}, input{}, nil +} + +type source struct{ cfg config } + +func (s *source) Name() string { return s.cfg.URL.String() } + +// Init initializes both wrapped input managers. +func (m InputManager) Init(grp unison.Group, mode v2.Mode) error { + return m.cursor.Init(grp, mode) +} + +// Create creates a cursor input manager. +func (m InputManager) Create(cfg *conf.C) (v2.Input, error) { + config := config{} + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + return m.cursor.Create(cfg) +} diff --git a/x-pack/filebeat/input/websocket/input_test.go b/x-pack/filebeat/input/websocket/input_test.go new file mode 100644 index 00000000000..fc98a2f0b46 --- /dev/null +++ b/x-pack/filebeat/input/websocket/input_test.go @@ -0,0 +1,597 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/beat" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const ( + basicToken = "dXNlcjpwYXNz" + bearerToken = "BXNlcjpwYVVz" + customHeader = "X-Api-Key" + customValue = "my-api-key" +) + +// WebSocketHandler is a type for handling WebSocket messages. +type WebSocketHandler func(*testing.T, *websocket.Conn, []string) + +var inputTests = []struct { + name string + server func(*testing.T, WebSocketHandler, map[string]interface{}, []string) + handler WebSocketHandler + config map[string]interface{} + response []string + time func() time.Time + persistCursor map[string]interface{} + want []map[string]interface{} + wantErr error +}{ + { + name: "single_event", + server: newWebSocketTestServer(httptest.NewServer), + handler: defaultHandler, + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + }, + response: []string{` + { + "pps": { + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071" + }, + "ts": "2017-08-17T14:54:12.949180-07:00", + "data": "2017-08-17T14:54:12.949180-07:00 example sendmail[30641]:v7HLqYbx029423: to=/dev/null, ctladdr= (8/0),delay=00:00:00, xdelay=00:00:00, mailer=*file*, tls_verify=NONE, pri=35342,dsn=2.0.0, stat=Sent", + "sm": { + "tls": { + "verify": "NONE" + }, + "stat": "Sent", + "qid": "v7HLqYbx029423", + "dsn": "2.0.0", + "mailer": "*file*", + "to": [ + "/dev/null" + ], + "ctladdr": " (8/0)", + "delay": "00:00:00", + "xdelay": "00:00:00", + "pri": 35342 + }, + "id": "ZeYGULpZmL5N0151HN1OyA" + }`}, + want: []map[string]interface{}{ + { + "pps": map[string]interface{}{ + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071", + }, + "ts": "2017-08-17T14:54:12.949180-07:00", + "data": "2017-08-17T14:54:12.949180-07:00 example sendmail[30641]:v7HLqYbx029423: to=/dev/null, ctladdr= (8/0),delay=00:00:00, xdelay=00:00:00, mailer=*file*, tls_verify=NONE, pri=35342,dsn=2.0.0, stat=Sent", + "sm": map[string]interface{}{ + "tls": map[string]interface{}{ + "verify": "NONE", + }, + "stat": "Sent", + "qid": "v7HLqYbx029423", + "dsn": "2.0.0", + "mailer": "*file*", + "to": []interface{}{ + "/dev/null", + }, + "ctladdr": " (8/0)", + "delay": "00:00:00", + "xdelay": "00:00:00", + "pri": float64(35342), + }, + "id": "ZeYGULpZmL5N0151HN1OyA", + }, + }, + }, + { + name: "multiple_events", + server: newWebSocketTestServer(httptest.NewServer), + handler: defaultHandler, + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + }, + response: []string{` + { + "pps": { + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071" + }, + "ts": "2017-08-17T14:54:12.949180-07:00", + "data": "2017-08-17T14:54:12.949180-07:00 example sendmail[30641]:v7HLqYbx029423: to=/dev/null, ctladdr= (8/0),delay=00:00:00, xdelay=00:00:00, mailer=*file*, tls_verify=NONE, pri=35342,dsn=2.0.0, stat=Sent", + "sm": { + "tls": { + "verify": "NONE" + }, + "stat": "Sent", + "qid": "v7HLqYbx029423", + "dsn": "2.0.0", + "mailer": "*file*", + "to": [ + "/dev/null" + ], + "ctladdr": " (8/0)", + "delay": "00:00:00", + "xdelay": "00:00:00", + "pri": 35342 + }, + "id": "ZeYGULpZmL5N0151HN1OyA" + }`, + `{ + "pps": { + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071" + }, + "ts": "2017-08-17T14:54:12.949180-07:00", + "data": "2017-08-17T14:54:12.949180-07:00 example sendmail[30641]:v7HLqYbx029423: to=/dev/null, ctladdr= (8/0),delay=00:00:00, xdelay=00:00:00, mailer=*file*, tls_verify=NONE, pri=35342,dsn=2.0.0, stat=Sent", + "sm": { + "tls": { + "verify": "NONE" + }, + "stat": "Sent", + "qid": "v7HLqYbx029423", + "dsn": "2.0.0", + "mailer": "*file*", + "to": [ + "/dev/null" + ], + "ctladdr": " (8/0)", + "delay": "00:00:00", + "xdelay": "00:00:00", + "pri": 35342 + }, + "id": "ZeYGULpZmL5N0151HN1OyX" + }`}, + want: []map[string]interface{}{ + { + "pps": map[string]interface{}{ + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071", + }, + "ts": "2017-08-17T14:54:12.949180-07:00", + "data": "2017-08-17T14:54:12.949180-07:00 example sendmail[30641]:v7HLqYbx029423: to=/dev/null, ctladdr= (8/0),delay=00:00:00, xdelay=00:00:00, mailer=*file*, tls_verify=NONE, pri=35342,dsn=2.0.0, stat=Sent", + "sm": map[string]interface{}{ + "tls": map[string]interface{}{ + "verify": "NONE", + }, + "stat": "Sent", + "qid": "v7HLqYbx029423", + "dsn": "2.0.0", + "mailer": "*file*", + "to": []interface{}{ + "/dev/null", + }, + "ctladdr": " (8/0)", + "delay": "00:00:00", + "xdelay": "00:00:00", + "pri": float64(35342), + }, + "id": "ZeYGULpZmL5N0151HN1OyA", + }, + { + "pps": map[string]interface{}{ + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071", + }, + "ts": "2017-08-17T14:54:12.949180-07:00", + "data": "2017-08-17T14:54:12.949180-07:00 example sendmail[30641]:v7HLqYbx029423: to=/dev/null, ctladdr= (8/0),delay=00:00:00, xdelay=00:00:00, mailer=*file*, tls_verify=NONE, pri=35342,dsn=2.0.0, stat=Sent", + "sm": map[string]interface{}{ + "tls": map[string]interface{}{ + "verify": "NONE", + }, + "stat": "Sent", + "qid": "v7HLqYbx029423", + "dsn": "2.0.0", + "mailer": "*file*", + "to": []interface{}{ + "/dev/null", + }, + "ctladdr": " (8/0)", + "delay": "00:00:00", + "xdelay": "00:00:00", + "pri": float64(35342), + }, + "id": "ZeYGULpZmL5N0151HN1OyX", + }, + }, + }, + { + name: "bad_cursor", + server: newWebSocketTestServer(httptest.NewServer), + handler: defaultHandler, + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + "cursor":["What's next?"], + })`, + }, + response: []string{` + { + "pps": { + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071" + }, + }`}, + wantErr: fmt.Errorf("unexpected type returned for evaluation cursor element: %T", "What's next?"), + }, + { + name: "invalid_url_scheme", + server: invalidWebSocketTestServer(httptest.NewServer), + handler: defaultHandler, + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + }, + wantErr: fmt.Errorf("unsupported scheme: http accessing config"), + }, + { + name: "cursor_condition_check", + server: newWebSocketTestServer(httptest.NewServer), + handler: defaultHandler, + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": has(state.cursor) && inner_body.ts > state.cursor.last_updated ? [inner_body] : [], + })`, + "state": map[string]interface{}{ + "cursor": map[string]int{ + "last_updated": 1502908200, + }, + }, + }, + response: []string{` + { + "pps": { + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071" + }, + "ts": 1502908200 + }`, + `{ + "pps": { + "agent": "example.proofpoint-1.com", + "cid": "mmeng_vxciml" + }, + "ts": 1503081000 + }`, + }, + want: []map[string]interface{}{ + { + "pps": map[string]interface{}{ + "agent": "example.proofpoint-1.com", + "cid": "mmeng_vxciml", + }, + "ts": float64(1503081000), + }, + }, + }, + { + name: "auth_basic_token", + server: webSocketTestServerWithAuth(httptest.NewServer), + handler: defaultHandler, + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + "auth": map[string]interface{}{ + "basic_token": basicToken, + }, + }, + response: []string{` + { + "pps": { + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071" + }, + "ts": 1502908200 + }`, + }, + want: []map[string]interface{}{ + { + "pps": map[string]interface{}{ + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071", + }, + "ts": float64(1502908200), + }, + }, + }, + { + name: "auth_bearer_token", + server: webSocketTestServerWithAuth(httptest.NewServer), + handler: defaultHandler, + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + "auth": map[string]interface{}{ + "bearer_token": bearerToken, + }, + }, + response: []string{` + { + "pps": { + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071" + }, + "ts": 1502908200 + }`, + }, + want: []map[string]interface{}{ + { + "pps": map[string]interface{}{ + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071", + }, + "ts": float64(1502908200), + }, + }, + }, + { + name: "auth_custom", + server: webSocketTestServerWithAuth(httptest.NewServer), + handler: defaultHandler, + config: map[string]interface{}{ + "program": ` + bytes(state.response).decode_json().as(inner_body,{ + "events": [inner_body], + })`, + "auth": map[string]interface{}{ + "custom": map[string]interface{}{ + "header": customHeader, + "value": customValue, + }, + }, + }, + response: []string{` + { + "pps": { + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071" + }, + "ts": 1502908200 + }`, + }, + want: []map[string]interface{}{ + { + "pps": map[string]interface{}{ + "agent": "example.proofpoint.com", + "cid": "mmeng_uivm071", + }, + "ts": float64(1502908200), + }, + }, + }, +} + +func TestInput(t *testing.T) { + // tests will ignore context cancelled errors, since they are expected + ctxCancelledError := fmt.Errorf("context canceled") + logp.TestingSetup() + for _, test := range inputTests { + t.Run(test.name, func(t *testing.T) { + if test.server != nil { + test.server(t, test.handler, test.config, test.response) + } + + cfg := conf.MustNewConfigFrom(test.config) + + conf := config{} + conf.Redact = &redact{} // Make sure we pass the redact requirement. + err := cfg.Unpack(&conf) + if err != nil { + if test.wantErr != nil { + assert.EqualError(t, err, test.wantErr.Error()) + return + } + t.Fatalf("unexpected error unpacking config: %v", err) + } + + name := input{}.Name() + if name != "websocket" { + t.Errorf(`unexpected input name: got:%q want:"websocket"`, name) + } + src := &source{conf} + err = input{}.Test(src, v2.TestContext{}) + if err != nil { + t.Fatalf("unexpected error running test: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Second) + defer cancel() + + v2Ctx := v2.Context{ + Logger: logp.NewLogger("websocket_test"), + ID: "test_id:" + test.name, + Cancelation: ctx, + } + var client publisher + client.done = func() { + if len(client.published) >= len(test.want) { + cancel() + } + } + + err = input{test.time, conf}.run(v2Ctx, src, test.persistCursor, &client) + if (fmt.Sprint(err) != fmt.Sprint(ctxCancelledError)) && (fmt.Sprint(err) != fmt.Sprint(test.wantErr)) { + t.Errorf("unexpected error from running input: got:%v want:%v", err, test.wantErr) + } + if test.wantErr != nil { + return + } + + if len(client.published) < len(test.want) { + t.Errorf("unexpected number of published events: got:%d want at least:%d", len(client.published), len(test.want)) + test.want = test.want[:len(client.published)] + } + client.published = client.published[:len(test.want)] + for i, got := range client.published { + if !reflect.DeepEqual(got.Fields, mapstr.M(test.want[i])) { + t.Errorf("unexpected result for event %d: got:- want:+\n%s", i, cmp.Diff(got.Fields, mapstr.M(test.want[i]))) + } + } + }) + } +} + +var _ inputcursor.Publisher = (*publisher)(nil) + +type publisher struct { + done func() + mu sync.Mutex + published []beat.Event + cursors []map[string]interface{} +} + +func (p *publisher) Publish(e beat.Event, cursor interface{}) error { + p.mu.Lock() + p.published = append(p.published, e) + if cursor != nil { + c, ok := cursor.(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid cursor type for testing: %T", cursor) + } + p.cursors = append(p.cursors, c) + } + p.done() + p.mu.Unlock() + return nil +} + +func newWebSocketTestServer(serve func(http.Handler) *httptest.Server) func(*testing.T, WebSocketHandler, map[string]interface{}, []string) { + return func(t *testing.T, handler WebSocketHandler, config map[string]interface{}, response []string) { + server := serve(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, + } + + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Fatalf("error upgrading connection to WebSocket: %v", err) + return + } + + handler(t, conn, response) + })) + // only set the resource URL if it is not already set + if config["url"] == nil { + config["url"] = "ws" + server.URL[4:] + } + t.Cleanup(server.Close) + } +} + +// invalidWebSocketTestServer returns a function that creates a WebSocket server with an invalid URL scheme. +func invalidWebSocketTestServer(serve func(http.Handler) *httptest.Server) func(*testing.T, WebSocketHandler, map[string]interface{}, []string) { + return func(t *testing.T, handler WebSocketHandler, config map[string]interface{}, response []string) { + server := serve(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, + } + + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Fatalf("error upgrading connection to WebSocket: %v", err) + return + } + + handler(t, conn, response) + })) + config["url"] = server.URL + t.Cleanup(server.Close) + } +} + +// webSocketTestServerWithAuth returns a function that creates a WebSocket server with authentication. This does not however simulate a TLS connection. +func webSocketTestServerWithAuth(serve func(http.Handler) *httptest.Server) func(*testing.T, WebSocketHandler, map[string]interface{}, []string) { + return func(t *testing.T, handler WebSocketHandler, config map[string]interface{}, response []string) { + server := serve(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + // check for auth token + authToken := r.Header.Get("Authorization") + if authToken == "" { + authToken = r.Header.Get(customHeader) + if authToken == "" { + return false + } + } + + switch { + case authToken == "Bearer "+bearerToken: + return true + case authToken == "Basic "+basicToken: + return true + case authToken == customValue: + return true + default: + return false + + } + }, + } + + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Fatalf("error upgrading connection to WebSocket: %v", err) + return + } + + handler(t, conn, response) + })) + // only set the resource URL if it is not already set + if config["url"] == nil { + config["url"] = "ws" + server.URL[4:] + } + t.Cleanup(server.Close) + } +} + +// defaultHandler is a default handler for WebSocket connections. +func defaultHandler(t *testing.T, conn *websocket.Conn, response []string) { + for _, r := range response { + err := conn.WriteMessage(websocket.TextMessage, []byte(r)) + if err != nil { + t.Fatalf("error writing message to WebSocket: %v", err) + } + } +} diff --git a/x-pack/filebeat/input/websocket/metrics.go b/x-pack/filebeat/input/websocket/metrics.go new file mode 100644 index 00000000000..34e6a9620f9 --- /dev/null +++ b/x-pack/filebeat/input/websocket/metrics.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "github.com/rcrowley/go-metrics" + + "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" + "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/elastic-agent-libs/monitoring/adapter" +) + +// inputMetrics handles the input's metric reporting. +type inputMetrics struct { + unregister func() + url *monitoring.String // URL of the input resource + celEvalErrors *monitoring.Uint // number of errors encountered during cel program evaluation + batchesReceived *monitoring.Uint // number of event arrays received + errorsTotal *monitoring.Uint // number of errors encountered + receivedBytesTotal *monitoring.Uint // number of bytes received + eventsReceived *monitoring.Uint // number of events received + batchesPublished *monitoring.Uint // number of event arrays published + eventsPublished *monitoring.Uint // number of events published + celProcessingTime metrics.Sample // histogram of the elapsed successful cel program processing times in nanoseconds + batchProcessingTime metrics.Sample // histogram of the elapsed successful batch processing times in nanoseconds (time of receipt to time of ACK for non-empty batches). +} + +func newInputMetrics(id string) *inputMetrics { + reg, unreg := inputmon.NewInputRegistry(inputName, id, nil) + out := &inputMetrics{ + unregister: unreg, + url: monitoring.NewString(reg, "url"), + celEvalErrors: monitoring.NewUint(reg, "cel_eval_errors"), + batchesReceived: monitoring.NewUint(reg, "batches_received_total"), + errorsTotal: monitoring.NewUint(reg, "errors_total"), + receivedBytesTotal: monitoring.NewUint(reg, "received_bytes_total"), + eventsReceived: monitoring.NewUint(reg, "events_received_total"), + batchesPublished: monitoring.NewUint(reg, "batches_published_total"), + eventsPublished: monitoring.NewUint(reg, "events_published_total"), + celProcessingTime: metrics.NewUniformSample(1024), + batchProcessingTime: metrics.NewUniformSample(1024), + } + _ = adapter.NewGoMetrics(reg, "cel_processing_time", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.celProcessingTime)) + _ = adapter.NewGoMetrics(reg, "batch_processing_time", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.batchProcessingTime)) + + return out +} + +func (m *inputMetrics) Close() { + m.unregister() +} diff --git a/x-pack/filebeat/input/websocket/redact.go b/x-pack/filebeat/input/websocket/redact.go new file mode 100644 index 00000000000..86583f0691c --- /dev/null +++ b/x-pack/filebeat/input/websocket/redact.go @@ -0,0 +1,111 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "strings" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// redactor implements lazy field redaction of sets of a mapstr.M. +type redactor struct { + state mapstr.M + cfg *redact +} + +// String renders the JSON corresponding to r.state after applying redaction +// operations. +func (r redactor) String() string { + if r.cfg == nil || len(r.cfg.Fields) == 0 { + return r.state.String() + } + c := make(mapstr.M, len(r.state)) + cloneMap(c, r.state) + for _, mask := range r.cfg.Fields { + if r.cfg.Delete { + walkMap(c, mask, func(parent mapstr.M, key string) { + delete(parent, key) + }) + continue + } + walkMap(c, mask, func(parent mapstr.M, key string) { + parent[key] = "*" + }) + } + return c.String() +} + +// cloneMap is an enhanced version of mapstr.M.Clone that handles cloning arrays +// within objects. Nested arrays are not handled. +func cloneMap(dst, src mapstr.M) { + for k, v := range src { + switch v := v.(type) { + case mapstr.M: + d := make(mapstr.M, len(v)) + dst[k] = d + cloneMap(d, v) + case map[string]interface{}: + d := make(map[string]interface{}, len(v)) + dst[k] = d + cloneMap(d, v) + case []mapstr.M: + a := make([]mapstr.M, 0, len(v)) + for _, m := range v { + d := make(mapstr.M, len(m)) + cloneMap(d, m) + a = append(a, d) + } + dst[k] = a + case []map[string]interface{}: + a := make([]map[string]interface{}, 0, len(v)) + for _, m := range v { + d := make(map[string]interface{}, len(m)) + cloneMap(d, m) + a = append(a, d) + } + dst[k] = a + default: + dst[k] = v + } + } +} + +// walkMap walks to all ends of the provided path in m and applies fn to the +// final element of each walk. Nested arrays are not handled. +func walkMap(m mapstr.M, path string, fn func(parent mapstr.M, key string)) { + key, rest, more := strings.Cut(path, ".") + v, ok := m[key] + if !ok { + return + } + if !more { + fn(m, key) + return + } + switch v := v.(type) { + case mapstr.M: + walkMap(v, rest, fn) + case map[string]interface{}: + walkMap(v, rest, fn) + case []mapstr.M: + for _, m := range v { + walkMap(m, rest, fn) + } + case []map[string]interface{}: + for _, m := range v { + walkMap(m, rest, fn) + } + case []interface{}: + for _, v := range v { + switch m := v.(type) { + case mapstr.M: + walkMap(m, rest, fn) + case map[string]interface{}: + walkMap(m, rest, fn) + } + } + } +} diff --git a/x-pack/filebeat/input/websocket/redact_test.go b/x-pack/filebeat/input/websocket/redact_test.go new file mode 100644 index 00000000000..c66db60d97b --- /dev/null +++ b/x-pack/filebeat/input/websocket/redact_test.go @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package websocket + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +var redactorTests = []struct { + name string + state mapstr.M + cfg *redact + + wantOrig string + wantRedact string +}{ + { + name: "nil_redact", + state: mapstr.M{ + "auth": mapstr.M{ + "user": "fred", + "pass": "top_secret", + }, + "other": "data", + }, + cfg: nil, + wantOrig: `{"auth":{"pass":"top_secret","user":"fred"},"other":"data"}`, + wantRedact: `{"auth":{"pass":"top_secret","user":"fred"},"other":"data"}`, + }, + { + name: "auth_no_delete", + state: mapstr.M{ + "auth": mapstr.M{ + "user": "fred", + "pass": "top_secret", + }, + "other": "data", + }, + cfg: &redact{ + Fields: []string{"auth"}, + Delete: false, + }, + wantOrig: `{"auth":{"pass":"top_secret","user":"fred"},"other":"data"}`, + wantRedact: `{"auth":"*","other":"data"}`, + }, + { + name: "auth_delete", + state: mapstr.M{ + "auth": mapstr.M{ + "user": "fred", + "pass": "top_secret", + }, + "other": "data", + }, + cfg: &redact{ + Fields: []string{"auth"}, + Delete: true, + }, + wantOrig: `{"auth":{"pass":"top_secret","user":"fred"},"other":"data"}`, + wantRedact: `{"other":"data"}`, + }, + { + name: "pass_no_delete", + state: mapstr.M{ + "auth": mapstr.M{ + "user": "fred", + "pass": "top_secret", + }, + "other": "data", + }, + cfg: &redact{ + Fields: []string{"auth.pass"}, + Delete: false, + }, + wantOrig: `{"auth":{"pass":"top_secret","user":"fred"},"other":"data"}`, + wantRedact: `{"auth":{"pass":"*","user":"fred"},"other":"data"}`, + }, + { + name: "pass_delete", + state: mapstr.M{ + "auth": mapstr.M{ + "user": "fred", + "pass": "top_secret", + }, + "other": "data", + }, + cfg: &redact{ + Fields: []string{"auth.pass"}, + Delete: true, + }, + wantOrig: `{"auth":{"pass":"top_secret","user":"fred"},"other":"data"}`, + wantRedact: `{"auth":{"user":"fred"},"other":"data"}`, + }, + { + name: "multi_cursor_no_delete", + state: mapstr.M{ + "cursor": []mapstr.M{ + {"key": "val_one", "other": "data"}, + {"key": "val_two", "other": "data"}, + }, + "other": "data", + }, + cfg: &redact{ + Fields: []string{"cursor.key"}, + Delete: false, + }, + wantOrig: `{"cursor":[{"key":"val_one","other":"data"},{"key":"val_two","other":"data"}],"other":"data"}`, + wantRedact: `{"cursor":[{"key":"*","other":"data"},{"key":"*","other":"data"}],"other":"data"}`, + }, + { + name: "multi_cursor_delete", + state: mapstr.M{ + "cursor": []mapstr.M{ + {"key": "val_one", "other": "data"}, + {"key": "val_two", "other": "data"}, + }, + "other": "data", + }, + cfg: &redact{ + Fields: []string{"cursor.key"}, + Delete: true, + }, + wantOrig: `{"cursor":[{"key":"val_one","other":"data"},{"key":"val_two","other":"data"}],"other":"data"}`, + wantRedact: `{"cursor":[{"other":"data"},{"other":"data"}],"other":"data"}`, + }, +} + +func TestRedactor(t *testing.T) { + for _, test := range redactorTests { + t.Run(test.name, func(t *testing.T) { + got := fmt.Sprint(redactor{state: test.state, cfg: test.cfg}) + orig := fmt.Sprint(test.state) + if orig != test.wantOrig { + t.Errorf("unexpected original state after redaction:\n--- got\n--- want\n%s", cmp.Diff(orig, test.wantOrig)) + } + if got != test.wantRedact { + t.Errorf("unexpected redaction:\n--- got\n--- want\n%s", cmp.Diff(got, test.wantRedact)) + } + }) + } +} diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml index 3bd5aac30ec..9ad66efcf54 100644 --- a/x-pack/filebeat/module/threatintel/misp/config/config.yml +++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml @@ -32,8 +32,20 @@ request.transforms: value: json - set: target: body.timestamp - value: '[[.cursor.timestamp]]' - default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "UnixDate" ]]' + value: >- + [[- if index .cursor "timestamp" -]] + [[- .cursor.timestamp -]] + [[- else -]] + [[- .last_response.url.params.Get "timestamp" -]] + [[- end -]] + default: '[[ (now (parseDuration "-{{ .first_interval }}")).Unix ]]' +- set: + target: body.order + value: timestamp +- set: + # Ignored by MISP, set as a workaround to make it available in response.pagination. + target: url.params.timestamp + value: '[[.body.timestamp]]' response.split: target: body.response @@ -51,8 +63,15 @@ response.request_body_on_pagination: true response.pagination: - set: target: body.page - value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 1]][[end]]' + # Add 2 because the httpjson page counter is zero-based while the MISP page parameter starts at 1. + value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 2]][[end]]' fail_on_template_error: true +- set: + target: body.timestamp + value: '[[.last_response.url.params.Get "timestamp"]]' +- set: + target: url.params.timestamp + value: '[[.last_response.url.params.Get "timestamp"]]' cursor: timestamp: value: '[[.last_event.Event.timestamp]]' diff --git a/x-pack/filebeat/tests/integration/managerV2_test.go b/x-pack/filebeat/tests/integration/managerV2_test.go index 3332d549fa2..b541b8d5409 100644 --- a/x-pack/filebeat/tests/integration/managerV2_test.go +++ b/x-pack/filebeat/tests/integration/managerV2_test.go @@ -7,21 +7,51 @@ package integration import ( + "bufio" + "crypto/tls" + "crypto/x509" + "encoding/json" "fmt" + "io" + "math" "os" "path/filepath" + "strings" "sync/atomic" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + protobuf "google.golang.org/protobuf/proto" "github.com/elastic/beats/v7/libbeat/tests/integration" + "github.com/elastic/beats/v7/libbeat/version" + "github.com/elastic/beats/v7/testing/certutil" "github.com/elastic/beats/v7/x-pack/libbeat/management" "github.com/elastic/elastic-agent-client/v7/pkg/client/mock" "github.com/elastic/elastic-agent-client/v7/pkg/proto" ) +// Event is the common part of a beats event, the beats and Elastic Agent +// metadata. +type Event struct { + Metadata struct { + Version string `json:"version"` + } `json:"@metadata"` + ElasticAgent struct { + Snapshot bool `json:"snapshot"` + Version string `json:"version"` + Id string `json:"id"` + } `json:"elastic_agent"` + Agent struct { + Version string `json:"version"` + Id string `json:"id"` + } `json:"agent"` +} + // TestInputReloadUnderElasticAgent will start a Filebeat and cause the input // reload issue described on https://github.com/elastic/beats/issues/33653. // In short, a new input for a file needs to be started while there are still @@ -500,6 +530,208 @@ func TestRecoverFromInvalidOutputConfiguration(t *testing.T) { } } +func TestAgentPackageVersionOnStartUpInfo(t *testing.T) { + wantVersion := "8.13.0+build20131123" + + filebeat := integration.NewBeat( + t, + "filebeat", + "../../filebeat.test", + ) + + logFilePath := filepath.Join(filebeat.TempDir(), "logs-to-ingest.log") + generateLogFile(t, logFilePath) + + eventsDir := filepath.Join(filebeat.TempDir(), "ingested-events") + logLevel := proto.UnitLogLevel_INFO + units := []*proto.UnitExpected{ + { + Id: "output-file-unit", + Type: proto.UnitType_OUTPUT, + ConfigStateIdx: 1, + State: proto.State_HEALTHY, + LogLevel: logLevel, + Config: &proto.UnitExpectedConfig{ + Id: "default", + Type: "file", + Name: "events-to-file", + Source: integration.RequireNewStruct(t, + map[string]interface{}{ + "name": "events-to-file", + "type": "file", + "path": eventsDir, + }), + }, + }, + { + Id: "input-unit-1", + Type: proto.UnitType_INPUT, + ConfigStateIdx: 1, + State: proto.State_HEALTHY, + LogLevel: logLevel, + Config: &proto.UnitExpectedConfig{ + Id: "filestream-monitoring-agent", + Type: "filestream", + Name: "filestream-monitoring-agent", + Streams: []*proto.Stream{ + { + Id: "log-input-1", + Source: integration.RequireNewStruct(t, map[string]interface{}{ + "enabled": true, + "type": "log", + "paths": []interface{}{logFilePath}, + }), + }, + }, + }, + }, + } + wantAgentInfo := proto.AgentInfo{ + Id: "agent-id", + Version: wantVersion, + Snapshot: true, + } + + observedCh := make(chan *proto.CheckinObserved, 5) + server := &mock.StubServerV2{ + CheckinV2Impl: func(observed *proto.CheckinObserved) *proto.CheckinExpected { + observedCh <- observed + return &proto.CheckinExpected{ + AgentInfo: &wantAgentInfo, + Units: units, + } + }, + ActionImpl: func(response *proto.ActionResponse) error { return nil }, + } + + rootKey, rootCACert, rootCertPem, err := certutil.NewRootCA() + require.NoError(t, err, "could not generate root CA") + + rootCertPool := x509.NewCertPool() + ok := rootCertPool.AppendCertsFromPEM(rootCertPem) + require.Truef(t, ok, "could not append certs from PEM to cert pool") + + beatPrivKeyPem, beatCertPem, beatTLSCert, err := + certutil.GenerateChildCert("localhost", rootKey, rootCACert) + require.NoError(t, err, "could not generate child TLS certificate") + + getCert := func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + // it's one of the child certificates. As there is only one, return it + return beatTLSCert, nil + } + + creds := credentials.NewTLS(&tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: rootCertPool, + GetCertificate: getCert, + MinVersion: tls.VersionTLS12, + }) + err = server.Start(grpc.Creds(creds)) + require.NoError(t, err, "failed starting GRPC server") + t.Cleanup(server.Stop) + + filebeat.Start("-E", "management.enabled=true") + + startUpInfo := &proto.StartUpInfo{ + Addr: fmt.Sprintf("localhost:%d", server.Port), + ServerName: "localhost", + Token: "token", + CaCert: rootCertPem, + PeerCert: beatCertPem, + PeerKey: beatPrivKeyPem, + Services: []proto.ConnInfoServices{proto.ConnInfoServices_CheckinV2}, + AgentInfo: &wantAgentInfo, + } + writeStartUpInfo(t, filebeat.Stdin(), startUpInfo) + // for some reason the pipe needs to be closed for filebeat to read it. + require.NoError(t, filebeat.Stdin().Close(), "failed closing stdin pipe") + + // get 1st observed + observed := <-observedCh + // drain observedCh so server won't block + go func() { + for { + <-observedCh + } + }() + + msg := strings.Builder{} + require.Eventuallyf(t, func() bool { + msg.Reset() + + _, err = os.Stat(eventsDir) + if err != nil { + fmt.Fprintf(&msg, "could not verify output directory exists: %v", + err) + return false + } + + entries, err := os.ReadDir(eventsDir) + if err != nil { + fmt.Fprintf(&msg, "failed checking output directory for files: %v", + err) + return false + } + + if len(entries) == 0 { + fmt.Fprintf(&msg, "no file found on %s", eventsDir) + return false + } + + for _, e := range entries { + if e.IsDir() { + continue + } + + i, err := e.Info() + if err != nil { + fmt.Fprintf(&msg, "could not read info of %q", e.Name()) + return false + } + if i.Size() == 0 { + fmt.Fprintf(&msg, "file %q was created, but it's still empty", + e.Name()) + return false + } + + // read one line to make sure it isn't a 1/2 written JSON + eventsFile := filepath.Join(eventsDir, e.Name()) + f, err := os.Open(eventsFile) + if err != nil { + fmt.Fprintf(&msg, "could not open file %q", eventsFile) + return false + } + + scanner := bufio.NewScanner(f) + if scanner.Scan() { + var ev Event + err := json.Unmarshal(scanner.Bytes(), &ev) + if err != nil { + fmt.Fprintf(&msg, "failed to read event from file: %v", err) + return false + } + return true + } + } + + return true + }, 30*time.Second, time.Second, "no event was produced: %s", &msg) + + assert.Equal(t, version.Commit(), observed.VersionInfo.BuildHash) + + evs := getEventsFromFileOutput[Event](t, eventsDir, 100) + for _, got := range evs { + assert.Equal(t, wantVersion, got.Metadata.Version) + + assert.Equal(t, wantAgentInfo.Id, got.ElasticAgent.Id) + assert.Equal(t, wantAgentInfo.Version, got.ElasticAgent.Version) + assert.Equal(t, wantAgentInfo.Snapshot, got.ElasticAgent.Snapshot) + + assert.Equal(t, wantAgentInfo.Id, got.Agent.Id) + assert.Equal(t, wantVersion, got.Agent.Version) + } +} + // generateLogFile generates a log file by appending the current // time to it every second. func generateLogFile(t *testing.T, fullPath string) { @@ -543,3 +775,52 @@ func generateLogFile(t *testing.T, fullPath string) { } }() } + +// getEventsFromFileOutput reads all events from all the files on dir. If n > 0, +// then it reads up to n events. It considers all files are ndjson, and it skips +// any directory within dir. +func getEventsFromFileOutput[E any](t *testing.T, dir string, n int) []E { + t.Helper() + + if n < 1 { + n = math.MaxInt + } + + var events []E + entries, err := os.ReadDir(dir) + require.NoError(t, err, "could not read events directory") + for _, e := range entries { + if e.IsDir() { + continue + } + f, err := os.Open(filepath.Join(dir, e.Name())) + require.NoErrorf(t, err, "could not open file %q", e.Name()) + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + var ev E + err := json.Unmarshal(scanner.Bytes(), &ev) + require.NoError(t, err, "failed to read event") + events = append(events, ev) + + if len(events) >= n { + return events + } + } + } + + return events +} + +func writeStartUpInfo(t *testing.T, w io.Writer, info *proto.StartUpInfo) { + t.Helper() + if len(info.Services) == 0 { + info.Services = []proto.ConnInfoServices{proto.ConnInfoServices_CheckinV2} + } + + infoBytes, err := protobuf.Marshal(info) + require.NoError(t, err, "failed to marshal connection information") + + _, err = w.Write(infoBytes) + require.NoError(t, err, "failed to write connection information") +} diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index aec1914698c..b2c85869921 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.6 +FROM golang:1.21.7 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 4e939b686a6..2284fedbcce 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -1071,9 +1071,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 2b2f28382e9..04df2d4dbcd 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1443,9 +1443,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 9d4ecfa7bd0..9947fd0096c 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -27,6 +27,43 @@ stages: branches: true ## for all the branches tags: true ## for all the tags stage: extended + ## For now Windows CI tests for Libbeat are only enabled for ETW + ## It only contains Go tests + windows-2022: + mage: "mage -w reader/etw build goUnitTest" + platforms: ## override default labels in this specific stage. + - "windows-2022" + stage: mandatory + windows-2019: + mage: "mage -w reader/etw build goUnitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" + stage: extended_win + windows-2016: + mage: "mage -w reader/etw build goUnitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + stage: mandatory + windows-2012: + mage: "mage -w reader/etw build goUnitTest" + platforms: ## override default labels in this specific stage. + - "windows-2012-r2" + stage: extended_win + windows-11: + mage: "mage -w reader/etw build goUnitTest" + platforms: ## override default labels in this specific stage. + - "windows-11" + stage: extended_win + windows-10: + mage: "mage -w reader/etw build goUnitTest" + platforms: ## override default labels in this specific stage. + - "windows-10" + stage: extended_win + windows-8: + mage: "mage -w reader/etw build goUnitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + stage: extended_win unitTest: mage: "mage build unitTest" stage: mandatory diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 84e88d10422..f6efde3e2b2 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -10,6 +10,7 @@ import ( "fmt" "net/http" "net/url" + "time" "github.com/aws/aws-sdk-go-v2/service/sts" @@ -44,6 +45,13 @@ type ConfigAWS struct { FIPSEnabled bool `config:"fips_enabled"` TLS *tlscommon.Config `config:"ssl" yaml:"ssl,omitempty" json:"ssl,omitempty"` DefaultRegion string `config:"default_region"` + + // The duration of the role session. Defaults to 15m when not set. + AssumeRoleDuration time.Duration `config:"assume_role.duration"` + + // AssumeRoleExpiryWindow will allow the credentials to trigger refreshing prior to the credentials + // actually expiring. If expiry_window is less than or equal to zero, the setting is ignored. + AssumeRoleExpiryWindow time.Duration `config:"assume_role.expiry_window"` } // InitializeAWSConfig function creates the awssdk.Config object from the provided config @@ -154,8 +162,15 @@ func addAssumeRoleProviderToAwsConfig(config ConfigAWS, awsConfig *awssdk.Config if config.ExternalID != "" { aro.ExternalID = awssdk.String(config.ExternalID) } + if config.AssumeRoleDuration > 0 { + aro.Duration = config.AssumeRoleDuration + } + }) + awsConfig.Credentials = awssdk.NewCredentialsCache(stsCredProvider, func(options *awssdk.CredentialsCacheOptions) { + if config.AssumeRoleExpiryWindow > 0 { + options.ExpiryWindow = config.AssumeRoleExpiryWindow + } }) - awsConfig.Credentials = stsCredProvider } // addStaticCredentialsProviderToAwsConfig adds a static credentials provider to the current AWS config by using the keys stored in Beats config diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index 172142d1aa8..423e241f896 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -15,6 +15,9 @@ To configure AWS credentials, either put the credentials into the {beatname_uc} * *fips_enabled*: Enabling this option instructs {beatname_uc} to use the FIPS endpoint of a service. All services used by {beatname_uc} are FIPS compatible except for `tagging` but only certain regions are FIPS compatible. See https://aws.amazon.com/compliance/fips/ or the appropriate service page, https://docs.aws.amazon.com/general/latest/gr/aws-service-information.html, for a full list of FIPS endpoints and regions. * *ssl*: This specifies SSL/TLS configuration. If the ssl section is missing, the host's CAs are used for HTTPS connections. See <> for more information. * *default_region*: Default region to query if no other region is set. Most AWS services offer a regional endpoint that can be used to make requests. Some services, such as IAM, do not support regions. If a region is not provided by any other way (environment variable, credential or instance profile), the value set here will be used. +* *assume_role.duration*: The duration of the requested assume role session. Defaults to 15m when not set. AWS allows a maximum session duration between 1h and 12h depending on your maximum session duration policies. +* *assume_role.expiry_window*: The expiry_window will allow refreshing the session prior to its expiration. + This is beneficial to prevent expiring tokens from causing requests to fail with an ExpiredTokenException. [float] ==== Supported Formats diff --git a/x-pack/libbeat/management/managerV2.go b/x-pack/libbeat/management/managerV2.go index 235325c0cbf..6152f4f5306 100644 --- a/x-pack/libbeat/management/managerV2.go +++ b/x-pack/libbeat/management/managerV2.go @@ -23,16 +23,14 @@ import ( "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/features" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" + "github.com/elastic/beats/v7/libbeat/version" "github.com/elastic/elastic-agent-client/v7/pkg/client" "github.com/elastic/elastic-agent-client/v7/pkg/proto" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" - - "github.com/elastic/beats/v7/libbeat/common/reload" - lbmanagement "github.com/elastic/beats/v7/libbeat/management" - "github.com/elastic/beats/v7/libbeat/publisher" - "github.com/elastic/beats/v7/libbeat/version" ) // diagnosticHandler is a wrapper type that's a bit of a hack, the compiler won't let us send the raw unit struct, @@ -161,6 +159,13 @@ func NewV2AgentManager(config *conf.C, registry *reload.Registry) (lbmanagement. } } + versionInfo := client.VersionInfo{ + Name: "beat-v2-client", + BuildHash: version.Commit(), + Meta: map[string]string{ + "commit": version.Commit(), + "build_time": version.BuildTime().String(), + }} var agentClient client.V2 var err error if c.InsecureGRPCURLForTesting != "" && c.Enabled { @@ -168,20 +173,11 @@ func NewV2AgentManager(config *conf.C, registry *reload.Registry) (lbmanagement. logger.Info("Using INSECURE GRPC connection, this should be only used for testing!") agentClient = client.NewV2(c.InsecureGRPCURLForTesting, "", // Insecure connection for test, no token needed - client.VersionInfo{ - Name: "beat-v2-client-for-testing", - Version: version.GetDefaultVersion(), - }, client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) + versionInfo, + client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) } else { // Normal Elastic-Agent-Client initialisation - agentClient, _, err = client.NewV2FromReader(os.Stdin, client.VersionInfo{ - Name: "beat-v2-client", - Version: version.GetDefaultVersion(), - Meta: map[string]string{ - "commit": version.Commit(), - "build_time": version.BuildTime().String(), - }, - }) + agentClient, _, err = client.NewV2FromReader(os.Stdin, versionInfo) if err != nil { return nil, fmt.Errorf("error reading control config from agent: %w", err) } @@ -190,7 +186,7 @@ func NewV2AgentManager(config *conf.C, registry *reload.Registry) (lbmanagement. // officially running under the elastic-agent; we set the publisher pipeline // to inform it that we are running under elastic-agent (used to ensure "Publish event: " // debug log messages are only outputted when running in trace mode - publisher.SetUnderAgent(true) + lbmanagement.SetUnderAgent(true) return NewV2AgentManagerWithClient(c, registry, agentClient) } @@ -231,6 +227,14 @@ func NewV2AgentManagerWithClient(config *Config, registry *reload.Registry, agen // Beats central management interface implementation // ================================ +func (cm *BeatV2Manager) AgentInfo() client.AgentInfo { + if cm.client.AgentInfo() == nil { + return client.AgentInfo{} + } + + return *cm.client.AgentInfo() +} + // RegisterDiagnosticHook will register a diagnostic callback function when elastic-agent asks for a diagnostics dump func (cm *BeatV2Manager) RegisterDiagnosticHook(name string, description string, filename string, contentType string, hook client.DiagnosticHook) { cm.client.RegisterDiagnosticHook(name, description, filename, contentType, hook) @@ -606,7 +610,7 @@ func (cm *BeatV2Manager) reload(units map[unitKey]*client.Unit) { // set the new log level (if nothing has changed is a noop) ll, trace := getZapcoreLevel(lowestLevel) logp.SetLevel(ll) - publisher.SetUnderAgentTrace(trace) + lbmanagement.SetUnderAgentTrace(trace) // reload the output configuration restartBeat, err := cm.reloadOutput(outputUnit) diff --git a/x-pack/libbeat/management/managerV2_test.go b/x-pack/libbeat/management/managerV2_test.go index ea67fdd89f4..66ca7f17966 100644 --- a/x-pack/libbeat/management/managerV2_test.go +++ b/x-pack/libbeat/management/managerV2_test.go @@ -204,8 +204,7 @@ func TestManagerV2(t *testing.T) { defer srv.Stop() client := client.NewV2(fmt.Sprintf(":%d", srv.Port), "", client.VersionInfo{ - Name: "program", - Version: "v1.0.0", + Name: "program", Meta: map[string]string{ "key": "value", }, diff --git a/x-pack/libbeat/management/tests/mock_server.go b/x-pack/libbeat/management/tests/mock_server.go index 8671b124233..a90ae633885 100644 --- a/x-pack/libbeat/management/tests/mock_server.go +++ b/x-pack/libbeat/management/tests/mock_server.go @@ -31,7 +31,7 @@ func NewMockServer(t *testing.T, canStop func(string) bool, inputConfig *proto.U unitOutID := mock.NewID() token := mock.NewID() - //var gotConfig bool + // var gotConfig bool var mut sync.Mutex @@ -98,8 +98,7 @@ func NewMockServer(t *testing.T, canStop func(string) bool, inputConfig *proto.U require.NoError(t, err) client := client.NewV2(fmt.Sprintf(":%d", srv.Port), token, client.VersionInfo{ - Name: "program", - Version: "v1.0.0", + Name: "program", Meta: map[string]string{ "key": "value", }, @@ -111,7 +110,7 @@ func NewMockServer(t *testing.T, canStop func(string) bool, inputConfig *proto.U // helper to wrap the CheckinExpected config we need with every refresh of the mock server func sendUnitsWithState(state proto.State, input, output *proto.UnitExpectedConfig, inId, outId string, stateIndex uint64) *proto.CheckinExpected { return &proto.CheckinExpected{ - AgentInfo: &proto.CheckinAgentInfo{ + AgentInfo: &proto.AgentInfo{ Id: "test-agent", Version: "8.4.0", Snapshot: true, diff --git a/x-pack/libbeat/reader/etw/config.go b/x-pack/libbeat/reader/etw/config.go new file mode 100644 index 00000000000..44f9e68ff2d --- /dev/null +++ b/x-pack/libbeat/reader/etw/config.go @@ -0,0 +1,16 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package etw + +type Config struct { + Logfile string // Path to the logfile + ProviderGUID string // GUID of the ETW provider + ProviderName string // Name of the ETW provider + SessionName string // Name for new ETW session + TraceLevel string // Level of tracing (e.g., "verbose") + MatchAnyKeyword uint64 // Filter for any matching keywords (bitmask) + MatchAllKeyword uint64 // Filter for all matching keywords (bitmask) + Session string // Existing session to attach +} diff --git a/x-pack/libbeat/reader/etw/controller.go b/x-pack/libbeat/reader/etw/controller.go new file mode 100644 index 00000000000..f17866440cf --- /dev/null +++ b/x-pack/libbeat/reader/etw/controller.go @@ -0,0 +1,121 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "errors" + "fmt" + "syscall" +) + +// AttachToExistingSession queries the status of an existing ETW session. +// On success, it updates the Session's handler with the queried information. +func (s *Session) AttachToExistingSession() error { + // Convert the session name to UTF16 for Windows API compatibility. + sessionNamePtr, err := syscall.UTF16PtrFromString(s.Name) + if err != nil { + return fmt.Errorf("failed to convert session name: %w", err) + } + + // Query the current state of the ETW session. + err = s.controlTrace(0, sessionNamePtr, s.properties, EVENT_TRACE_CONTROL_QUERY) + switch { + case err == nil: + // Get the session handler from the properties struct. + s.handler = uintptr(s.properties.Wnode.Union1) + + return nil + + // Handle specific errors related to the query operation. + case errors.Is(err, ERROR_BAD_LENGTH): + return fmt.Errorf("bad length when querying handler: %w", err) + case errors.Is(err, ERROR_INVALID_PARAMETER): + return fmt.Errorf("invalid parameters when querying handler: %w", err) + case errors.Is(err, ERROR_WMI_INSTANCE_NOT_FOUND): + return fmt.Errorf("session is not running: %w", err) + default: + return fmt.Errorf("failed to get handler: %w", err) + } +} + +// CreateRealtimeSession initializes and starts a new real-time ETW session. +func (s *Session) CreateRealtimeSession() error { + // Convert the session name to UTF16 format for Windows API compatibility. + sessionPtr, err := syscall.UTF16PtrFromString(s.Name) + if err != nil { + return fmt.Errorf("failed to convert session name: %w", err) + } + + // Start the ETW trace session. + err = s.startTrace(&s.handler, sessionPtr, s.properties) + switch { + case err == nil: + + // Handle specific errors related to starting the trace session. + case errors.Is(err, ERROR_ALREADY_EXISTS): + return fmt.Errorf("session already exists: %w", err) + case errors.Is(err, ERROR_INVALID_PARAMETER): + return fmt.Errorf("invalid parameters when starting session trace: %w", err) + default: + return fmt.Errorf("failed to start trace: %w", err) + } + + // Set additional parameters for trace enabling. + // See https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-enable_trace_parameters#members + params := EnableTraceParameters{ + Version: 2, // ENABLE_TRACE_PARAMETERS_VERSION_2 + } + + // Zero timeout means asynchronous enablement + const timeout = 0 + + // Enable the trace session with extended options. + err = s.enableTrace(s.handler, &s.GUID, EVENT_CONTROL_CODE_ENABLE_PROVIDER, s.traceLevel, s.matchAnyKeyword, s.matchAllKeyword, timeout, ¶ms) + switch { + case err == nil: + return nil + // Handle specific errors related to enabling the trace session. + case errors.Is(err, ERROR_INVALID_PARAMETER): + return fmt.Errorf("invalid parameters when enabling session trace: %w", err) + case errors.Is(err, ERROR_TIMEOUT): + return fmt.Errorf("timeout value expired before the enable callback completed: %w", err) + case errors.Is(err, ERROR_NO_SYSTEM_RESOURCES): + return fmt.Errorf("exceeded the number of trace sessions that can enable the provider: %w", err) + default: + return fmt.Errorf("failed to enable trace: %w", err) + } +} + +// StopSession closes the ETW session and associated handles if they were created. +func (s *Session) StopSession() error { + if !s.Realtime { + return nil + } + + if isValidHandler(s.traceHandler) { + // Attempt to close the trace and handle potential errors. + if err := s.closeTrace(s.traceHandler); err != nil && !errors.Is(err, ERROR_CTX_CLOSE_PENDING) { + return fmt.Errorf("failed to close trace: %w", err) + } + } + + if s.NewSession { + // If we created the session, send a control command to stop it. + return s.controlTrace( + s.handler, + nil, + s.properties, + EVENT_TRACE_CONTROL_STOP, + ) + } + + return nil +} + +func isValidHandler(handler uint64) bool { + return handler != 0 && handler != INVALID_PROCESSTRACE_HANDLE +} diff --git a/x-pack/libbeat/reader/etw/controller_test.go b/x-pack/libbeat/reader/etw/controller_test.go new file mode 100644 index 00000000000..0c663433ad1 --- /dev/null +++ b/x-pack/libbeat/reader/etw/controller_test.go @@ -0,0 +1,190 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" +) + +func TestAttachToExistingSession_Error(t *testing.T) { + // Mock implementation of controlTrace + controlTrace := func(traceHandle uintptr, + instanceName *uint16, + properties *EventTraceProperties, + controlCode uint32) error { + return ERROR_WMI_INSTANCE_NOT_FOUND + } + + // Create a Session instance + session := &Session{ + Name: "TestSession", + properties: &EventTraceProperties{}, + controlTrace: controlTrace, + } + + err := session.AttachToExistingSession() + assert.EqualError(t, err, "session is not running: The instance name passed was not recognized as valid by a WMI data provider.") +} + +func TestAttachToExistingSession_Success(t *testing.T) { + // Mock implementation of controlTrace + controlTrace := func(traceHandle uintptr, + instanceName *uint16, + properties *EventTraceProperties, + controlCode uint32) error { + // Set a mock handler value + properties.Wnode.Union1 = 12345 + return nil + } + + // Create a Session instance with initialized Properties + session := &Session{ + Name: "TestSession", + properties: &EventTraceProperties{}, + controlTrace: controlTrace, + } + + err := session.AttachToExistingSession() + + assert.NoError(t, err) + assert.Equal(t, uintptr(12345), session.handler, "Handler should be set to the mock value") +} + +func TestCreateRealtimeSession_StartTraceError(t *testing.T) { + // Mock implementation of startTrace + startTrace := func(traceHandle *uintptr, + instanceName *uint16, + properties *EventTraceProperties) error { + return ERROR_ALREADY_EXISTS + } + + // Create a Session instance + session := &Session{ + Name: "TestSession", + properties: &EventTraceProperties{}, + startTrace: startTrace, + } + + err := session.CreateRealtimeSession() + assert.EqualError(t, err, "session already exists: Cannot create a file when that file already exists.") +} + +func TestCreateRealtimeSession_EnableTraceError(t *testing.T) { + // Mock implementations + startTrace := func(traceHandle *uintptr, + instanceName *uint16, + properties *EventTraceProperties) error { + *traceHandle = 12345 // Mock handler value + return nil + } + + enableTrace := func(traceHandle uintptr, + providerId *windows.GUID, + isEnabled uint32, + level uint8, + matchAnyKeyword uint64, + matchAllKeyword uint64, + enableProperty uint32, + enableParameters *EnableTraceParameters) error { + return ERROR_INVALID_PARAMETER + } + + // Create a Session instance + session := &Session{ + Name: "TestSession", + properties: &EventTraceProperties{}, + startTrace: startTrace, + enableTrace: enableTrace, + } + + err := session.CreateRealtimeSession() + assert.EqualError(t, err, "invalid parameters when enabling session trace: The parameter is incorrect.") +} + +func TestCreateRealtimeSession_Success(t *testing.T) { + // Mock implementations + startTrace := func(traceHandle *uintptr, + instanceName *uint16, + properties *EventTraceProperties) error { + *traceHandle = 12345 // Mock handler value + return nil + } + + enableTrace := func(traceHandle uintptr, + providerId *windows.GUID, + isEnabled uint32, + level uint8, + matchAnyKeyword uint64, + matchAllKeyword uint64, + enableProperty uint32, + enableParameters *EnableTraceParameters) error { + return nil + } + + // Create a Session instance + session := &Session{ + Name: "TestSession", + properties: &EventTraceProperties{}, + startTrace: startTrace, + enableTrace: enableTrace, + } + + err := session.CreateRealtimeSession() + + assert.NoError(t, err) + assert.Equal(t, uintptr(12345), session.handler, "Handler should be set to the mock value") +} + +func TestStopSession_Error(t *testing.T) { + // Mock implementation of closeTrace + closeTrace := func(traceHandle uint64) error { + return ERROR_INVALID_PARAMETER + } + + // Create a Session instance + session := &Session{ + Realtime: true, + NewSession: true, + traceHandler: 12345, // Example handler value + properties: &EventTraceProperties{}, + closeTrace: closeTrace, + } + + err := session.StopSession() + assert.EqualError(t, err, "failed to close trace: The parameter is incorrect.") +} + +func TestStopSession_Success(t *testing.T) { + // Mock implementations + closeTrace := func(traceHandle uint64) error { + return nil + } + + controlTrace := func(traceHandle uintptr, + instanceName *uint16, + properties *EventTraceProperties, + controlCode uint32) error { + // Set a mock handler value + return nil + } + + // Create a Session instance + session := &Session{ + Realtime: true, + NewSession: true, + traceHandler: 12345, // Example handler value + properties: &EventTraceProperties{}, + closeTrace: closeTrace, + controlTrace: controlTrace, + } + + err := session.StopSession() + assert.NoError(t, err) +} diff --git a/x-pack/libbeat/reader/etw/event.go b/x-pack/libbeat/reader/etw/event.go new file mode 100644 index 00000000000..34faa8d21cb --- /dev/null +++ b/x-pack/libbeat/reader/etw/event.go @@ -0,0 +1,340 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "errors" + "fmt" + "unsafe" + + "golang.org/x/sys/windows" +) + +// propertyParser is used for parsing properties from raw EVENT_RECORD structures. +type propertyParser struct { + r *EventRecord + info *TraceEventInfo + data []byte + ptrSize uint32 +} + +// GetEventProperties extracts and returns properties from an ETW event record. +func GetEventProperties(r *EventRecord) (map[string]interface{}, error) { + // Handle the case where the event only contains a string. + if r.EventHeader.Flags == EVENT_HEADER_FLAG_STRING_ONLY { + userDataPtr := (*uint16)(unsafe.Pointer(r.UserData)) + return map[string]interface{}{ + "_": utf16AtOffsetToString(uintptr(unsafe.Pointer(userDataPtr)), 0), // Convert the user data from UTF16 to string. + }, nil + } + + // Initialize a new property parser for the event record. + p, err := newPropertyParser(r) + if err != nil { + return nil, fmt.Errorf("failed to parse event properties: %w", err) + } + + // Iterate through each property of the event and format it + properties := make(map[string]interface{}, int(p.info.TopLevelPropertyCount)) + for i := 0; i < int(p.info.TopLevelPropertyCount); i++ { + name := p.getPropertyName(i) + value, err := p.getPropertyValue(i) + if err != nil { + return nil, fmt.Errorf("failed to parse %q value: %w", name, err) + } + properties[name] = value + } + + return properties, nil +} + +// newPropertyParser initializes a new property parser for a given event record. +func newPropertyParser(r *EventRecord) (*propertyParser, error) { + info, err := getEventInformation(r) + if err != nil { + return nil, fmt.Errorf("failed to get event information: %w", err) + } + ptrSize := r.pointerSize() + // Return a new propertyParser instance initialized with event record data and metadata. + return &propertyParser{ + r: r, + info: info, + ptrSize: ptrSize, + data: unsafe.Slice((*uint8)(unsafe.Pointer(r.UserData)), r.UserDataLength), + }, nil +} + +// getEventPropertyInfoAtIndex looks for the EventPropertyInfo object at a specified index. +func (info *TraceEventInfo) getEventPropertyInfoAtIndex(i uint32) *EventPropertyInfo { + if i < info.PropertyCount { + // Calculate the address of the first element in EventPropertyInfoArray. + eventPropertyInfoPtr := uintptr(unsafe.Pointer(&info.EventPropertyInfoArray[0])) + // Adjust the pointer to point to the i-th EventPropertyInfo element. + eventPropertyInfoPtr += uintptr(i) * unsafe.Sizeof(EventPropertyInfo{}) + + return ((*EventPropertyInfo)(unsafe.Pointer(eventPropertyInfoPtr))) + } + return nil +} + +// getEventInformation retrieves detailed metadata about an event record. +func getEventInformation(r *EventRecord) (info *TraceEventInfo, err error) { + // Initially call TdhGetEventInformation to get the required buffer size. + var bufSize uint32 + if err = _TdhGetEventInformation(r, 0, nil, nil, &bufSize); errors.Is(err, ERROR_INSUFFICIENT_BUFFER) { + // Allocate enough memory for TRACE_EVENT_INFO based on the required size. + buff := make([]byte, bufSize) + info = ((*TraceEventInfo)(unsafe.Pointer(&buff[0]))) + // Retrieve the event information into the allocated buffer. + err = _TdhGetEventInformation(r, 0, nil, info, &bufSize) + } + + // Check for errors in retrieving the event information. + if err != nil { + return nil, fmt.Errorf("TdhGetEventInformation failed: %w", err) + } + + return info, nil +} + +// getPropertyName retrieves the name of the i-th event property in the event record. +func (p *propertyParser) getPropertyName(i int) string { + // Convert the UTF16 property name to a Go string. + namePtr := readPropertyName(p, i) + return windows.UTF16PtrToString((*uint16)(namePtr)) +} + +// readPropertyName gets the pointer to the property name in the event information structure. +func readPropertyName(p *propertyParser, i int) unsafe.Pointer { + // Calculate the pointer to the property name using its offset in the event property array. + return unsafe.Add(unsafe.Pointer(p.info), p.info.getEventPropertyInfoAtIndex(uint32(i)).NameOffset) +} + +// getPropertyValue retrieves the value of a specified event property. +func (p *propertyParser) getPropertyValue(i int) (interface{}, error) { + propertyInfo := p.info.getEventPropertyInfoAtIndex(uint32(i)) + + // Determine the size of the property array. + arraySize, err := p.getArraySize(*propertyInfo) + if err != nil { + return nil, fmt.Errorf("failed to get array size: %w", err) + } + + // Initialize a slice to hold the property values. + result := make([]interface{}, arraySize) + for j := 0; j < int(arraySize); j++ { + var ( + value interface{} + err error + ) + // Parse the property value based on its type (simple or structured). + if (propertyInfo.Flags & PropertyStruct) == PropertyStruct { + value, err = p.parseStruct(*propertyInfo) + } else { + value, err = p.parseSimpleType(*propertyInfo) + } + if err != nil { + return nil, err + } + result[j] = value + } + + // Return the entire result set or the single value, based on the property count. + if ((propertyInfo.Flags & PropertyParamCount) == PropertyParamCount) || + (propertyInfo.count() > 1) { + return result, nil + } + return result[0], nil +} + +// getArraySize calculates the size of an array property within an event. +func (p *propertyParser) getArraySize(propertyInfo EventPropertyInfo) (uint32, error) { + // Check if the property's count is specified by another property. + if (propertyInfo.Flags & PropertyParamCount) == PropertyParamCount { + var dataDescriptor PropertyDataDescriptor + // Locate the property containing the array size using the countPropertyIndex. + dataDescriptor.PropertyName = readPropertyName(p, int(propertyInfo.count())) + dataDescriptor.ArrayIndex = 0xFFFFFFFF + // Retrieve the length of the array from the specified property. + return getLengthFromProperty(p.r, &dataDescriptor) + } else { + // If the array size is directly specified, return it. + return uint32(propertyInfo.count()), nil + } +} + +// getLengthFromProperty retrieves the length of a property from an event record. +func getLengthFromProperty(r *EventRecord, dataDescriptor *PropertyDataDescriptor) (uint32, error) { + var length uint32 + // Call TdhGetProperty to get the length of the property specified by the dataDescriptor. + err := _TdhGetProperty( + r, + 0, + nil, + 1, + dataDescriptor, + uint32(unsafe.Sizeof(length)), + (*byte)(unsafe.Pointer(&length)), + ) + if err != nil { + return 0, err + } + return length, nil +} + +// parseStruct extracts and returns the fields from an embedded structure within a property. +func (p *propertyParser) parseStruct(propertyInfo EventPropertyInfo) (map[string]interface{}, error) { + // Determine the start and end indexes of the structure members within the property info. + startIndex := propertyInfo.structStartIndex() + lastIndex := startIndex + propertyInfo.numOfStructMembers() + + // Initialize a map to hold the structure's fields. + structure := make(map[string]interface{}, (lastIndex - startIndex)) + // Iterate through each member of the structure. + for j := startIndex; j < lastIndex; j++ { + name := p.getPropertyName(int(j)) + value, err := p.getPropertyValue(int(j)) + if err != nil { + return nil, fmt.Errorf("failed parse field '%s' of complex property type: %w", name, err) + } + structure[name] = value // Add the field to the structure map. + } + + return structure, nil +} + +// parseSimpleType parses a simple property type using TdhFormatProperty. +func (p *propertyParser) parseSimpleType(propertyInfo EventPropertyInfo) (string, error) { + var mapInfo *EventMapInfo + if propertyInfo.mapNameOffset() > 0 { + // If failed retrieving the map information, returns on error + var err error + mapInfo, err = p.getMapInfo(propertyInfo) + if err != nil { + return "", fmt.Errorf("failed to get map information due to: %w", err) + } + } + + // Get the length of the property. + propertyLength, err := p.getPropertyLength(propertyInfo) + if err != nil { + return "", fmt.Errorf("failed to get property length due to: %w", err) + } + + var userDataConsumed uint16 + + // Set a default buffer size for formatted data. + formattedDataSize := uint32(DEFAULT_PROPERTY_BUFFER_SIZE) + formattedData := make([]byte, int(formattedDataSize)) + + // Retry loop to handle buffer size adjustments. +retryLoop: + for { + var dataPtr *uint8 + if len(p.data) > 0 { + dataPtr = &p.data[0] + } + err := _TdhFormatProperty( + p.info, + mapInfo, + p.ptrSize, + propertyInfo.inType(), + propertyInfo.outType(), + uint16(propertyLength), + uint16(len(p.data)), + dataPtr, + &formattedDataSize, + &formattedData[0], + &userDataConsumed, + ) + + switch { + case err == nil: + // If formatting is successful, break out of the loop. + break retryLoop + case errors.Is(err, ERROR_INSUFFICIENT_BUFFER): + // Increase the buffer size if it's insufficient. + formattedData = make([]byte, formattedDataSize) + continue + case errors.Is(err, ERROR_EVT_INVALID_EVENT_DATA): + // Handle invalid event data error. + // Discarding MapInfo allows us to access + // at least the non-interpreted data. + if mapInfo != nil { + mapInfo = nil + continue + } + return "", fmt.Errorf("TdhFormatProperty failed: %w", err) // Handle unknown error + default: + return "", fmt.Errorf("TdhFormatProperty failed: %w", err) + } + } + // Update the data slice to account for consumed data. + p.data = p.data[userDataConsumed:] + + // Convert the formatted data to string and return. + return windows.UTF16PtrToString((*uint16)(unsafe.Pointer(&formattedData[0]))), nil +} + +// getMapInfo retrieves mapping information for a given property. +func (p *propertyParser) getMapInfo(propertyInfo EventPropertyInfo) (*EventMapInfo, error) { + var mapSize uint32 + // Get the name of the map from the property info. + mapName := (*uint16)(unsafe.Add(unsafe.Pointer(p.info), propertyInfo.mapNameOffset())) + + // First call to get the required size of the map info. + err := _TdhGetEventMapInformation(p.r, mapName, nil, &mapSize) + switch { + case errors.Is(err, ERROR_NOT_FOUND): + // No mapping information available. This is not an error. + return nil, nil + case errors.Is(err, ERROR_INSUFFICIENT_BUFFER): + // Resize the buffer and try again. + default: + return nil, fmt.Errorf("TdhGetEventMapInformation failed to get size: %w", err) + } + + // Allocate buffer and retrieve the actual map information. + buff := make([]byte, int(mapSize)) + mapInfo := ((*EventMapInfo)(unsafe.Pointer(&buff[0]))) + err = _TdhGetEventMapInformation(p.r, mapName, mapInfo, &mapSize) + if err != nil { + return nil, fmt.Errorf("TdhGetEventMapInformation failed: %w", err) + } + + if mapInfo.EntryCount == 0 { + return nil, nil // No entries in the map. + } + + return mapInfo, nil +} + +// getPropertyLength returns the length of a specific property within TraceEventInfo. +func (p *propertyParser) getPropertyLength(propertyInfo EventPropertyInfo) (uint32, error) { + // Check if the length of the property is defined by another property. + if (propertyInfo.Flags & PropertyParamLength) == PropertyParamLength { + var dataDescriptor PropertyDataDescriptor + // Read the property name that contains the length information. + dataDescriptor.PropertyName = readPropertyName(p, int(propertyInfo.length())) + dataDescriptor.ArrayIndex = 0xFFFFFFFF + // Retrieve the length from the specified property. + return getLengthFromProperty(p.r, &dataDescriptor) + } + + inType := propertyInfo.inType() + outType := propertyInfo.outType() + // Special handling for properties representing IPv6 addresses. + // https://docs.microsoft.com/en-us/windows/win32/api/tdh/nf-tdh-tdhformatproperty#remarks + if TdhIntypeBinary == inType && TdhOuttypeIpv6 == outType { + // Return the fixed size of an IPv6 address. + return 16, nil + } + + // Default case: return the length as defined in the property info. + // Note: A length of 0 can indicate a variable-length field (e.g., structure, string). + return uint32(propertyInfo.length()), nil +} diff --git a/x-pack/libbeat/reader/etw/provider.go b/x-pack/libbeat/reader/etw/provider.go new file mode 100644 index 00000000000..63042d0f772 --- /dev/null +++ b/x-pack/libbeat/reader/etw/provider.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "errors" + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +// utf16AtOffsetToString converts a UTF-16 encoded string +// at a specific offset in a struct to a Go string. +func utf16AtOffsetToString(pstruct uintptr, offset uintptr) string { + // Initialize a slice to store UTF-16 characters. + out := make([]uint16, 0, 64) + + // Start reading at the given offset. + wc := (*uint16)(unsafe.Pointer(pstruct + offset)) + + // Iterate over the UTF-16 characters until a null terminator is encountered. + for i := uintptr(2); *wc != 0; i += 2 { + out = append(out, *wc) + wc = (*uint16)(unsafe.Pointer(pstruct + offset + i)) + } + + // Convert the UTF-16 slice to a Go string and return. + return syscall.UTF16ToString(out) +} + +// guidFromProviderName searches for a provider by name and returns its GUID. +func guidFromProviderName(providerName string) (windows.GUID, error) { + // Returns if the provider name is empty. + if providerName == "" { + return windows.GUID{}, fmt.Errorf("empty provider name") + } + + var buf *ProviderEnumerationInfo + size := uint32(1) + + // Attempt to retrieve provider information with a buffer that increases in size until it's sufficient. + for { + tmp := make([]byte, size) + buf = (*ProviderEnumerationInfo)(unsafe.Pointer(&tmp[0])) + if err := enumerateProvidersFunc(buf, &size); !errors.Is(err, ERROR_INSUFFICIENT_BUFFER) { + break + } + } + + if buf.NumberOfProviders == 0 { + return windows.GUID{}, fmt.Errorf("no providers found") + } + + // Iterate through the list of providers to find a match by name. + startProvEnumInfo := uintptr(unsafe.Pointer(buf)) + it := uintptr(unsafe.Pointer(&buf.TraceProviderInfoArray[0])) + for i := uintptr(0); i < uintptr(buf.NumberOfProviders); i++ { + pInfo := (*TraceProviderInfo)(unsafe.Pointer(it + i*unsafe.Sizeof(buf.TraceProviderInfoArray[0]))) + name := utf16AtOffsetToString(startProvEnumInfo, uintptr(pInfo.ProviderNameOffset)) + + // If a match is found, return the corresponding GUID. + if name == providerName { + return pInfo.ProviderGuid, nil + } + } + + // No matching provider is found. + return windows.GUID{}, fmt.Errorf("unable to find GUID from provider name") +} diff --git a/x-pack/libbeat/reader/etw/provider_test.go b/x-pack/libbeat/reader/etw/provider_test.go new file mode 100644 index 00000000000..0a10e0b495b --- /dev/null +++ b/x-pack/libbeat/reader/etw/provider_test.go @@ -0,0 +1,178 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "encoding/binary" + "syscall" + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" +) + +func TestUTF16AtOffsetToString(t *testing.T) { + // Create a UTF-16 string + sampleText := "This is a string test!" + utf16Str, _ := syscall.UTF16FromString(sampleText) + + // Convert it to uintptr (simulate as if it's part of a larger struct) + ptr := uintptr(unsafe.Pointer(&utf16Str[0])) + + // Test the function + result := utf16AtOffsetToString(ptr, 0) + assert.Equal(t, sampleText, result, "The converted string should match the original") + + // Test with offset (skip the first character) + offset := unsafe.Sizeof(utf16Str[0]) // Size of one UTF-16 character + resultWithOffset := utf16AtOffsetToString(ptr, offset) + assert.Equal(t, sampleText[1:], resultWithOffset, "The converted string with offset should skip the first character") +} + +func TestGUIDFromProviderName_EmptyName(t *testing.T) { + guid, err := guidFromProviderName("") + assert.EqualError(t, err, "empty provider name") + assert.Equal(t, windows.GUID{}, guid, "GUID should be empty for an empty provider name") +} + +func TestGUIDFromProviderName_EmptyProviderList(t *testing.T) { + // Defer restoration of the original function + t.Cleanup(func() { + enumerateProvidersFunc = _TdhEnumerateProviders + }) + + // Define a mock provider name and GUID for testing. + mockProviderName := "NonExistentProvider" + + enumerateProvidersFunc = func(pBuffer *ProviderEnumerationInfo, pBufferSize *uint32) error { + // Check if the buffer size is sufficient + requiredSize := uint32(unsafe.Sizeof(ProviderEnumerationInfo{})) + uint32(unsafe.Sizeof(TraceProviderInfo{}))*0 // As there are no providers + if *pBufferSize < requiredSize { + // Set the size required and return the error + *pBufferSize = requiredSize + return ERROR_INSUFFICIENT_BUFFER + } + + // Empty list of providers + *pBuffer = ProviderEnumerationInfo{ + NumberOfProviders: 0, + TraceProviderInfoArray: [anysizeArray]TraceProviderInfo{}, + } + return nil + } + + guid, err := guidFromProviderName(mockProviderName) + assert.EqualError(t, err, "no providers found") + assert.Equal(t, windows.GUID{}, guid, "GUID should be empty when the provider is not found") +} + +func TestGUIDFromProviderName_GUIDNotFound(t *testing.T) { + // Defer restoration of the original function + t.Cleanup(func() { + enumerateProvidersFunc = _TdhEnumerateProviders + }) + + // Define a mock provider name and GUID for testing. + mockProviderName := "NonExistentProvider" + realProviderName := "ExistentProvider" + mockGUID := windows.GUID{Data1: 1234, Data2: 5678} + + enumerateProvidersFunc = func(pBuffer *ProviderEnumerationInfo, pBufferSize *uint32) error { + // Convert provider name to UTF-16 + utf16ProviderName, _ := syscall.UTF16FromString(realProviderName) + + // Calculate size needed for the provider name string + nameSize := (len(utf16ProviderName) + 1) * 2 // +1 for null-terminator + + requiredSize := uint32(unsafe.Sizeof(ProviderEnumerationInfo{})) + uint32(unsafe.Sizeof(TraceProviderInfo{})) + uint32(nameSize) + if *pBufferSize < requiredSize { + *pBufferSize = requiredSize + return ERROR_INSUFFICIENT_BUFFER + } + + // Calculate the offset for the provider name + // It's placed after ProviderEnumerationInfo and TraceProviderInfo + nameOffset := unsafe.Sizeof(ProviderEnumerationInfo{}) + unsafe.Sizeof(TraceProviderInfo{}) + + // Convert pBuffer to a byte slice starting at the calculated offset for the name + byteBuffer := (*[1 << 30]byte)(unsafe.Pointer(pBuffer))[:] + // Copy the UTF-16 encoded name into the buffer + for i, char := range utf16ProviderName { + binary.LittleEndian.PutUint16(byteBuffer[nameOffset+(uintptr(i)*2):], char) + } + + // Create and populate the ProviderEnumerationInfo struct + *pBuffer = ProviderEnumerationInfo{ + NumberOfProviders: 1, + TraceProviderInfoArray: [anysizeArray]TraceProviderInfo{ + { + ProviderGuid: mockGUID, + ProviderNameOffset: uint32(nameOffset), + }, + }, + } + return nil + } + + guid, err := guidFromProviderName(mockProviderName) + assert.EqualError(t, err, "unable to find GUID from provider name") + assert.Equal(t, windows.GUID{}, guid, "GUID should be empty when the provider is not found") +} + +func TestGUIDFromProviderName_Success(t *testing.T) { + // Defer restoration of the original function + t.Cleanup(func() { + enumerateProvidersFunc = _TdhEnumerateProviders + }) + + // Define a mock provider name and GUID for testing. + mockProviderName := "MockProvider" + mockGUID := windows.GUID{Data1: 1234, Data2: 5678} + + enumerateProvidersFunc = func(pBuffer *ProviderEnumerationInfo, pBufferSize *uint32) error { + // Convert provider name to UTF-16 + utf16ProviderName, _ := syscall.UTF16FromString(mockProviderName) + + // Calculate size needed for the provider name string + nameSize := (len(utf16ProviderName) + 1) * 2 // +1 for null-terminator + + requiredSize := uint32(unsafe.Sizeof(ProviderEnumerationInfo{})) + uint32(unsafe.Sizeof(TraceProviderInfo{})) + uint32(nameSize) + if *pBufferSize < requiredSize { + *pBufferSize = requiredSize + return ERROR_INSUFFICIENT_BUFFER + } + + // Calculate the offset for the provider name + // It's placed after ProviderEnumerationInfo and TraceProviderInfo + nameOffset := unsafe.Sizeof(ProviderEnumerationInfo{}) + unsafe.Sizeof(TraceProviderInfo{}) + + // Convert pBuffer to a byte slice starting at the calculated offset for the name + byteBuffer := (*[1 << 30]byte)(unsafe.Pointer(pBuffer))[:] + // Copy the UTF-16 encoded name into the buffer + for i, char := range utf16ProviderName { + binary.LittleEndian.PutUint16(byteBuffer[nameOffset+(uintptr(i)*2):], char) + } + + // Create and populate the ProviderEnumerationInfo struct + *pBuffer = ProviderEnumerationInfo{ + NumberOfProviders: 1, + TraceProviderInfoArray: [anysizeArray]TraceProviderInfo{ + { + ProviderGuid: mockGUID, + ProviderNameOffset: uint32(nameOffset), + }, + }, + } + return nil + } + + // Run the test + guid, err := guidFromProviderName(mockProviderName) + assert.NoError(t, err) + assert.Equal(t, mockGUID, guid, "GUID should match the mock GUID") +} diff --git a/x-pack/libbeat/reader/etw/session.go b/x-pack/libbeat/reader/etw/session.go new file mode 100644 index 00000000000..9d78d279de2 --- /dev/null +++ b/x-pack/libbeat/reader/etw/session.go @@ -0,0 +1,248 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "errors" + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +// For testing purposes we create a variable to store the function to call +// When running tests, these variables point to a mock function +var ( + guidFromProviderNameFunc = guidFromProviderName + setSessionGUIDFunc = setSessionGUID +) + +type Session struct { + // Name is the identifier for the session. + // It is used to identify the session in logs and also for Windows processes. + Name string + // GUID is the provider GUID to configure the session. + GUID windows.GUID + // properties of the session that are initialized in newSessionProperties() + // See https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-event_trace_properties for more information + properties *EventTraceProperties + // handler of the event tracing session for which the provider is being configured. + // It is obtained from StartTrace when a new trace is started. + // This handler is needed to enable, query or stop the trace. + handler uintptr + // Realtime is a flag to know if the consumer reads from a logfile or real-time session. + Realtime bool // Real-time flag + // NewSession is a flag to indicate whether a new session has been created or attached to an existing one. + NewSession bool + // TraceLevel sets the maximum level of events that we want the provider to write. + traceLevel uint8 + // matchAnyKeyword is a 64-bit bitmask of keywords that determine the categories of events that we want the provider to write. + // The provider writes an event if the event's keyword bits match any of the bits set in this value + // or if the event has no keyword bits set, in addition to meeting the level and matchAllKeyword criteria. + matchAnyKeyword uint64 + // matchAllKeyword is a 64-bit bitmask of keywords that restricts the events that we want the provider to write. + // The provider typically writes an event if the event's keyword bits match all of the bits set in this value + // or if the event has no keyword bits set, in addition to meeting the level and matchAnyKeyword criteria. + matchAllKeyword uint64 + // traceHandler is the trace processing handle. + // It is used to control the trace that receives and processes events. + traceHandler uint64 + // Callback is the pointer to EventRecordCallback which receives and processes event trace events. + Callback func(*EventRecord) uintptr + // BufferCallback is the pointer to BufferCallback which processes retrieved metadata about the ETW buffers (optional). + BufferCallback func(*EventTraceLogfile) uintptr + + // Pointers to functions that make calls to the Windows API. + // In tests, these pointers can be replaced with mock functions to simulate API behavior without making actual calls to the Windows API. + startTrace func(*uintptr, *uint16, *EventTraceProperties) error + controlTrace func(traceHandle uintptr, instanceName *uint16, properties *EventTraceProperties, controlCode uint32) error + enableTrace func(traceHandle uintptr, providerId *windows.GUID, isEnabled uint32, level uint8, matchAnyKeyword uint64, matchAllKeyword uint64, enableProperty uint32, enableParameters *EnableTraceParameters) error + closeTrace func(traceHandle uint64) error + openTrace func(elf *EventTraceLogfile) (uint64, error) + processTrace func(handleArray *uint64, handleCount uint32, startTime *FileTime, endTime *FileTime) error +} + +// setSessionName determines the session name based on the provided configuration. +func setSessionName(conf Config) string { + // Iterate through potential session name values, returning the first non-empty one. + for _, value := range []string{conf.Logfile, conf.Session, conf.SessionName} { + if value != "" { + return value + } + } + + if conf.ProviderName != "" { + return fmt.Sprintf("Elastic-%s", conf.ProviderName) + } + + return fmt.Sprintf("Elastic-%s", conf.ProviderGUID) +} + +// setSessionGUID determines the session GUID based on the provided configuration. +func setSessionGUID(conf Config) (windows.GUID, error) { + var guid windows.GUID + var err error + + // If ProviderGUID is not set in the configuration, attempt to resolve it using the provider name. + if conf.ProviderGUID == "" { + guid, err = guidFromProviderNameFunc(conf.ProviderName) + if err != nil { + return windows.GUID{}, fmt.Errorf("error resolving GUID: %w", err) + } + } else { + // If ProviderGUID is set, parse it into a GUID structure. + guid, err = windows.GUIDFromString(conf.ProviderGUID) + if err != nil { + return windows.GUID{}, fmt.Errorf("error parsing Windows GUID: %w", err) + } + } + + return guid, nil +} + +// getTraceLevel converts a string representation of a trace level +// to its corresponding uint8 constant value +func getTraceLevel(level string) uint8 { + switch level { + case "critical": + return TRACE_LEVEL_CRITICAL + case "error": + return TRACE_LEVEL_ERROR + case "warning": + return TRACE_LEVEL_WARNING + case "information": + return TRACE_LEVEL_INFORMATION + case "verbose": + return TRACE_LEVEL_VERBOSE + default: + return TRACE_LEVEL_INFORMATION + } +} + +// newSessionProperties initializes and returns a pointer to EventTraceProperties +// with the necessary settings for starting an ETW session. +// See https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-event_trace_properties +func newSessionProperties(sessionName string) *EventTraceProperties { + // Calculate buffer size for session properties. + sessionNameSize := (len(sessionName) + 1) * 2 + bufSize := sessionNameSize + int(unsafe.Sizeof(EventTraceProperties{})) + + // Allocate buffer and cast to EventTraceProperties. + propertiesBuf := make([]byte, bufSize) + sessionProperties := (*EventTraceProperties)(unsafe.Pointer(&propertiesBuf[0])) + + // Initialize mandatory fields of the EventTraceProperties struct. + // Filled based on https://learn.microsoft.com/en-us/windows/win32/etw/wnode-header + sessionProperties.Wnode.BufferSize = uint32(bufSize) + sessionProperties.Wnode.Guid = windows.GUID{} // GUID not required for non-private/kernel sessions + // ClientContext is used for timestamp resolution + // Not used unless adding PROCESS_TRACE_MODE_RAW_TIMESTAMP flag to EVENT_TRACE_LOGFILE struct + // See https://learn.microsoft.com/en-us/windows/win32/etw/wnode-header + sessionProperties.Wnode.ClientContext = 1 + sessionProperties.Wnode.Flags = WNODE_FLAG_TRACED_GUID + // Set logging mode to real-time + // See https://learn.microsoft.com/en-us/windows/win32/etw/logging-mode-constants + sessionProperties.LogFileMode = EVENT_TRACE_REAL_TIME_MODE + sessionProperties.LogFileNameOffset = 0 // Can be specified to log to a file as well as to a real-time session + sessionProperties.BufferSize = 64 // Default buffer size, can be configurable + sessionProperties.LoggerNameOffset = uint32(unsafe.Sizeof(EventTraceProperties{})) // Offset to the logger name + + return sessionProperties +} + +// NewSession initializes and returns a new ETW Session based on the provided configuration. +func NewSession(conf Config) (*Session, error) { + session := &Session{} + + // Assign ETW Windows API functions + session.startTrace = _StartTrace + session.controlTrace = _ControlTrace + session.enableTrace = _EnableTraceEx2 + session.openTrace = _OpenTrace + session.processTrace = _ProcessTrace + session.closeTrace = _CloseTrace + + session.Name = setSessionName(conf) + session.Realtime = true + + // If a current session is configured, set up the session properties and return. + if conf.Session != "" { + session.properties = newSessionProperties(session.Name) + return session, nil + } else if conf.Logfile != "" { + // If a logfile is specified, set up for non-realtime session. + session.Realtime = false + return session, nil + } + + session.NewSession = true // Indicate this is a new session + + var err error + session.GUID, err = setSessionGUIDFunc(conf) + if err != nil { + return nil, fmt.Errorf("error when initializing session '%s': %w", session.Name, err) + } + + // Initialize additional session properties. + session.properties = newSessionProperties(session.Name) + session.traceLevel = getTraceLevel(conf.TraceLevel) + session.matchAnyKeyword = conf.MatchAnyKeyword + session.matchAllKeyword = conf.MatchAllKeyword + + return session, nil +} + +// StartConsumer initializes and starts the ETW event tracing session. +func (s *Session) StartConsumer() error { + var elf EventTraceLogfile + var err error + + // Configure EventTraceLogfile based on the session type (realtime or not). + if !s.Realtime { + elf.LogFileMode = PROCESS_TRACE_MODE_EVENT_RECORD + logfilePtr, err := syscall.UTF16PtrFromString(s.Name) + if err != nil { + return fmt.Errorf("failed to convert logfile name: %w", err) + } + elf.LogFileName = logfilePtr + } else { + elf.LogFileMode = PROCESS_TRACE_MODE_EVENT_RECORD | PROCESS_TRACE_MODE_REAL_TIME + sessionPtr, err := syscall.UTF16PtrFromString(s.Name) + if err != nil { + return fmt.Errorf("failed to convert session name: %w", err) + } + elf.LoggerName = sessionPtr + } + + // Set callback and context for the session. + if s.Callback == nil { + return fmt.Errorf("error loading callback") + } + elf.Callback = syscall.NewCallback(s.Callback) + elf.Context = 0 + + // Open an ETW trace processing handle for consuming events + // from an ETW real-time trace session or an ETW log file. + s.traceHandler, err = s.openTrace(&elf) + switch { + case err == nil: + // Handle specific errors for trace opening. + case errors.Is(err, ERROR_BAD_PATHNAME): + return fmt.Errorf("invalid log source when opening trace: %w", err) + case errors.Is(err, ERROR_ACCESS_DENIED): + return fmt.Errorf("access denied when opening trace: %w", err) + default: + return fmt.Errorf("failed to open trace: %w", err) + } + + // Process the trace. This function blocks until processing ends. + if err := s.processTrace(&s.traceHandler, 1, nil, nil); err != nil { + return fmt.Errorf("failed to process trace: %w", err) + } + return nil +} diff --git a/x-pack/libbeat/reader/etw/session_test.go b/x-pack/libbeat/reader/etw/session_test.go new file mode 100644 index 00000000000..f79c9473f3e --- /dev/null +++ b/x-pack/libbeat/reader/etw/session_test.go @@ -0,0 +1,337 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "fmt" + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" +) + +// TestSetSessionName tests the setSessionName function with various configurations. +func TestSetSessionName(t *testing.T) { + testCases := []struct { + name string + config Config + expectedName string + }{ + { + name: "ProviderNameSet", + config: Config{ + ProviderName: "Provider1", + }, + expectedName: "Elastic-Provider1", + }, + { + name: "SessionNameSet", + config: Config{ + SessionName: "Session1", + }, + expectedName: "Session1", + }, + { + name: "LogFileSet", + config: Config{ + Logfile: "LogFile1.etl", + }, + expectedName: "LogFile1.etl", + }, + { + name: "FallbackToProviderGUID", + config: Config{ + ProviderGUID: "12345", + }, + expectedName: "Elastic-12345", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + sessionName := setSessionName(tc.config) + assert.Equal(t, tc.expectedName, sessionName, "The session name should be correctly determined") + }) + } +} + +func mockGUIDFromProviderName(providerName string) (windows.GUID, error) { + // Return a mock GUID regardless of the input + return windows.GUID{Data1: 0x12345678, Data2: 0x1234, Data3: 0x5678, Data4: [8]byte{0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x78}}, nil +} + +func TestSetSessionGUID_ProviderName(t *testing.T) { + // Defer restoration of original function + t.Cleanup(func() { + guidFromProviderNameFunc = guidFromProviderName + }) + + // Replace with mock function + guidFromProviderNameFunc = mockGUIDFromProviderName + + conf := Config{ProviderName: "Provider1"} + expectedGUID := windows.GUID{Data1: 0x12345678, Data2: 0x1234, Data3: 0x5678, Data4: [8]byte{0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x78}} + + guid, err := setSessionGUID(conf) + assert.NoError(t, err) + assert.Equal(t, expectedGUID, guid, "The GUID should match the mock GUID") +} + +func TestSetSessionGUID_ProviderGUID(t *testing.T) { + // Example GUID string + guidString := "{12345678-1234-5678-1234-567812345678}" + + // Configuration with a set ProviderGUID + conf := Config{ProviderGUID: guidString} + + // Expected GUID based on the GUID string + expectedGUID := windows.GUID{Data1: 0x12345678, Data2: 0x1234, Data3: 0x5678, Data4: [8]byte{0x12, 0x34, 0x56, 0x78, 0x12, 0x34, 0x56, 0x78}} + + guid, err := setSessionGUID(conf) + + assert.NoError(t, err) + assert.Equal(t, expectedGUID, guid, "The GUID should match the expected value") +} + +func TestGetTraceLevel(t *testing.T) { + testCases := []struct { + name string + level string + expectedCode uint8 + }{ + {"CriticalLevel", "critical", TRACE_LEVEL_CRITICAL}, + {"ErrorLevel", "error", TRACE_LEVEL_ERROR}, + {"WarningLevel", "warning", TRACE_LEVEL_WARNING}, + {"InformationLevel", "information", TRACE_LEVEL_INFORMATION}, + {"VerboseLevel", "verbose", TRACE_LEVEL_VERBOSE}, + {"DefaultLevel", "unknown", TRACE_LEVEL_INFORMATION}, // Default case + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := getTraceLevel(tc.level) + assert.Equal(t, tc.expectedCode, result, "Trace level code should match the expected value") + }) + } +} + +func TestNewSessionProperties(t *testing.T) { + testCases := []struct { + name string + sessionName string + expectedSize uint32 + }{ + {"EmptyName", "", 2 + uint32(unsafe.Sizeof(EventTraceProperties{}))}, + {"NormalName", "Session1", 18 + uint32(unsafe.Sizeof(EventTraceProperties{}))}, + // Additional test cases can be added here + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + props := newSessionProperties(tc.sessionName) + + assert.Equal(t, tc.expectedSize, props.Wnode.BufferSize, "BufferSize should match expected value") + assert.Equal(t, windows.GUID{}, props.Wnode.Guid, "GUID should be empty") + assert.Equal(t, uint32(1), props.Wnode.ClientContext, "ClientContext should be 1") + assert.Equal(t, uint32(WNODE_FLAG_TRACED_GUID), props.Wnode.Flags, "Flags should match WNODE_FLAG_TRACED_GUID") + assert.Equal(t, uint32(EVENT_TRACE_REAL_TIME_MODE), props.LogFileMode, "LogFileMode should be set to real-time") + assert.Equal(t, uint32(0), props.LogFileNameOffset, "LogFileNameOffset should be 0") + assert.Equal(t, uint32(64), props.BufferSize, "BufferSize should be 64") + assert.Equal(t, uint32(unsafe.Sizeof(EventTraceProperties{})), props.LoggerNameOffset, "LoggerNameOffset should be the size of EventTraceProperties") + }) + } +} + +func TestNewSession_ProviderName(t *testing.T) { + // Defer restoration of original function + t.Cleanup(func() { + setSessionGUIDFunc = setSessionGUID + }) + + // Override setSessionGUIDFunc with mock + setSessionGUIDFunc = func(conf Config) (windows.GUID, error) { + return windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x5678, + Data4: [8]byte{0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x78}, + }, nil + } + + expectedGUID := windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x5678, + Data4: [8]byte{0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x78}, + } + + conf := Config{ + ProviderName: "Provider1", + SessionName: "Session1", + TraceLevel: "warning", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + } + session, err := NewSession(conf) + + assert.NoError(t, err) + assert.Equal(t, "Session1", session.Name, "SessionName should match expected value") + assert.Equal(t, expectedGUID, session.GUID, "The GUID in the session should match the expected GUID") + assert.Equal(t, uint8(3), session.traceLevel, "TraceLevel should be 3 (warning)") + assert.Equal(t, true, session.NewSession) + assert.Equal(t, true, session.Realtime) + assert.NotNil(t, session.properties) +} + +func TestNewSession_GUIDError(t *testing.T) { + // Defer restoration of original function + t.Cleanup(func() { + setSessionGUIDFunc = setSessionGUID + }) + + // Override setSessionGUIDFunc with mock + setSessionGUIDFunc = func(conf Config) (windows.GUID, error) { + // Return an empty GUID and an error + return windows.GUID{}, fmt.Errorf("mock error") + } + + conf := Config{ + ProviderName: "Provider1", + SessionName: "Session1", + TraceLevel: "warning", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + } + session, err := NewSession(conf) + + assert.EqualError(t, err, "error when initializing session 'Session1': mock error") + assert.Nil(t, session) + +} + +func TestNewSession_AttachSession(t *testing.T) { + // Test case + conf := Config{ + Session: "Session1", + SessionName: "TestSession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + } + session, err := NewSession(conf) + + assert.NoError(t, err) + assert.Equal(t, "Session1", session.Name, "SessionName should match expected value") + assert.Equal(t, false, session.NewSession) + assert.Equal(t, true, session.Realtime) + assert.NotNil(t, session.properties) +} + +func TestNewSession_Logfile(t *testing.T) { + // Test case + conf := Config{ + Logfile: "LogFile1.etl", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + } + session, err := NewSession(conf) + + assert.NoError(t, err) + assert.Equal(t, "LogFile1.etl", session.Name, "SessionName should match expected value") + assert.Equal(t, false, session.NewSession) + assert.Equal(t, false, session.Realtime) + assert.Nil(t, session.properties) +} + +func TestStartConsumer_CallbackNull(t *testing.T) { + // Create a Session instance + session := &Session{ + Name: "TestSession", + Realtime: false, + BufferCallback: nil, + Callback: nil, + } + + err := session.StartConsumer() + assert.EqualError(t, err, "error loading callback") +} + +func TestStartConsumer_OpenTraceError(t *testing.T) { + // Mock implementation of openTrace + openTrace := func(elf *EventTraceLogfile) (uint64, error) { + return 0, ERROR_ACCESS_DENIED // Mock a valid session handler + } + + // Create a Session instance + session := &Session{ + Name: "TestSession", + Realtime: false, + BufferCallback: nil, + Callback: func(*EventRecord) uintptr { + return 1 + }, + openTrace: openTrace, + } + + err := session.StartConsumer() + assert.EqualError(t, err, "access denied when opening trace: Access is denied.") +} + +func TestStartConsumer_ProcessTraceError(t *testing.T) { + // Mock implementations + openTrace := func(elf *EventTraceLogfile) (uint64, error) { + return 12345, nil // Mock a valid session handler + } + + processTrace := func(handleArray *uint64, handleCount uint32, startTime *FileTime, endTime *FileTime) error { + return ERROR_INVALID_PARAMETER + } + + // Create a Session instance + session := &Session{ + Name: "TestSession", + Realtime: true, + BufferCallback: nil, + Callback: func(*EventRecord) uintptr { + return 1 + }, + openTrace: openTrace, + processTrace: processTrace, + } + + err := session.StartConsumer() + assert.EqualError(t, err, "failed to process trace: The parameter is incorrect.") +} + +func TestStartConsumer_Success(t *testing.T) { + // Mock implementations + openTrace := func(elf *EventTraceLogfile) (uint64, error) { + return 12345, nil // Mock a valid session handler + } + + processTrace := func(handleArray *uint64, handleCount uint32, startTime *FileTime, endTime *FileTime) error { + return nil + } + + // Create a Session instance + session := &Session{ + Name: "TestSession", + Realtime: true, + BufferCallback: nil, + Callback: func(*EventRecord) uintptr { + return 1 + }, + openTrace: openTrace, + processTrace: processTrace, + } + + err := session.StartConsumer() + assert.NoError(t, err) + assert.Equal(t, uint64(12345), session.traceHandler, "traceHandler should be set to the mock value") +} diff --git a/x-pack/libbeat/reader/etw/syscall_advapi32.go b/x-pack/libbeat/reader/etw/syscall_advapi32.go new file mode 100644 index 00000000000..fe44b0022a4 --- /dev/null +++ b/x-pack/libbeat/reader/etw/syscall_advapi32.go @@ -0,0 +1,318 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "errors" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + advapi32 = windows.NewLazySystemDLL("advapi32.dll") + // Controller + startTraceW = advapi32.NewProc("StartTraceW") + enableTraceEx2 = advapi32.NewProc("EnableTraceEx2") // Manifest-based providers and filtering + controlTraceW = advapi32.NewProc("ControlTraceW") + // Consumer + openTraceW = advapi32.NewProc("OpenTraceW") + processTrace = advapi32.NewProc("ProcessTrace") + closeTrace = advapi32.NewProc("CloseTrace") +) + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-event_trace +type EventTrace struct { + Header EventTraceHeader + InstanceId uint32 + ParentInstanceId uint32 + ParentGuid windows.GUID + MofData uintptr + MofLength uint32 + UnionCtx uint32 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-event_trace_header +type EventTraceHeader struct { + Size uint16 + Union1 uint16 + Union2 uint32 + ThreadId uint32 + ProcessId uint32 + TimeStamp int64 + Union3 [16]byte + Union4 uint64 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-event_trace_properties +type EventTraceProperties struct { + Wnode WnodeHeader + BufferSize uint32 + MinimumBuffers uint32 + MaximumBuffers uint32 + MaximumFileSize uint32 + LogFileMode uint32 + FlushTimer uint32 + EnableFlags uint32 + AgeLimit int32 + NumberOfBuffers uint32 + FreeBuffers uint32 + EventsLost uint32 + BuffersWritten uint32 + LogBuffersLost uint32 + RealTimeBuffersLost uint32 + LoggerThreadId syscall.Handle + LogFileNameOffset uint32 + LoggerNameOffset uint32 +} + +// https://learn.microsoft.com/en-us/windows/win32/etw/wnode-header +type WnodeHeader struct { + BufferSize uint32 + ProviderId uint32 + Union1 uint64 + Union2 int64 + Guid windows.GUID + ClientContext uint32 + Flags uint32 +} + +// Used to enable a provider via EnableTraceEx2 +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-enable_trace_parameters +type EnableTraceParameters struct { + Version uint32 + EnableProperty uint32 + ControlFlags uint32 + SourceId windows.GUID + EnableFilterDesc *EventFilterDescriptor + FilterDescrCount uint32 +} + +// Defines the filter data that a session passes +// to the provider's enable callback function +// https://learn.microsoft.com/en-us/windows/win32/api/evntprov/ns-evntprov-event_filter_descriptor +type EventFilterDescriptor struct { + Ptr uint64 + Size uint32 + Type uint32 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-event_trace_logfilew +type EventTraceLogfile struct { + LogFileName *uint16 // Logfile + LoggerName *uint16 // Real-time session + CurrentTime int64 + BuffersRead uint32 + LogFileMode uint32 + CurrentEvent EventTrace + LogfileHeader TraceLogfileHeader + BufferCallback uintptr + BufferSize uint32 + Filled uint32 + EventsLost uint32 + // Receive events (EventRecordCallback (TDH) or EventCallback) + // Tip: New code should use EventRecordCallback instead of EventCallback. + // The EventRecordCallback receives an EVENT_RECORD which contains + // more complete event information + Callback uintptr + IsKernelTrace uint32 + Context uintptr +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-trace_logfile_header +type TraceLogfileHeader struct { + BufferSize uint32 + VersionUnion uint32 + ProviderVersion uint32 + NumberOfProcessors uint32 + EndTime int64 + TimerResolution uint32 + MaximumFileSize uint32 + LogFileMode uint32 + BuffersWritten uint32 + Union1 [16]byte + LoggerName *uint16 + LogFileName *uint16 + TimeZone windows.Timezoneinformation + BootTime int64 + PerfFreq int64 + StartTime int64 + ReservedFlags uint32 + BuffersLost uint32 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-filetime +type FileTime struct { + dwLowDateTime uint32 + dwHighDateTime uint32 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-systemtime +type SystemTime struct { + Year uint16 + Month uint16 + DayOfWeek uint16 + Day uint16 + Hour uint16 + Minute uint16 + Second uint16 + Milliseconds uint16 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/nf-evntrace-enabletrace +const ( + TRACE_LEVEL_NONE = 0 + TRACE_LEVEL_CRITICAL = 1 + TRACE_LEVEL_FATAL = 1 + TRACE_LEVEL_ERROR = 2 + TRACE_LEVEL_WARNING = 3 + TRACE_LEVEL_INFORMATION = 4 + TRACE_LEVEL_VERBOSE = 5 +) + +// https://learn.microsoft.com/en-us/windows/win32/api/evntprov/nc-evntprov-penablecallback +const ( + EVENT_CONTROL_CODE_DISABLE_PROVIDER = 0 + EVENT_CONTROL_CODE_ENABLE_PROVIDER = 1 + EVENT_CONTROL_CODE_CAPTURE_STATE = 2 +) + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/nf-evntrace-controltracea +const ( + EVENT_TRACE_CONTROL_QUERY = 0 + EVENT_TRACE_CONTROL_STOP = 1 + EVENT_TRACE_CONTROL_UPDATE = 2 + EVENT_TRACE_CONTROL_FLUSH = 3 +) + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/ns-evntrace-event_trace_logfilea +const ( + PROCESS_TRACE_MODE_REAL_TIME = 0x00000100 + PROCESS_TRACE_MODE_RAW_TIMESTAMP = 0x00001000 + PROCESS_TRACE_MODE_EVENT_RECORD = 0x10000000 +) + +const INVALID_PROCESSTRACE_HANDLE = 0xFFFFFFFFFFFFFFFF + +// https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes +const ( + ERROR_ACCESS_DENIED syscall.Errno = 5 + ERROR_INVALID_HANDLE syscall.Errno = 6 + ERROR_BAD_LENGTH syscall.Errno = 24 + ERROR_INVALID_PARAMETER syscall.Errno = 87 + ERROR_INSUFFICIENT_BUFFER syscall.Errno = 122 + ERROR_BAD_PATHNAME syscall.Errno = 161 + ERROR_ALREADY_EXISTS syscall.Errno = 183 + ERROR_NOT_FOUND syscall.Errno = 1168 + ERROR_NO_SYSTEM_RESOURCES syscall.Errno = 1450 + ERROR_TIMEOUT syscall.Errno = 1460 + ERROR_WMI_INSTANCE_NOT_FOUND syscall.Errno = 4201 + ERROR_CTX_CLOSE_PENDING syscall.Errno = 7007 + ERROR_EVT_INVALID_EVENT_DATA syscall.Errno = 15005 +) + +// https://learn.microsoft.com/en-us/windows/win32/etw/logging-mode-constants (to extend modes) +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wmistr/ns-wmistr-_wnode_header (to extend flags) +const ( + WNODE_FLAG_ALL_DATA = 0x00000001 + WNODE_FLAG_TRACED_GUID = 0x00020000 + EVENT_TRACE_REAL_TIME_MODE = 0x00000100 +) + +// Wrappers + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/nf-evntrace-starttracew +func _StartTrace(traceHandle *uintptr, + instanceName *uint16, + properties *EventTraceProperties) error { + r0, _, _ := startTraceW.Call( + uintptr(unsafe.Pointer(traceHandle)), + uintptr(unsafe.Pointer(instanceName)), + uintptr(unsafe.Pointer(properties))) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/nf-evntrace-enabletraceex2 +func _EnableTraceEx2(traceHandle uintptr, + providerId *windows.GUID, + isEnabled uint32, + level uint8, + matchAnyKeyword uint64, + matchAllKeyword uint64, + enableProperty uint32, + enableParameters *EnableTraceParameters) error { + r0, _, _ := enableTraceEx2.Call( + traceHandle, + uintptr(unsafe.Pointer(providerId)), + uintptr(isEnabled), + uintptr(level), + uintptr(matchAnyKeyword), + uintptr(matchAllKeyword), + uintptr(enableProperty), + uintptr(unsafe.Pointer(enableParameters))) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/nf-evntrace-controltracew +func _ControlTrace(traceHandle uintptr, + instanceName *uint16, + properties *EventTraceProperties, + controlCode uint32) error { + r0, _, _ := controlTraceW.Call( + traceHandle, + uintptr(unsafe.Pointer(instanceName)), + uintptr(unsafe.Pointer(properties)), + uintptr(controlCode)) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/nf-evntrace-opentracew +func _OpenTrace(logfile *EventTraceLogfile) (uint64, error) { + r0, _, err := openTraceW.Call( + uintptr(unsafe.Pointer(logfile))) + var errno syscall.Errno + if errors.As(err, &errno) && errno == 0 { + return uint64(r0), nil + } + return uint64(r0), err +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/nf-evntrace-processtrace +func _ProcessTrace(handleArray *uint64, + handleCount uint32, + startTime *FileTime, + endTime *FileTime) error { + r0, _, _ := processTrace.Call( + uintptr(unsafe.Pointer(handleArray)), + uintptr(handleCount), + uintptr(unsafe.Pointer(startTime)), + uintptr(unsafe.Pointer(endTime))) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntrace/nf-evntrace-closetrace +func _CloseTrace(traceHandle uint64) error { + r0, _, _ := closeTrace.Call( + uintptr(traceHandle)) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} diff --git a/x-pack/libbeat/reader/etw/syscall_tdh.go b/x-pack/libbeat/reader/etw/syscall_tdh.go new file mode 100644 index 00000000000..73551ee123e --- /dev/null +++ b/x-pack/libbeat/reader/etw/syscall_tdh.go @@ -0,0 +1,323 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + tdh = windows.NewLazySystemDLL("tdh.dll") + tdhEnumerateProviders = tdh.NewProc("TdhEnumerateProviders") + tdhGetEventInformation = tdh.NewProc("TdhGetEventInformation") + tdhGetEventMapInformation = tdh.NewProc("TdhGetEventMapInformation") + tdhFormatProperty = tdh.NewProc("TdhFormatProperty") + tdhGetProperty = tdh.NewProc("TdhGetProperty") +) + +const anysizeArray = 1 +const DEFAULT_PROPERTY_BUFFER_SIZE = 256 + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/ns-tdh-provider_enumeration_info +type ProviderEnumerationInfo struct { + NumberOfProviders uint32 + Reserved uint32 + TraceProviderInfoArray [anysizeArray]TraceProviderInfo +} + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/ns-tdh-trace_provider_info +type TraceProviderInfo struct { + ProviderGuid windows.GUID + SchemaSource uint32 + ProviderNameOffset uint32 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntcons/ns-evntcons-event_record +type EventRecord struct { + EventHeader EventHeader + BufferContext EtwBufferContext + ExtendedDataCount uint16 + UserDataLength uint16 + ExtendedData *EventHeaderExtendedDataItem + UserData uintptr // Event data + UserContext uintptr +} + +// https://learn.microsoft.com/en-us/windows/win32/api/relogger/ns-relogger-event_header +const ( + EVENT_HEADER_FLAG_STRING_ONLY = 0x0004 + EVENT_HEADER_FLAG_32_BIT_HEADER = 0x0020 + EVENT_HEADER_FLAG_64_BIT_HEADER = 0x0040 +) + +// https://learn.microsoft.com/en-us/windows/win32/api/relogger/ns-relogger-event_header +type EventHeader struct { + Size uint16 + HeaderType uint16 + Flags uint16 + EventProperty uint16 + ThreadId uint32 + ProcessId uint32 + TimeStamp int64 + ProviderId windows.GUID + EventDescriptor EventDescriptor + Time int64 + ActivityId windows.GUID +} + +func (e *EventRecord) pointerSize() uint32 { + if e.EventHeader.Flags&EVENT_HEADER_FLAG_32_BIT_HEADER == EVENT_HEADER_FLAG_32_BIT_HEADER { + return 4 + } + return 8 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntprov/ns-evntprov-event_descriptor +type EventDescriptor struct { + Id uint16 + Version uint8 + Channel uint8 + Level uint8 + Opcode uint8 + Task uint16 + Keyword uint64 +} + +// https://learn.microsoft.com/en-us/windows/desktop/api/relogger/ns-relogger-etw_buffer_context +type EtwBufferContext struct { + Union uint16 + LoggerId uint16 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/evntcons/ns-evntcons-event_header_extended_data_item +type EventHeaderExtendedDataItem struct { + Reserved1 uint16 + ExtType uint16 + InternalStruct uint16 + DataSize uint16 + DataPtr uint64 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/ns-tdh-tdh_context +type TdhContext struct { + ParameterValue uint32 + ParameterType int32 + ParameterSize uint32 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/ns-tdh-trace_event_info +type TraceEventInfo struct { + ProviderGUID windows.GUID + EventGUID windows.GUID + EventDescriptor EventDescriptor + DecodingSource DecodingSource + ProviderNameOffset uint32 + LevelNameOffset uint32 + ChannelNameOffset uint32 + KeywordsNameOffset uint32 + TaskNameOffset uint32 + OpcodeNameOffset uint32 + EventMessageOffset uint32 + ProviderMessageOffset uint32 + BinaryXMLOffset uint32 + BinaryXMLSize uint32 + ActivityIDNameOffset uint32 + RelatedActivityIDNameOffset uint32 + PropertyCount uint32 + TopLevelPropertyCount uint32 + Flags TemplateFlags + EventPropertyInfoArray [anysizeArray]EventPropertyInfo +} + +// https://learn.microsoft.com/en-us/windows/desktop/api/tdh/ns-tdh-event_property_info +type EventPropertyInfo struct { + Flags PropertyFlags + NameOffset uint32 + TypeUnion struct { + u1 uint16 + u2 uint16 + u3 uint32 + } + CountUnion uint16 + LengthUnion uint16 + ResTagUnion uint32 +} + +func (i *EventPropertyInfo) count() uint16 { + return i.CountUnion +} + +func (i *EventPropertyInfo) length() uint16 { + return i.LengthUnion +} + +func (i *EventPropertyInfo) inType() uint16 { + return i.TypeUnion.u1 +} + +func (i *EventPropertyInfo) outType() uint16 { + return i.TypeUnion.u2 +} + +func (i *EventPropertyInfo) structStartIndex() uint16 { + return i.inType() +} + +func (i *EventPropertyInfo) numOfStructMembers() uint16 { + return i.outType() +} + +func (i *EventPropertyInfo) mapNameOffset() uint32 { + return i.TypeUnion.u3 +} + +const ( + TdhIntypeBinary = 14 + TdhOuttypeIpv6 = 24 +) + +type DecodingSource int32 +type TemplateFlags int32 + +type PropertyFlags int32 + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/ne-tdh-property_flags +const ( + PropertyStruct = PropertyFlags(0x1) + PropertyParamLength = PropertyFlags(0x2) + PropertyParamCount = PropertyFlags(0x4) +) + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/ns-tdh-event_map_info +type EventMapInfo struct { + NameOffset uint32 + Flag MapFlags + EntryCount uint32 + Union uint32 + MapEntryArray [anysizeArray]EventMapEntry +} + +type MapFlags int32 + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/ns-tdh-event_map_entry +type EventMapEntry struct { + OutputOffset uint32 + Union uint32 +} + +// https://learn.microsoft.com/en-us/windows/desktop/api/tdh/ns-tdh-property_data_descriptor +type PropertyDataDescriptor struct { + PropertyName unsafe.Pointer + ArrayIndex uint32 + Reserved uint32 +} + +// enumerateProvidersFunc is used to replace the pointer to the function in unit tests +var enumerateProvidersFunc = _TdhEnumerateProviders + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/nf-tdh-tdhenumerateproviders +func _TdhEnumerateProviders( + pBuffer *ProviderEnumerationInfo, + pBufferSize *uint32) error { + r0, _, _ := tdhEnumerateProviders.Call( + uintptr(unsafe.Pointer(pBuffer)), + uintptr(unsafe.Pointer(pBufferSize))) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/nf-tdh-tdhgeteventinformation +func _TdhGetEventInformation(pEvent *EventRecord, + tdhContextCount uint32, + pTdhContext *TdhContext, + pBuffer *TraceEventInfo, + pBufferSize *uint32) error { + r0, _, _ := tdhGetEventInformation.Call( + uintptr(unsafe.Pointer(pEvent)), + uintptr(tdhContextCount), + uintptr(unsafe.Pointer(pTdhContext)), + uintptr(unsafe.Pointer(pBuffer)), + uintptr(unsafe.Pointer(pBufferSize))) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/nf-tdh-tdhformatproperty +func _TdhFormatProperty( + eventInfo *TraceEventInfo, + mapInfo *EventMapInfo, + pointerSize uint32, + propertyInType uint16, + propertyOutType uint16, + propertyLength uint16, + userDataLength uint16, + userData *byte, + bufferSize *uint32, + buffer *uint8, + userDataConsumed *uint16) error { + r0, _, _ := tdhFormatProperty.Call( + uintptr(unsafe.Pointer(eventInfo)), + uintptr(unsafe.Pointer(mapInfo)), + uintptr(pointerSize), + uintptr(propertyInType), + uintptr(propertyOutType), + uintptr(propertyLength), + uintptr(userDataLength), + uintptr(unsafe.Pointer(userData)), + uintptr(unsafe.Pointer(bufferSize)), + uintptr(unsafe.Pointer(buffer)), + uintptr(unsafe.Pointer(userDataConsumed))) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/nf-tdh-tdhgetproperty +func _TdhGetProperty(pEvent *EventRecord, + tdhContextCount uint32, + pTdhContext *TdhContext, + propertyDataCount uint32, + pPropertyData *PropertyDataDescriptor, + bufferSize uint32, + pBuffer *byte) error { + r0, _, _ := tdhGetProperty.Call( + uintptr(unsafe.Pointer(pEvent)), + uintptr(tdhContextCount), + uintptr(unsafe.Pointer(pTdhContext)), + uintptr(propertyDataCount), + uintptr(unsafe.Pointer(pPropertyData)), + uintptr(bufferSize), + uintptr(unsafe.Pointer(pBuffer))) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} + +// https://learn.microsoft.com/en-us/windows/win32/api/tdh/nf-tdh-tdhgeteventmapinformation +func _TdhGetEventMapInformation(pEvent *EventRecord, + pMapName *uint16, + pBuffer *EventMapInfo, + pBufferSize *uint32) error { + r0, _, _ := tdhGetEventMapInformation.Call( + uintptr(unsafe.Pointer(pEvent)), + uintptr(unsafe.Pointer(pMapName)), + uintptr(unsafe.Pointer(pBuffer)), + uintptr(unsafe.Pointer(pBufferSize))) + if r0 == 0 { + return nil + } + return syscall.Errno(r0) +} diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml index f1396fdf9e8..42b946e4221 100644 --- a/x-pack/metricbeat/docker-compose.yml +++ b/x-pack/metricbeat/docker-compose.yml @@ -24,11 +24,11 @@ services: kibana: # Copied configuration from OSS metricbeat because services with depends_on # cannot be extended with extends - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.11.2}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.12.1}-1 build: context: ../../metricbeat/module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-8.11.2} + KIBANA_VERSION: ${KIBANA_VERSION:-8.12.1} depends_on: - elasticsearch ports: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index a22db4f7f8c..1e6abf11a60 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2762,9 +2762,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/metricbeat/module/oracle/_meta/docs.asciidoc b/x-pack/metricbeat/module/oracle/_meta/docs.asciidoc index 7a93e306981..887b0601939 100644 --- a/x-pack/metricbeat/module/oracle/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/oracle/_meta/docs.asciidoc @@ -48,19 +48,24 @@ Then, Metricbeat can be launched. *Host Configuration* -The following two types of host configurations are supported: +The following types of host configuration are supported: -1. Old style host configuration for backwards compatibility: +1. An old-style Oracle connection string, for backwards compatibility: a. `hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"]` b. `hosts: ["user/password@0.0.0.0:1521/ORCLPDB1.localdomain as sysdba"]` -2. DSN host configuration: +2. DSN configuration as a URL: + a. `hosts: ["oracle://user:pass@0.0.0.0:1521/ORCLPDB1.localdomain?sysdba=1"]` + +3. DSN configuration as a logfmt-encoded parameter list: a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']` b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']` -DSN host configuration is the recommended way to configure the Oracle Metricbeat Module as it supports the usage of special characters in the password. +DSN host configuration is the recommended configuration type as it supports the use of special characters in the password. + +In a URL any special characters should be URL encoded. -Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`. +In the logfmt-encoded DSN format, if the password contains a backslash character (`\`), it must be escaped with another backslash. For example, if the password is `my\_password`, it must be written as `my\\_password`. [float] == Metricsets diff --git a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc index 17175cb5878..95ae9376e4d 100644 --- a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc @@ -859,19 +859,26 @@ Then, Metricbeat can be launched. ===== Host Configuration for Oracle -The following two types of host configurations are supported: +The following types of host configuration are supported: -1. DSN host configuration as URL: +1. An old-style Oracle connection string, for backwards compatibility: a. `hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"]` b. `hosts: ["user/password@0.0.0.0:1521/ORCLPDB1.localdomain as sysdba"]` -2. DSN host configuration: +2. DSN configuration as a URL: + a. `hosts: ["oracle://user:pass@0.0.0.0:1521/ORCLPDB1.localdomain?sysdba=1"]` + +3. DSN configuration as a logfmt-encoded parameter list: a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']` b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']` -Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`. +DSN host configuration is the recommended configuration type as it supports the use of special characters in the password. + +In a URL any special characters should be URL encoded. -The username and password to connect to the database can be provided as values to `username` and `password` keys of `sql.yml`. +In the logfmt-encoded DSN format, if the password contains a backslash character (`\`), it must be escaped with another backslash. For example, if the password is `my\_password`, it must be written as `my\\_password`. + +The username and password to connect to the database can be provided as values to the `username` and `password` keys of `sql.yml`. [source,yml] ---- @@ -887,4 +894,4 @@ The username and password to connect to the database can be provided as values t sql_queries: - query: SELECT METRIC_NAME, VALUE FROM V$SYSMETRIC WHERE GROUP_ID = 2 and METRIC_NAME LIKE '%' response_format: variables ----- \ No newline at end of file +---- diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index b4da8bf79a3..ffce883a60c 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.21.6 AS build-env +FROM golang:1.21.7 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build . diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 1de9a267ae5..0c28af89144 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -790,9 +790,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/packetbeat/_meta/config/output-elasticsearch.yml.tmpl b/x-pack/packetbeat/_meta/config/output-elasticsearch.yml.tmpl new file mode 100644 index 00000000000..ffb3bc696fc --- /dev/null +++ b/x-pack/packetbeat/_meta/config/output-elasticsearch.yml.tmpl @@ -0,0 +1,15 @@ +{{subheader "Elasticsearch Output"}} +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + + # Pipeline to route events to protocol pipelines. + pipeline: "packetbeat-%{[agent.version]}-routing" diff --git a/x-pack/packetbeat/cmd/root.go b/x-pack/packetbeat/cmd/root.go index f77bd827bf2..8611fe8d115 100644 --- a/x-pack/packetbeat/cmd/root.go +++ b/x-pack/packetbeat/cmd/root.go @@ -21,6 +21,9 @@ import ( // This registers the Npcap installer on Windows. _ "github.com/elastic/beats/v7/x-pack/packetbeat/npcap" + + // Enable pipelines. + _ "github.com/elastic/beats/v7/x-pack/packetbeat/module" ) // Name of this beat. diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index acef21538d6..03104ab9157 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -47,6 +47,7 @@ func init() { devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch." devtools.BeatLicense = "Elastic License" + packetbeat.SelectLogic = devtools.XPackProject } // Update updates the generated files. diff --git a/x-pack/packetbeat/module/amqp/ingest/default.yml b/x-pack/packetbeat/module/amqp/ingest/default.yml new file mode 100644 index 00000000000..7b2268f4812 --- /dev/null +++ b/x-pack/packetbeat/module/amqp/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing amqp traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + tag: gsubmac + ignore_missing: true +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + tag: gsubmac + ignore_missing: true +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreachip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipelineprocessor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/amqp/ingest/geoip.yml b/x-pack/packetbeat/module/amqp/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/amqp/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/cassandra/ingest/default.yml b/x-pack/packetbeat/module/cassandra/ingest/default.yml new file mode 100644 index 00000000000..61ce5ff4d73 --- /dev/null +++ b/x-pack/packetbeat/module/cassandra/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing cassandra traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsubmac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsubmac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreachip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipelineprocessor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/cassandra/ingest/geoip.yml b/x-pack/packetbeat/module/cassandra/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/cassandra/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/dhcpv4/ingest/default.yml b/x-pack/packetbeat/module/dhcpv4/ingest/default.yml new file mode 100644 index 00000000000..1c3a2a57264 --- /dev/null +++ b/x-pack/packetbeat/module/dhcpv4/ingest/default.yml @@ -0,0 +1,74 @@ +--- +description: Pipeline for processing dhcpv4 traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: dhcpv4.client_mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_dhcpv4_client_mac +- gsub: + field: dhcpv4.client_mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_dhcpv4_client_mac +- uppercase: + field: dhcpv4.client_mac + ignore_missing: true +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/dhcpv4/ingest/geoip.yml b/x-pack/packetbeat/module/dhcpv4/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/dhcpv4/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/dns/ingest/default.yml b/x-pack/packetbeat/module/dns/ingest/default.yml new file mode 100644 index 00000000000..ff055c3c9b3 --- /dev/null +++ b/x-pack/packetbeat/module/dns/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing dhcpv4 traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/dns/ingest/geoip.yml b/x-pack/packetbeat/module/dns/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/dns/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/flow/ingest/default.yml b/x-pack/packetbeat/module/flow/ingest/default.yml new file mode 100644 index 00000000000..6e969ea1a61 --- /dev/null +++ b/x-pack/packetbeat/module/flow/ingest/default.yml @@ -0,0 +1,89 @@ +--- +description: Pipeline for processing traffic flows +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set {host,source,destination}.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + tag: foreach_observer_ip + field: observer.ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host +- gsub: + field: source.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_source_mac +- gsub: + field: source.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_source_mac +- uppercase: + field: source.mac + ignore_missing: true +- gsub: + field: destination.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_destination_mac +- gsub: + field: destination.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_destination_mac +- uppercase: + field: destination.mac + ignore_missing: true + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/flow/ingest/geoip.yml b/x-pack/packetbeat/module/flow/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/flow/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/http/ingest/default.yml b/x-pack/packetbeat/module/http/ingest/default.yml new file mode 100644 index 00000000000..e066200becb --- /dev/null +++ b/x-pack/packetbeat/module/http/ingest/default.yml @@ -0,0 +1,72 @@ +--- +description: Pipeline for processing http traffic +processors: +- set: + field: ecs.version + value: '8.11.0' + +# Detection Rules compatibility +- set: + tag: set_compatibility_request_authorization + field: network_traffic.http.request.headers.authorization + copy_from: http.request.headers.authorization + ignore_empty_value: true +- set: + tag: set_compatibility_response_type + field: http.response.mime_type + copy_from: http.response.headers.content-type + ignore_empty_value: true + +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + tag: foreach_observer_ip + field: observer.ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/http/ingest/geoip.yml b/x-pack/packetbeat/module/http/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/http/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/icmp/ingest/default.yml b/x-pack/packetbeat/module/icmp/ingest/default.yml new file mode 100644 index 00000000000..7a50bb91cc5 --- /dev/null +++ b/x-pack/packetbeat/module/icmp/ingest/default.yml @@ -0,0 +1,66 @@ +--- +description: Pipeline for processing icmp traffic +processors: +- set: + field: ecs.version + value: '8.11.0' + +# Detection Rules compatibility +- set: + tag: set_compatibility_type + field: network.protocol + copy_from: type + +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + tag: foreach_observer_ip + field: observer.ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/icmp/ingest/geoip.yml b/x-pack/packetbeat/module/icmp/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/icmp/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/memcached/ingest/default.yml b/x-pack/packetbeat/module/memcached/ingest/default.yml new file mode 100644 index 00000000000..d0f5f18088c --- /dev/null +++ b/x-pack/packetbeat/module/memcached/ingest/default.yml @@ -0,0 +1,79 @@ +--- +description: Pipeline for processing memcached traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + tag: foreach_observer_ip + field: observer.ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +## +# Reformat memcache stats response data as a single object +## +- rename: + field: memcache.response.stats + target_field: memcache.response.stats_objects + ignore_missing: true +- foreach: + description: Build an object for memcache stats response data + if: ctx.memcache?.response?.stats_objects instanceof List + tag: foreach_memcache_response_stats_objects + field: memcache.response.stats_objects + processor: + set: + field: "memcache.response.stats.{{{_ingest._value.name}}}" + value: "{{{_ingest._value.value}}}" +- remove: + field: memcache.response.stats_objects + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/memcached/ingest/geoip.yml b/x-pack/packetbeat/module/memcached/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/memcached/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/mongodb/ingest/default.yml b/x-pack/packetbeat/module/mongodb/ingest/default.yml new file mode 100644 index 00000000000..a40e27da35d --- /dev/null +++ b/x-pack/packetbeat/module/mongodb/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing mongodb traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/mongodb/ingest/geoip.yml b/x-pack/packetbeat/module/mongodb/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/mongodb/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/mysql/ingest/default.yml b/x-pack/packetbeat/module/mysql/ingest/default.yml new file mode 100644 index 00000000000..e9cb2ebcdb0 --- /dev/null +++ b/x-pack/packetbeat/module/mysql/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing mysql traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/mysql/ingest/geoip.yml b/x-pack/packetbeat/module/mysql/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/mysql/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/nfs/ingest/default.yml b/x-pack/packetbeat/module/nfs/ingest/default.yml new file mode 100644 index 00000000000..a1b72a25217 --- /dev/null +++ b/x-pack/packetbeat/module/nfs/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing nfs traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/nfs/ingest/geoip.yml b/x-pack/packetbeat/module/nfs/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/nfs/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/pgsql/ingest/default.yml b/x-pack/packetbeat/module/pgsql/ingest/default.yml new file mode 100644 index 00000000000..bd28f9211e1 --- /dev/null +++ b/x-pack/packetbeat/module/pgsql/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing pgsql traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/pgsql/ingest/geoip.yml b/x-pack/packetbeat/module/pgsql/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/pgsql/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/pipeline.go b/x-pack/packetbeat/module/pipeline.go new file mode 100644 index 00000000000..a325fba7de4 --- /dev/null +++ b/x-pack/packetbeat/module/pipeline.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package module + +import ( + "embed" + + "github.com/elastic/beats/v7/packetbeat/module" +) + +// pipelineFS holds the yml representation of the ingest node pipelines +// +//go:embed */ingest/*.yml +var pipelinesFS embed.FS + +func init() { + module.PipelinesFS = &pipelinesFS +} diff --git a/x-pack/packetbeat/module/redis/ingest/default.yml b/x-pack/packetbeat/module/redis/ingest/default.yml new file mode 100644 index 00000000000..4f815adc3a9 --- /dev/null +++ b/x-pack/packetbeat/module/redis/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing redis traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/redis/ingest/geoip.yml b/x-pack/packetbeat/module/redis/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/redis/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/routing/ingest/default.yml b/x-pack/packetbeat/module/routing/ingest/default.yml new file mode 100644 index 00000000000..a11b5e79f7a --- /dev/null +++ b/x-pack/packetbeat/module/routing/ingest/default.yml @@ -0,0 +1,64 @@ +--- +description: Route to appropriate data source pipenline. +processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + + - pipeline: + if: ctx.type == "amqp" + name: '{< IngestPipeline "amqp" >}' + - pipeline: + if: ctx.type == "cassandra" + name: '{< IngestPipeline "cassandra" >}' + - pipeline: + if: ctx.type == "dhcpv4" + name: '{< IngestPipeline "dhcpv4" >}' + - pipeline: + if: ctx.type == "dns" + name: '{< IngestPipeline "dns" >}' + - pipeline: + if: ctx.type == "flow" + name: '{< IngestPipeline "flow" >}' + - pipeline: + if: ctx.type == "http" + name: '{< IngestPipeline "http" >}' + - pipeline: + if: ctx.type == "icmp" + name: '{< IngestPipeline "icmp" >}' + - pipeline: + if: ctx.type == "memcache" + name: '{< IngestPipeline "memcached" >}' + - pipeline: + if: ctx.type == "mongodb" + name: '{< IngestPipeline "mongodb" >}' + - pipeline: + if: ctx.type == "mysql" + name: '{< IngestPipeline "mysql" >}' + - pipeline: + if: ctx.type == "nfs" + name: '{< IngestPipeline "nfs" >}' + - pipeline: + if: ctx.type == "pgsql" + name: '{< IngestPipeline "pgsql" >}' + - pipeline: + if: ctx.type == "redis" + name: '{< IngestPipeline "redis" >}' + - pipeline: + if: ctx.type == "sip" + name: '{< IngestPipeline "sip" >}' + - pipeline: + if: ctx.type == "thrift" + name: '{< IngestPipeline "thrift" >}' + - pipeline: + if: ctx.type == "tls" + name: '{< IngestPipeline "tls" >}' + +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" diff --git a/x-pack/packetbeat/module/sip/ingest/default.yml b/x-pack/packetbeat/module/sip/ingest/default.yml new file mode 100644 index 00000000000..62f3d6c1c42 --- /dev/null +++ b/x-pack/packetbeat/module/sip/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing sip traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/sip/ingest/geoip.yml b/x-pack/packetbeat/module/sip/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/sip/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/thrift/ingest/default.yml b/x-pack/packetbeat/module/thrift/ingest/default.yml new file mode 100644 index 00000000000..f2726cea96b --- /dev/null +++ b/x-pack/packetbeat/module/thrift/ingest/default.yml @@ -0,0 +1,59 @@ +--- +description: Pipeline for processing thrift traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/thrift/ingest/geoip.yml b/x-pack/packetbeat/module/thrift/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/thrift/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/tls/ingest/default.yml b/x-pack/packetbeat/module/tls/ingest/default.yml new file mode 100644 index 00000000000..94ef3b55d22 --- /dev/null +++ b/x-pack/packetbeat/module/tls/ingest/default.yml @@ -0,0 +1,99 @@ +--- +description: Pipeline for processing tls traffic +processors: +- set: + field: ecs.version + value: '8.11.0' +## +# Set host.mac to dash separated upper case value +# as per ECS recommendation +## +- gsub: + field: host.mac + pattern: '[-:.]' + replacement: '' + ignore_missing: true + tag: gsub_host_mac +- gsub: + field: host.mac + pattern: '(..)(?!$)' + replacement: '$1-' + ignore_missing: true + tag: gsub_host_mac +- uppercase: + field: host.mac + ignore_missing: true +- append: + field: related.hosts + value: "{{{observer.hostname}}}" + if: ctx.observer?.hostname != null && ctx.observer?.hostname != '' + allow_duplicates: false +- foreach: + if: ctx.observer?.ip != null && ctx.observer.ip instanceof List + field: observer.ip + tag: foreach_observer_ip + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false +- remove: + if: ctx.host != null && ctx.tags != null && ctx.tags.contains('forwarded') + field: host + +- pipeline: + if: ctx._conf?.geoip_enrich != null && ctx._conf.geoip_enrich + name: '{{ IngestPipeline "geoip" }}' + tag: pipeline_processor +- remove: + field: _conf + ignore_missing: true + +## +# Make tls.{client,server}.x509.version_number a string as per ECS. +## +- convert: + field: tls.client.x509.version_number + type: string + ignore_missing: true + tag: convert_tls_client_x509_version_number +- convert: + field: tls.server.x509.version_number + type: string + ignore_missing: true + tag: convert_tls_server_x509_version_number + +## +# This handles legacy TLS fields from Packetbeat 7.17. +## +- remove: + description: Remove legacy fields from Packetbeat 7.17 that are duplicated. + field: + - tls.client.x509.issuer.province # Duplicated as tls.client.x509.issuer.state_or_province. + - tls.client.x509.subject.province # Duplicated as tls.client.x509.subject.state_or_province. + - tls.client.x509.version # Duplicated as tls.client.x509.version_number. + - tls.detailed.client_certificate # Duplicated as tls.client.x509. + - tls.detailed.server_certificate # Duplicated as tls.server.x509. + - tls.server.x509.issuer.province # Duplicated as tls.server.x509.issuer.state_or_province. + - tls.server.x509.subject.province # Duplicated as tls.server.x509.subject.state_or_province. + - tls.server.x509.version # Duplicated as tls.server.x509.version_number. + ignore_missing: true + +- append: + field: related.hash + value: "{{tls.server.ja3s}}" + if: "ctx?.tls?.server?.ja3s != null" +- append: + field: related.hash + value: "{{tls.client.ja3}}" + if: "ctx?.tls?.client?.ja3 != null" + allow_duplicates: false + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/module/tls/ingest/geoip.yml b/x-pack/packetbeat/module/tls/ingest/geoip.yml new file mode 100644 index 00000000000..eb88d38caf0 --- /dev/null +++ b/x-pack/packetbeat/module/tls/ingest/geoip.yml @@ -0,0 +1,103 @@ +--- +description: GeoIP enrichment. +processors: + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + tag: source_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: source_geo + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: destination_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: destination_geo + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + - geoip: + field: server.ip + target_field: server.geo + ignore_missing: true + tag: server_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + tag: server_geo + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + tag: client_geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + tag: client_geo + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + +on_failure: + - append: + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + - set: + field: event.kind + value: pipeline_error diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 1e013fb081f..6eaee863da0 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -78,6 +78,11 @@ packetbeat.interfaces.internal_networks: # can stay enabled even after beat is shut down. #packetbeat.interfaces.auto_promisc_mode: true +# By default Ingest pipelines are not updated if a pipeline with the same ID +# already exists. If this option is enabled Packetbeat overwrites pipelines +# every time a new Elasticsearch connection is established. +#packetbeat.overwrite_pipelines: false + # =================================== Flows ==================================== packetbeat.flows: @@ -1817,9 +1822,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/packetbeat/packetbeat.yml b/x-pack/packetbeat/packetbeat.yml index fea1a2fb115..d78fb6a7ccd 100644 --- a/x-pack/packetbeat/packetbeat.yml +++ b/x-pack/packetbeat/packetbeat.yml @@ -213,10 +213,6 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] - # Performance preset - one of "balanced", "throughput", "scale", - # "latency", or "custom". - preset: balanced - # Protocol - either `http` (default) or `https`. #protocol: "https" @@ -225,6 +221,9 @@ output.elasticsearch: #username: "elastic" #password: "changeme" + # Pipeline to route events to protocol pipelines. + pipeline: "packetbeat-%{[agent.version]}-routing" + # ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 528560748fb..5bc8f774e03 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1235,9 +1235,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false.