From bb1ee5df9d043cb4cf648cc2279474aa05fd30d8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 20 Mar 2024 15:27:41 +0100 Subject: [PATCH] Beats pipeline generator (#38162) This commit introduces a main pipeline generator similar to the one used with Jenkins #20104 to help with the migration to Buildkite. Co-authored-by: Pavel Zorin Co-authored-by: Alexandros Sapranidis (cherry picked from commit 4b7467b5c5a72bf867b73c9493b73d145f80ebb8) --- .buildkite/auditbeat/scripts/unit-tests.sh | 3 +- .buildkite/buildkite.yml | 44 +++ .buildkite/hooks/pre-command | 3 +- .buildkite/pipeline.py | 431 +++++++++++++++++++++ .buildkite/pipeline.yml | 9 +- .buildkite/scripts/generate_pipeline.sh | 11 + auditbeat/buildkite.yml | 56 +++ filebeat/buildkite.yml | 53 +++ 8 files changed, 605 insertions(+), 5 deletions(-) create mode 100644 .buildkite/buildkite.yml create mode 100755 .buildkite/pipeline.py create mode 100755 .buildkite/scripts/generate_pipeline.sh create mode 100644 auditbeat/buildkite.yml create mode 100644 filebeat/buildkite.yml diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh index 8a7514d747a..4b8e86243c9 100755 --- a/.buildkite/auditbeat/scripts/unit-tests.sh +++ b/.buildkite/auditbeat/scripts/unit-tests.sh @@ -5,6 +5,5 @@ set -euo pipefail echo "--- Running Unit Tests" sudo chmod -R go-w auditbeat/ -cd auditbeat umask 0022 -mage build unitTest +mage -d auditbeat build unitTest diff --git a/.buildkite/buildkite.yml b/.buildkite/buildkite.yml new file mode 100644 index 00000000000..4707707e07c --- /dev/null +++ b/.buildkite/buildkite.yml @@ -0,0 +1,44 @@ +projects: + - "auditbeat" + - "deploy/kubernetes" + - "filebeat" + - "heartbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/heartbeat" + - "x-pack/libbeat" + - "x-pack/metricbeat" + - "x-pack/osquerybeat" + - "x-pack/packetbeat" + - "x-pack/winlogbeat" + +## Changeset macros that are defined here and used in each specific 3.0 pipeline. +changeset: + ci: + - "^Jenkinsfile" + - "^\\.ci/scripts/.*" + oss: + - "^go.mod" + - "^pytest.ini" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + xpack: + - "^go.mod" + - "^pytest.ini" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + - "^x-pack/libbeat/.*" + +disabled: + when: + labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. + - skip-ci + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 9df1bed50a7..d7f4b336b9e 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -29,7 +29,8 @@ retry() { return 0 } -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py new file mode 100755 index 00000000000..95530b89628 --- /dev/null +++ b/.buildkite/pipeline.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +from typing import Any +from ruamel.yaml import YAML +import os +import subprocess +import fnmatch +import sys + + +class Agent: + """Buildkite Agent object""" + + def __init__(self, image: str, provider: str): + self.image: str = image + self.provider: str = provider + + def create_entity(self): + raise NotImplementedError("Not implemented yet") + + +class AWSAgent(Agent): + """AWS Agent object""" + + def __init__(self, image: str, instance_type: str = None): + super().__init__(image, "aws") + if instance_type is None: + self.instance_type: str = "t4g.large" + else: + self.instance_type = instance_type + + def create_entity(self) -> dict[str, str]: + return { + "provider": self.provider, + "imagePrefix": self.image, + "instanceType": self.instance_type, + } + + +class GCPAgent(Agent): + """GCP Agent object""" + + def __init__(self, image: str): + super().__init__(image, "gcp") + + def create_entity(self) -> dict[str, str]: + return { + "provider": self.provider, + "image": self.image, + } + + +class OrkaAgent(Agent): + """Orka Agent object""" + + def __init__(self, image: str): + super().__init__(image, "orka") + + def create_entity(self) -> dict[str, str]: + return { + "provider": self.provider, + "imagePrefix": self.image, + } + + +class Step: + """Buildkite Step object""" + + def __init__( + self, + name: str, + project: str, + category: str, + agent: Agent, + definition: dict[str, Any], + ): + self.command = definition.get("command", "") + self.env = definition.get("env", {}) + self.agent: Agent = agent + self.name: str = name + self.project: str = project + self.category: str = category + self.comment = "/test " + self.project + " " + self.name + self.label = self.name + + def __lt__(self, other): + return self.name < other.name + + def step_command(self) -> list[str]: + commands = [ + f"cd {self.project}", + self.command, + ] + return commands + + def create_entity(self) -> dict[str, Any]: + data = { + "label": f"{self.project} {self.name}", + "command": self.step_command(), + "notify": [ + { + "github_commit_status": { + "context": f"{self.project.title()}: {self.name}", + } + } + ], + "agents": self.agent.create_entity(), + "artifact_paths": [ + f"{self.project}/build/*.xml", + f"{self.project}/build/*.json", + ], + } + if self.env: + data["env"] = self.env + return data + + +class Group: + """Buildkite Group object""" + + def __init__(self, project: str, category: str, steps: list[Step]): + self.project: str = project + self.category: str = category + self.steps: list[Step] = steps + + def __lt__(self, other): + return self.project < other.project + + def create_entity(self) -> dict[str, Any]: + if len(self.steps) == 0: + return {} + + data = { + "group": f"{self.project} {self.category}", + "key": f"{self.project}-{self.category}", + "steps": [step.create_entity() for step in self.steps], + } + + return data + + +class GitHelper: + def __init__(self): + self.files: list[str] = [] + + def get_pr_changeset(self) -> list[str]: + base_branch = os.getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main") + diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(base_branch)] + result = subprocess.run(diff_command, stdout=subprocess.PIPE) + if result.returncode == 0: + self.files = result.stdout.decode().splitlines() + else: + print(f"Detecting changed files failed, exiting [{result.returncode}]") + exit(result.returncode) + return self.files + + +class BuildkitePipeline: + """Buildkite Pipeline object""" + + def __init__(self, groups: list[Group] = None): + if groups is None: + groups = [] + self.groups: list[Group] = groups + + def create_entity(self): + data = {"steps": [group.create_entity() for group in self.groups]} + return data + + +def is_pr() -> bool: + return os.getenv("BUILDKITE_PULL_REQUEST") != "false" + + +def group_comment(group: Group) -> bool: + comment = os.getenv("GITHUB_PR_TRIGGER_COMMENT") + if comment: + # the comment should be a subset of the values + # in .buildkite/pull-requests.json + # TODO: change /test + comment_prefix = "buildkite test" + if group.category == "mandatory": + # i.e: /test filebeat + return comment_prefix + " " + group.project in comment + else: + # i.e: test filebeat extended + return ( + comment_prefix + " " + group.project + " " + group.category in comment + ) + + +def filter_files_by_glob(files, patterns: list[str]): + for pattern in patterns: + # TODO: Support glob extended patterns: ^ and etc. + # Now it supports only linux glob syntax + if fnmatch.filter(files, pattern): + return True + return False + + +def is_in_pr_changeset( + project_changeset_filters: list[str], changeset: list[str] +) -> bool: + return filter_files_by_glob(changeset, project_changeset_filters) + + +def is_group_enabled( + group: Group, changeset_filters: list[str], changeset: list[str] +) -> bool: + if not is_pr(): + return True + + if ( + is_pr() + and is_in_pr_changeset(changeset_filters, changeset) + and group.category.startswith("mandatory") + ): + return True + + return group_comment(group) + + +def fetch_stage(name: str, stage, project: str, category: str) -> Step: + """Create a step given the yaml object.""" + + agent: Agent = None + if ("provider" not in stage) or stage["provider"] == "gcp": + agent = GCPAgent(image=stage["platform"]) + elif stage["provider"] == "aws": + agent = AWSAgent( + image=stage["platform"], + ) + elif stage["provider"] == "orka": + agent = OrkaAgent(image=stage["platform"]) + + return Step( + category=category, name=name, agent=agent, project=project, definition=stage + ) + + +def fetch_group(stages, project: str, category: str) -> Group: + """Create a group given the yaml object.""" + + steps = [] + + for stage in stages: + steps.append( + fetch_stage( + category=category, name=stage, project=project, stage=stages[stage] + ) + ) + + return Group(project=project, category=category, steps=steps) + + +def fetch_pr_pipeline(yaml: YAML) -> list[Group]: + git_helper = GitHelper() + changeset = git_helper.get_pr_changeset() + groups: list[Group] = [] + doc = pipeline_loader(yaml) + for project in doc["projects"]: + project_file = os.path.join(project, "buildkite.yml") + if not os.path.isfile(project_file): + continue + project_obj = project_loader(yaml, project_file) + group = fetch_group( + stages=project_obj["stages"]["mandatory"], + project=project, + category="mandatory", + ) + + if is_group_enabled(group, project_obj["when"]["changeset"], changeset): + groups.append(group) + + group = fetch_group( + stages=project_obj["stages"]["extended"], + project=project, + category="extended", + ) + + if is_group_enabled(group, project_obj["when"]["changeset"], changeset): + groups.append(group) + + # TODO: improve this merging lists + all_groups = [] + for group in groups: + all_groups.append(group) + + return all_groups + + +class PRComment: + command: str + group: str + project: str + step: str + + def __init__(self, comment: str): + words = comment.split() + self.command = words.pop(0) if words else "" + self.project = words.pop(0) if words else "" + self.group = words.pop(0) if words else "" + self.step = words.pop(0) if words else "" + + +# A comment like "/test filebeat extended" +# Returns a group of steps corresponding to the comment +def fetch_pr_comment_group_pipeline(comment: PRComment, yaml: YAML) -> list[Group]: + groups = [] + doc = pipeline_loader(yaml) + if comment.project in doc["projects"]: + project_file = os.path.join(comment.project, "buildkite.yml") + if not os.path.isfile(project_file): + raise FileNotFoundError( + "buildkite.yml not found in: " + "{}".format(comment.project) + ) + project_obj = project_loader(yaml, project_file) + if not project_obj["stages"][comment.group]: + raise ValueError( + "Group not found in {} buildkite.yml: {}".format( + comment.project, comment.group + ) + ) + + group = fetch_group( + stages=project_obj["stages"][comment.group], + project=comment.project, + category="mandatory", + ) + groups.append(group) + + return groups + + +# A comment like "/test filebeat extended unitTest-macos" +def fetch_pr_comment_step_pipeline(comment: PRComment, yaml: YAML) -> list[Group]: + groups = [] + doc = pipeline_loader(yaml) + if comment.project in doc["projects"]: + project_file = os.path.join(comment.project, "buildkite.yml") + if not os.path.isfile(project_file): + raise FileNotFoundError( + "buildkite.yml not found in: " + "{}".format(comment.project) + ) + project_obj = project_loader(yaml, project_file) + if not project_obj["stages"][comment.group]: + raise ValueError( + "Group not found in {} buildkite.yml: {}".format( + comment.project, comment.group + ) + ) + group = fetch_group( + stages=project_obj["stages"][comment.group], + project=comment.project, + category="mandatory", + ) + + filtered_steps = list( + filter(lambda step: step.name == comment.step, group.steps) + ) + + if not filtered_steps: + raise ValueError( + "Step {} not found in {} buildkite.yml".format( + comment.step, comment.project + ) + ) + group.steps = filtered_steps + groups.append(group) + + return groups + + +def pr_comment_pipeline(pr_comment: PRComment, yaml: YAML) -> list[Group]: + + if pr_comment.command == "/test": + + # A comment like "/test" for a PR + # We rerun the PR pipeline + if not pr_comment.group: + return fetch_pr_pipeline(yaml) + + # A comment like "/test filebeat" + # We don't know what group to run hence raise an error + if pr_comment.project and not pr_comment.group: + raise ValueError( + "Specify group or/and step for {}".format(pr_comment.project) + ) + + # A comment like "/test filebeat extended" + # We rerun the filebeat extended pipeline for the PR + if pr_comment.group and not pr_comment.step: + return fetch_pr_comment_group_pipeline(pr_comment, yaml) + + # A comment like "/test filebeat extended unitTest-macos" + if pr_comment.step: + return fetch_pr_comment_step_pipeline(pr_comment, yaml) + + +# TODO: validate unique stages! +def main() -> None: + yaml = YAML(typ="safe") + all_groups = [] + if is_pr(): + if os.getenv("GITHUB_PR_TRIGGER_COMMENT"): + comment = PRComment(os.getenv("GITHUB_PR_TRIGGER_COMMENT")) + all_groups = pr_comment_pipeline(comment, yaml) + else: + all_groups = fetch_pr_pipeline(yaml) + # TODO what to load when not in PR + + # Produce the dynamic pipeline + print( + "# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json" + ) + yaml.dump(BuildkitePipeline(all_groups).create_entity(), sys.stdout) + + +def pipeline_loader(yaml: YAML = YAML(typ="safe")): + with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: + return yaml.load(file) + + +def project_loader(yaml: YAML = YAML(typ="safe"), project_file: str = ""): + with open(project_file, "r", encoding="utf8") as project_fp: + return yaml.load(project_fp) + + +if __name__ == "__main__": + + # pylint: disable=E1120 + main() diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 34321b61161..8f124fea4ae 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,10 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + ASDF_MAGE_VERSION: '1.15.0' steps: - - label: "Example test" - command: echo "Hello!" + - label: "Generate dynamic pipeline" + command: ".buildkite/scripts/generate_pipeline.sh" + agents: + provider: "gcp" + image: "family/platform-ingest-beats-ubuntu-2204" diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh new file mode 100755 index 00000000000..342dc8bf7ac --- /dev/null +++ b/.buildkite/scripts/generate_pipeline.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "~~~ Install dependencies" +pip3 install --quiet "ruamel.yaml<0.18.0" + +echo "+++ Run pipeline generator in dry-run mode" +python3 .buildkite/pipeline.py | yq . + +echo "~~~ Upload pipeline" +python3 .buildkite/pipeline.py | buildkite-agent pipeline upload diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml new file mode 100644 index 00000000000..2abf9d68407 --- /dev/null +++ b/auditbeat/buildkite.yml @@ -0,0 +1,56 @@ +when: + changeset: ## when PR contains any of those entries in the changeset + - "auditbeat/**" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss +stages: + # mandatory stage - it runs always for: + # - branches/tags + # - on PRs + # - GitHub comment /test auditbeat + # - GitHub label auditbeat + mandatory: + # NOTE: stage name should be unique! + unitTest: + command: "mage build unitTest" + platform: "family/platform-ingest-beats-ubuntu-2204" + crosscompile: + command: "make crosscompile" + platform: "family/platform-ingest-beats-ubuntu-2204" + env: + GOX_FLAGS: "-arch amd64" + unitTest-rhel-9: + command: "mage build unitTest" + platform: "family/platform-ingest-beats-rhel-9" + unitTest-windows-2022: + command: "mage build unitTest" + platform: "family/platform-ingest-beats-windows-2022" + unitTest-windows-2016: + command: "mage build unitTest" + platform: "family/platform-ingest-beats-windows-2016" + # optional stage - it runs on: + # - branches/tags + # - on PRs if: + # - GitHub comment /test auditbeat . i.e: /test auditbeat integTest + # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... + extended: + # NOTE: stage name should be unique! + integTest: + command: "mage build integTest" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" + integTest-arm: + command: "mage build integTest" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" + unitTest-arm: + command: "mage build unitTest" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" + unitTest-macos: + command: "mage build unitTest" + platform: "generic-13-ventura-x64" + provider: "orka" + unitTest-windows-2019: + command: "mage build unitTest" + platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml new file mode 100644 index 00000000000..372fbcf8caf --- /dev/null +++ b/filebeat/buildkite.yml @@ -0,0 +1,53 @@ +when: + changeset: ## when PR contains any of those entries in the changeset + - "filebeat/**" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss +stages: + # default stage - it runs always for: + # - branches/tags + # - on PRs + # - GitHub comment /test filebeat + # - GitHub label filebeat + mandatory: + # NOTE: stage name should be unique! + unitTest: + command: "mage unitTest" + platform: "family/platform-ingest-beats-ubuntu-2204" + crosscompile: + command: "make crosscompile" + platform: "family/platform-ingest-beats-ubuntu-2204" + env: + GOX_FLAGS: "-arch amd64" + goIntegTest: + command: "mage goIntegTest" + platform: "family/platform-ingest-beats-ubuntu-2204" + pythonIntegTest: + command: "mage pythonIntegTest" + platform: "family/platform-ingest-beats-ubuntu-2204" + rhel-9: + command: "mage unitTest" + platform: "family/platform-ingest-beats-rhel-9" + unitTest-windows-2022: + command: "mage build unitTest" + platform: "family/platform-ingest-beats-windows-2022" + unitTest-windows-2016: + command: "mage build unitTest" + platform: "family/platform-ingest-beats-windows-2016" + # optional stage - it runs on: + # - branches/tags + # - on PRs if: + # - GitHub comment /test filebeat . i.e: /test filebeat integTest + # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... + extended: + unitTest-arm: + command: ".buildkite/filebeat/scripts/unit-tests.sh" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" + unitTest-macos: + command: ".buildkite/filebeat/scripts/unit-tests.sh" + platform: "generic-13-ventura-x64" + provider: "orka" + unitTest-windows-2019: + command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + platform: "family/platform-ingest-beats-windows-2019"