Merge branch 'main' into CONTP-358/wassim.dhif/e2e-test-external-data

DataDog · Nov 14, 2024 · fb823f7 · fb823f7
2 parents a11b544 + e1ea1f1
commit fb823f7
Show file tree

Hide file tree

Showing 110 changed files with 4,964 additions and 1,075 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -101,7 +101,7 @@
 /.gitlab/package_deps_build/package_deps_build.yml   @DataDog/agent-devx-infra @DataDog/ebpf-platform
 /.gitlab/powershell_script_signing/powershell_script_signing.yml    @DataDog/agent-delivery @DataDog/windows-agent
 /.gitlab/source_test/golang_deps_diff.yml            @DataDog/agent-devx-infra @DataDog/agent-devx-loops
-/.gitlab/source_test/include.yml                     @DataDog/agent-devx-infra
+/.gitlab/source_test/*                               @DataDog/agent-devx-infra
 /.gitlab/source_test/linux.yml                       @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/source_test/macos.yml                       @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/source_test/notify.yml                      @DataDog/agent-devx-infra @DataDog/agent-devx-loops
@@ -614,7 +614,7 @@
 /test/new-e2e/tests/otel                      @DataDog/opentelemetry
 /test/new-e2e/tests/process                   @DataDog/processes
 /test/new-e2e/tests/sysprobe-functional     @DataDog/windows-kernel-integrations
-/test/new-e2e/tests/security-agent-functional     @DataDog/windows-kernel-integrations
+/test/new-e2e/tests/security-agent-functional     @DataDog/windows-kernel-integrations @DataDog/agent-security
 /test/new-e2e/tests/cws                       @DataDog/agent-security
 /test/new-e2e/tests/agent-metrics-logs   @DataDog/agent-metrics-logs
 /test/new-e2e/tests/windows                   @DataDog/windows-agent @DataDog/windows-kernel-integrations

diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml
@@ -164,8 +164,8 @@ jobs:
               title: 'CWS: sync BTFHub constants',
               owner,
               repo,
-              head: "$BRANCH_NAME",
-              base: "$BASE_BRANCH",
+              head: process.env.BRANCH_NAME,
+              base: process.env.BASE_BRANCH,
               body: [
                 '### What does this PR do?',
                 'This PR syncs the BTFHub constants used by CWS',

diff --git a/.gitlab/common/macos.yml b/.gitlab/common/macos.yml
@@ -33,6 +33,11 @@
     fi
   - pyenv activate $VENV_NAME
 
+.vault_login:
+  # Point the CLI to our internal vault
+  - export VAULT_ADDR=https://vault.us1.ddbuild.io
+  - vault login -method=aws -no-print
+
 .macos_gitlab:
   before_script:
     # Selecting the current Go version

diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml
@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 6c7a0d456c6f
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: b436617374bf
diff --git a/.gitlab/e2e/e2e.yml b/.gitlab/e2e/e2e.yml
@@ -500,7 +500,7 @@ new-e2e-otel-eks-init:
   variables:
     TARGETS: ./tests/otel
     TEAM: otel
-    EXTRA_PARAMS: --run TestOTelAgentIAEKS
+    EXTRA_PARAMS: --run "TestOTelAgentIA(EKS|USTEKS)"
     E2E_INIT_ONLY: "true"
     SHOULD_RUN_IN_FLAKES_FINDER: "false"
 
@@ -519,7 +519,7 @@ new-e2e-otel-eks:
     - new-e2e-otel-eks-init
   variables:
     TARGETS: ./tests/otel
-    EXTRA_PARAMS: --run TestOTelAgentIAEKS
+    EXTRA_PARAMS: --run "TestOTelAgentIA(EKS|USTEKS)"
     TEAM: otel
     E2E_PRE_INITIALIZED: "true"
 
@@ -535,7 +535,7 @@ new-e2e-otel:
     - qa_agent_ot
   variables:
     TARGETS: ./tests/otel
-    EXTRA_PARAMS: --skip TestOTelAgentIAEKS
+    EXTRA_PARAMS: --skip "TestOTelAgentIA(EKS|USTEKS)"
     TEAM: otel
 
 .new-e2e_package_signing:

diff --git a/.gitlab/functional_test/regression_detector.yml b/.gitlab/functional_test/regression_detector.yml
@@ -18,20 +18,13 @@ single-machine-performance-regression_detector:
       - outputs/regression_signal.json # for debugging, also on S3
       - outputs/bounds_check_signal.json # for debugging, also on S3
       - outputs/junit.xml # for debugging, also on S3
+      - outputs/report.json # for debugging, also on S3
+      - outputs/decision_record.md # for posterity, this is appended to final PR comment
     when: always
   variables:
     SMP_VERSION: 0.18.2
-  # At present we require two artifacts to exist for the 'baseline' and the
-  # 'comparison'. We are guaranteed by the structure of the pipeline that
-  # 'comparison' exists, not so much with 'baseline' as it has to come from main
-  # merge pipeline run. This is solved in datadog-agent by updating a file in S3
-  # with the SHA of the merge base from main. It's solved in Vector by
-  # building Vector twice for each Regression Detector run.
-  #
-  # We allow failure for now. _Unfortunately_ this also means that if the
-  # Regression Detector finds a performance issue with a PR it will not be
-  # flagged.
-  allow_failure: true
+  # See 'decision_record.md' for the determination of whether this job passes or fails.
+  allow_failure: false
   script:
     # Ensure output files exist for artifact downloads step
     - mkdir outputs # Also needed for smp job sync step
@@ -129,33 +122,88 @@ single-machine-performance-regression_detector:
     # uploading JUnit XML, so the upload command below respects that convention.
     - DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2" token)" || exit $?; export DATADOG_API_KEY
     - datadog-ci junit upload --service datadog-agent outputs/junit.xml
-    # Finally, exit 1 if the job signals a regression else 0.
-    - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE}
-      job result
-      --submission-metadata submission_metadata
+    # Run quality gate check script
+    - |
+      python3 <<'EOF'
+      import json
+      import sys
+
+      try:
+          with open('outputs/report.json') as f:
+              data = json.load(f)
+      except FileNotFoundError:
+          print("Machine readable report not found.")
+          sys.exit(1)
+      except json.JSONDecodeError as e:
+          print(f"Error parsing JSON report: {e}")
+          sys.exit(1)
+
+      experiments = data.get('experiments', {})
+      failed = False
+      decision_record = []
+
+      for exp_name, exp_data in experiments.items():
+          if exp_name.startswith('quality_gate_'):
+              bounds_checks = exp_data.get('bounds_checks', {})
+              for check_name, check_data in bounds_checks.items():
+                  results = check_data.get('results', {})
+                  comparison = results.get('comparison', [])
+                  num_total = len(comparison)
+                  failed_replicates = [
+                      replicate for replicate in comparison if not replicate.get('passed', False)
+                  ]
+                  num_failed = len(failed_replicates)
+                  num_passed = num_total - num_failed
+                  if failed_replicates:
+                      decision_record.append(
+                          f"- **{exp_name}**, bounds check **{check_name}**: {num_passed}/{num_total} replicas passed. Failed {num_failed} which is > 0. Gate **FAILED**."
+                      )
+                      failed = True
+                  else:
+                      decision_record.append(
+                          f"- **{exp_name}**, bounds check **{check_name}**: {num_passed}/{num_total} replicas passed. Gate passed."
+                      )
+
+      with open('outputs/decision_record.md', 'w') as f:
+          # Extra newline since this is appended to another report
+          f.write('\n\n## CI Pass/Fail Decision\n\n')
+          if failed:
+              f.write('❌ **Failed.** Some Quality Gates were violated.\n\n')
+              f.write('\n'.join(decision_record))
+          else:
+              f.write('✅ **Passed.** All Quality Gates passed.\n\n')
+              f.write('\n'.join(decision_record))
+
+      if failed:
+          print("Quality gate failed, see decision record")
+          sys.exit(1)
+      else:
+          print("Quality gate passed.")
+          sys.exit(0)
+      EOF
 
 # Shamelessly adapted from golang_deps_commenter job config in
 # golang_deps_diff.yml at commit 01da274032e510d617161cf4e264a53292f44e55.
 single-machine-performance-regression_detector-pr-comment:
   stage: functional_test
   rules:
     - !reference [.except_main_or_release_branch]
-    - when: on_success
+    - when: always
   image:
     name: "486234852809.dkr.ecr.us-east-1.amazonaws.com/pr-commenter:3"
-    entrypoint: [""]  # disable entrypoint script for the pr-commenter image
+    entrypoint: [""] # disable entrypoint script for the pr-commenter image
   tags: ["arch:amd64"]
   needs:
     - job: single-machine-performance-regression_detector
   artifacts:
     expire_in: 1 weeks
     paths:
       - report_as_json_string.txt # for debugging transform to valid JSON string
-      - pr_comment_payload.json  # for debugging PR commenter JSON payload bugs
+      - pr_comment_payload.json # for debugging PR commenter JSON payload bugs
   variables:
     # Not using the entrypoint script for the pr-commenter image
     FF_KUBERNETES_HONOR_ENTRYPOINT: false
-  allow_failure: true  # allow_failure here should have same setting as in job above
+  allow_failure: true # allow_failure here should have same setting as in job above
   script: # ignore error message about no PR, because it happens for dev branches without PRs
     # Prevent posting empty Regression Detector report if Markdown report is not found or
     # has zero size.
@@ -176,7 +224,8 @@ single-machine-performance-regression_detector-pr-comment:
     # to escape double quotes to distinguish literal quotes in the report from
     # the double quotes that delimit the value of the "message" field in the
     # payload.
-    - cat outputs/report.md | sed -z 's/\n/\\n/g' | sed -z 's/"/\\"/g' > report_as_json_string.txt
+    # Appends the Decision Record to final report
+    - cat outputs/report.md outputs/decision_record.md | sed -z 's/\n/\\n/g' | sed -z 's/"/\\"/g' > report_as_json_string.txt
     - cat report_as_json_string.txt
     # Transforming the Markdown report to a valid JSON string is easy to foul
     # up, so to make debugging easier, we store the payload in a variable to

diff --git a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml
@@ -67,8 +67,5 @@ notify-slack:
   needs: ["internal_kubernetes_deploy_experimental"]
   script:
     - export SDM_JWT=$(vault read -field=token identity/oidc/token/sdm)
-    # Python 3.12 changes default behavior how packages are installed.
-    # In particular, --break-system-packages command line option is
-    # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615
-    - python3 -m pip install -r tasks/requirements.txt --break-system-packages
+    - python3 -m pip install -r tasks/requirements.txt
     - inv pipeline.changelog ${CI_COMMIT_SHORT_SHA} || exit $?
diff --git a/.gitlab/package_build/linux.yml b/.gitlab/package_build/linux.yml
@@ -80,36 +80,6 @@
   before_script:
     - export RELEASE_VERSION=$RELEASE_VERSION_7
 
-# Temporary custom agent build test to prevent regression
-# This test will be removed when custom path are used to build macos agent
-# with in-house macos runner builds.
-datadog-agent-7-x64-custom-path-test:
-  extends: [.agent_build_x86, .agent_7_build]
-  rules:
-    - !reference [.except_mergequeue]
-    - when: on_success
-  stage: package_build
-  script:
-    - mkdir /custom
-    - export CONFIG_DIR="/custom"
-    - export INSTALL_DIR="/custom/datadog-agent"
-    - !reference [.agent_build_script]
-    - ls -la $OMNIBUS_PACKAGE_DIR
-    - ls -la $INSTALL_DIR
-    - ls -la /custom/etc
-    - (ls -la /opt/datadog-agent 2>/dev/null && exit 1) || echo "/opt/datadog-agent has correctly not been generated"
-    - (ls -la /etc/datadog-agent 2>/dev/null && exit 1) || echo "/etc/datadog-agent has correctly not been generated"
-  variables:
-    KUBERNETES_CPU_REQUEST: 16
-    KUBERNETES_MEMORY_REQUEST: "32Gi"
-    KUBERNETES_MEMORY_LIMIT: "32Gi"
-  artifacts:
-    expire_in: 2 weeks
-    paths:
-      - $OMNIBUS_PACKAGE_DIR
-  cache:
-    - !reference [.cache_omnibus_ruby_deps, cache]
-
 # build Agent 7 binaries for x86_64
 datadog-agent-7-x64:
   extends: [.agent_build_common, .agent_build_x86, .agent_7_build]

diff --git a/.gitlab/source_test/common.yml b/.gitlab/source_test/common.yml
@@ -0,0 +1,8 @@
+---
+.upload_junit_source:
+  - $CI_PROJECT_DIR/tools/ci/junit_upload.sh
+
+.upload_coverage:
+  # Upload coverage files to Codecov. Never fail on coverage upload.
+  - CODECOV_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $CODECOV token) || exit $?; export CODECOV_TOKEN
+  - inv -e coverage.upload-to-codecov $COVERAGE_CACHE_FLAG || true
diff --git a/.gitlab/source_test/include.yml b/.gitlab/source_test/include.yml
@@ -4,6 +4,7 @@
 # security scans & go.mod checks.
 
 include:
+  - .gitlab/source_test/common.yml # Included first for shared definitions
   - .gitlab/source_test/ebpf.yml
   - .gitlab/source_test/linux.yml
   - .gitlab/source_test/macos.yml

diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml
@@ -45,14 +45,6 @@
       annotations:
         - $EXTERNAL_LINKS_PATH
 
-.upload_junit_source:
-  - $CI_PROJECT_DIR/tools/ci/junit_upload.sh
-
-.upload_coverage:
-  # Upload coverage files to Codecov. Never fail on coverage upload.
-  - CODECOV_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $CODECOV token) || exit $?; export CODECOV_TOKEN
-  - inv -e coverage.upload-to-codecov $COVERAGE_CACHE_FLAG || true
-
 .linux_x64:
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]

diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml
@@ -64,18 +64,12 @@ tests_macos:
       annotations:
         - $EXTERNAL_LINKS_PATH
 
-.upload_junit_source:
-  - $CI_PROJECT_DIR/tools/ci/junit_upload.sh
-
-.upload_coverage:
-  # Upload coverage files to Codecov. Never fail on coverage upload.
-  - CODECOV_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $CODECOV_TOKEN) || exit $?; export CODECOV_TOKEN
-  - inv -e coverage.upload-to-codecov $COVERAGE_CACHE_FLAG || true
-
 tests_macos_gitlab_amd64:
   extends: .tests_macos_gitlab
   tags: ["macos:monterey-amd64", "specific:true"]
   after_script:
+    - !reference [.vault_login]
+    - !reference [.select_python_env_commands]
     - !reference [.upload_junit_source]
     - !reference [.upload_coverage]
 
@@ -85,5 +79,7 @@ tests_macos_gitlab_arm64:
     !reference [.manual]
   tags: ["macos:monterey-arm64", "specific:true"]
   after_script:
+    - !reference [.vault_login]
+    - !reference [.select_python_env_commands]
     - !reference [.upload_junit_source]
     - !reference [.upload_coverage]
diff --git a/cmd/agent/subcommands/jmx/command.go b/cmd/agent/subcommands/jmx/command.go
@@ -39,6 +39,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration"
 	"github.com/DataDog/datadog-agent/comp/core/config"
 	log "github.com/DataDog/datadog-agent/comp/core/log/def"
+	remoteagentregistry "github.com/DataDog/datadog-agent/comp/core/remoteagentregistry/def"
 	"github.com/DataDog/datadog-agent/comp/core/secrets"
 	"github.com/DataDog/datadog-agent/comp/core/settings"
 	"github.com/DataDog/datadog-agent/comp/core/settings/settingsimpl"
@@ -165,6 +166,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 			fx.Invoke(func(wmeta workloadmeta.Component, tagger tagger.Component) {
 				proccontainers.InitSharedContainerProvider(wmeta, tagger)
 			}),
+			fx.Provide(func() remoteagentregistry.Component { return nil }),
 		)
 	}
 

diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go
@@ -66,6 +66,7 @@ import (
 	healthprobe "github.com/DataDog/datadog-agent/comp/core/healthprobe/def"
 	healthprobefx "github.com/DataDog/datadog-agent/comp/core/healthprobe/fx"
 	lsof "github.com/DataDog/datadog-agent/comp/core/lsof/fx"
+	remoteagentregistryfx "github.com/DataDog/datadog-agent/comp/core/remoteagentregistry/fx"
 	"github.com/DataDog/datadog-agent/comp/core/secrets"
 	"github.com/DataDog/datadog-agent/comp/core/settings"
 	"github.com/DataDog/datadog-agent/comp/core/settings/settingsimpl"
@@ -470,6 +471,7 @@ func getSharedFxOption() fx.Option {
 		settingsimpl.Module(),
 		agenttelemetryfx.Module(),
 		networkpath.Bundle(),
+		remoteagentregistryfx.Module(),
 	)
 }
 

diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt
@@ -79,6 +79,7 @@ github.com/DataDog/datadog-agent/comp/core/tagger/common
 github.com/DataDog/datadog-agent/comp/core/tagger/def
 github.com/DataDog/datadog-agent/comp/core/tagger/fx-noop
 github.com/DataDog/datadog-agent/comp/core/tagger/impl-noop
+github.com/DataDog/datadog-agent/comp/core/tagger/tags
 github.com/DataDog/datadog-agent/comp/core/tagger/telemetry
 github.com/DataDog/datadog-agent/comp/core/tagger/types
 github.com/DataDog/datadog-agent/comp/core/tagger/utils

diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt
@@ -79,6 +79,7 @@ github.com/DataDog/datadog-agent/comp/core/tagger/common
 github.com/DataDog/datadog-agent/comp/core/tagger/def
 github.com/DataDog/datadog-agent/comp/core/tagger/fx-noop
 github.com/DataDog/datadog-agent/comp/core/tagger/impl-noop
+github.com/DataDog/datadog-agent/comp/core/tagger/tags
 github.com/DataDog/datadog-agent/comp/core/tagger/telemetry
 github.com/DataDog/datadog-agent/comp/core/tagger/types
 github.com/DataDog/datadog-agent/comp/core/tagger/utils

diff --git a/comp/README.md b/comp/README.md
@@ -159,6 +159,11 @@ Package lsof provides a flare file with data about files opened by the agent pro
 Package pid writes the current PID to a file, ensuring that the file
 doesn't exist or doesn't contain a PID for a running process.
 
+### [comp/core/remoteagentregistry](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/core/remoteagentregistry)
+
+Package remoteagentregistry provides an integration point for remote agents to register and be able to report their
+status and emit flare data
+
 ### [comp/core/secrets](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/core/secrets)
 
 Package secrets decodes secret values by invoking the configured executable command