Merge branch 'main' into stanley.liu/fix-otel-logs-hostname

.copyright-overrides.yml

@@ -216,6 +216,7 @@ github.com/gogo/protobuf: ["Copyright (c) 2013, The GoGo Authors. All rights res
 github.com/moby/sys/mountinfo: Copyright (c) 2014-2018 The Docker & Go Authors. All rights reserved.
 github.com/moby/sys/signal: Copyright (c) 2014-2018 The Docker & Go Authors. All rights reserved.
 github.com/moby/sys/user: Copyright (c) 2014-2018 The Docker & Go Authors. All rights reserved.
+github.com/moby/sys/userns: Copyright (c) 2014-2018 The Docker & Go Authors. All rights reserved.
 github.com/modern-go/concurrent: Copyright (c) 2018 Tao Wen
 github.com/modern-go/reflect2: Copyright (c) 2018 Tao Wen
 github.com/opencontainers/selinux/*: Copyright (c) 2017 The Authors
@@ -247,6 +248,7 @@ github.com/chrusty/protoc-gen-jsonschema: Copyright (c) 2017 The Authors
 github.com/antchfx/htmlquery: Copyright (c) 2016 Zheng Chun
 github.com/antchfx/xmlquery: Copyright (c) 2016 Zheng Chun
 github.com/antchfx/xpath: Copyright (c) 2016 Zheng Chun
+github.com/kouhin/envflag: Copyright (c) 2015 kouhin
 
 
 # The Copyright information is not contained in the LICENSE file, but it can be found in other

.github/CODEOWNERS

@@ -92,12 +92,14 @@
 
 /.gitlab/kitchen_*                                   @DataDog/agent-delivery
 /.gitlab/kitchen_testing/windows.yml                 @DataDog/agent-delivery @DataDog/windows-agent
-/.gitlab/kitchen_testing/new-e2e_testing/windows.yml @DataDog/agent-delivery @DataDog/windows-agent
+/.gitlab/e2e_install_packages/windows.yml            @DataDog/agent-delivery @DataDog/windows-agent
 
 /.gitlab/common/                                     @DataDog/agent-devx-infra
 /.gitlab/common/test_infra_version.yml               @DataDog/agent-devx-loops @DataDog/agent-devx-infra
 
 /.gitlab/e2e/e2e.yml                                 @DataDog/container-integrations @DataDog/agent-devx-loops
+/.gitlab/e2e/k8s_e2e.yml                             @DataDog/container-integrations @DataDog/agent-devx-loops
+/.gitlab/e2e/install_packages                        @DataDog/agent-delivery
 /.gitlab/container_build/fakeintake.yml              @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /.gitlab/binary_build/fakeintake.yml                 @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 
@@ -106,6 +108,9 @@
 /.gitlab/functional_test_cleanup.yml                 @DataDog/agent-security @DataDog/windows-kernel-integrations @DataDog/agent-devx-infra
 /.gitlab/functional_test/system_probe_windows.yml    @DataDog/agent-devx-infra @DataDog/windows-kernel-integrations
 /.gitlab/functional_test/common.yml                  @DataDog/agent-devx-infra @DataDog/windows-kernel-integrations
+/.gitlab/functional_test/oracle.yml                  @DataDog/agent-devx-infra @DataDog/database-monitoring
+
+
 /.gitlab/choco_build/choco_build.yml                 @DataDog/agent-delivery @DataDog/windows-agent
 /.gitlab/choco_deploy/choco_deploy.yml               @DataDog/agent-delivery @DataDog/windows-agent
 
@@ -586,6 +591,8 @@
 /test/fakeintake/                             @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/network-device-monitoring
 /test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/network-device-monitoring
+/test/fakeintake/aggregator/servicediscoveryAggregator.go @DataDog/apm-onboarding @DataDog/universal-service-monitoring
+/test/fakeintake/aggregator/servicediscoveryAggregator_test.go @DataDog/apm-onboarding @DataDog/universal-service-monitoring
 /test/new-e2e/                                @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/new-e2e/test-infra-definition           @DataDog/agent-devx-loops
 /test/new-e2e/system-probe                    @DataDog/ebpf-platform

.github/workflows/create_rc_pr.yml

@@ -14,6 +14,7 @@ jobs:
         runs-on: ubuntu-latest
         outputs:
           branches: ${{ steps.branches.outputs.value }}
+          warning: ${{ steps.warning.outputs.value }}
         steps:
             - name: Checkout repository
               uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -39,8 +40,10 @@ jobs:
                 echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT
 
             - name: Set the warning option
+              id: warning
               if: github.event.schedule == '0 8 * * 1,3,5'
-              run: echo "WARNING='-w'" >> $GITHUB_ENV
+              run: |
+                echo "value=-w" >> $GITHUB_OUTPUT
 
     create_rc_pr:
       runs-on: ubuntu-latest
@@ -74,7 +77,8 @@ jobs:
                 ATLASSIAN_USERNAME: ${{ secrets.ATLASSIAN_USERNAME }}
                 ATLASSIAN_PASSWORD: ${{ secrets.ATLASSIAN_PASSWORD }}
               run: |
-                echo "CHANGES=$(inv -e release.check-for-changes -r ${{ matrix.value }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT
+                export SLACK_API_TOKEN="${{ secrets.SLACK_API_TOKEN }}"
+                echo "CHANGES=$(inv -e release.check-for-changes -r ${{ matrix.value }} ${{ needs.find_release_branches.outputs.warning }})" >> $GITHUB_OUTPUT
 
             - name: Create RC PR
               if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}}

.github/workflows/serverless-binary-size.yml

@@ -138,28 +138,23 @@ jobs:
           name: dependency-graphs
           path: go/src/github.com/DataDog/datadog-lambda-extension/graphs
 
+      - name: Write message
+        id: write
+        if: steps.should.outputs.should_run == 'true'
+        env:
+          VAR_COLD_START: ${{ steps.compare.outputs.coldstart }}
+          VAR_DIFF: ${{ steps.compare.outputs.diff }}
+          VAR_DEPS: ${{ steps.deps.outputs.deps }}
+          VAR_RUN_ID: ${{ github.run_id }}
+        run: |
+          cd go/src/github.com/DataDog/datadog-agent
+          ./test/integration/serverless_perf/write_message.sh
+
       - name: Post comment
         uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2.9.0
         if: steps.should.outputs.should_run == 'true'
         with:
           header: serverless-binary-size
           hide_and_recreate: true
           hide_classify: "RESOLVED"
-          message: |
-            :warning::rotating_light: Warning, this pull request increases the binary size of serverless extension by ${{ steps.compare.outputs.diff }} bytes. Each MB of binary size increase means about 10ms of additional cold start time, so this pull request would increase cold start time by ${{ steps.compare.outputs.coldstart }}ms.
-
-            If you have questions, we are happy to help, come visit us in the [#serverless](https://dd.slack.com/archives/CBWDFKWV8) slack channel and provide a link to this comment.
-
-            <details>
-            <summary>Debug info</summary>
-
-            These dependencies were added to the serverless extension by this pull request:
-
-            ```
-            ${{ steps.deps.outputs.deps }}
-            ```
-
-            View dependency graphs for each added dependency in the [artifacts section](https://github.com/DataDog/datadog-agent/actions/runs/${{ github.run_id }}#artifacts) of the github action.
-
-            We suggest you consider adding the `!serverless` build tag to remove any new dependencies not needed in the serverless extension.
-            </details>
+          path: ${{ steps.write.outputs.filename }}

.gitlab-ci.yml

@@ -22,6 +22,8 @@ include:
   - .gitlab/deps_fetch/deps_fetch.yml
   - .gitlab/dev_container_deploy/include.yml
   - .gitlab/e2e/e2e.yml
+  - .gitlab/e2e_k8s/e2e_k8s.yml
+  - .gitlab/e2e_install_packages/include.yml
   - .gitlab/e2e_pre_test/e2e_pre_test.yml
   - .gitlab/functional_test/include.yml
   - .gitlab/functional_test_cleanup/functional_test_cleanup.yml
@@ -93,6 +95,8 @@ stages:
   - install_script_testing
   - e2e_pre_test
   - e2e
+  - e2e_k8s
+  - e2e_install_packages
   - kitchen_cleanup
   - functional_test
   - functional_test_cleanup
@@ -718,7 +722,7 @@ workflow:
   changes:
       paths:
         - tools/windows/DatadogAgentInstaller/**/*
-        - .gitlab/kitchen_testing/new-e2e_testing/windows.yml
+        - .gitlab/e2e_install_packages/windows.yml
         - test/new-e2e/tests/windows/install-test/**/*
         - test/new-e2e/tests/windows/domain-test/**/*
         - tasks/msi.py
@@ -735,9 +739,6 @@ workflow:
   - pkg/network/**/*
   - pkg/process/monitor/*
   - pkg/util/kernel/**/*
-  - test/kitchen/site-cookbooks/dd-system-probe-check/**/*
-  - test/kitchen/test/integration/win-sysprobe-test/**/*
-  - .gitlab/functional_test/system_probe_windows.yml
   - .gitlab/kernel_matrix_testing/system_probe.yml
   - .gitlab/kernel_matrix_testing/common.yml
   - test/new-e2e/system-probe/**/*
@@ -1026,6 +1027,28 @@ workflow:
   - when: manual
     allow_failure: true
 
+.on_windows_systemprobe_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - pkg/collector/corechecks/servicediscovery/module/*
+        - pkg/network/**/*
+        - pkg/process/monitor/*
+        - pkg/util/kernel/**/*
+        - test/new-e2e/tests/sysprobe-functional/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+    when: on_success
+
+.on_windows_security_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - pkg/security/**/*
+        - pkg/eventmonitor/**/*
+        - test/new-e2e/tests/security-agent-functional/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+    when: on_success
+
 .on_scheduled_main:
   - <<: *if_scheduled_main
 

.gitlab/JOBOWNERS

@@ -126,6 +126,7 @@ kitchen_*_security_agent*                      @DataDog/agent-security
 kitchen_*_process_agent*                       @DataDog/processes
 cleanup_kitchen_functional_test                @DataDog/windows-kernel-integrations @DataDog/agent-security
 serverless_cold_start_performance-deb_x64      @DataDog/serverless
+oracle*                                        @DataDog/database-monitoring
 
 # E2E
 k8s-e2e-*                         @DataDog/container-integrations

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0f82f8cbf8ea
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: c9ee795ec752

.gitlab/e2e/e2e.yml

@@ -1,84 +1,6 @@
 ---
 # e2e stage
-# Contains jobs which runs e2e tests on our Docker images.
-
-.k8s_e2e_template:
-  stage: e2e
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
-  tags: ["arch:amd64"]
-  dependencies: []
-  variables:
-    LANG: C.UTF-8
-  before_script:
-    - export DOCKER_REGISTRY_LOGIN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $DOCKER_REGISTRY_LOGIN_SSM_KEY)
-    - export DOCKER_REGISTRY_PWD=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $DOCKER_REGISTRY_PWD_SSM_KEY)
-    - export DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $API_KEY_DDDEV_SSM_NAME)
-
-.k8s-e2e-cws-cspm-init:
-  - set +x
-  - export DATADOG_AGENT_SITE=datadoghq.com
-  - export DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $E2E_TESTS_API_KEY_SSM_NAME)
-  - export DATADOG_AGENT_APP_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $E2E_TESTS_APP_KEY_SSM_NAME)
-  - export DATADOG_AGENT_RC_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $E2E_TESTS_RC_KEY_SSM_NAME)
-
-.k8s_e2e_template_needs_dev:
-  extends: .k8s_e2e_template
-  needs:
-    - dev_branch_multiarch-a7
-    - dca_dev_branch
-
-.k8s_e2e_template_dev:
-  extends: .k8s_e2e_template_needs_dev
-  script:
-    - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=$ARGO_WORKFLOW
-
-.k8s_e2e_template_dev_with_cws_cspm_init:
-  extends: .k8s_e2e_template_needs_dev
-  script:
-    - !reference [.k8s-e2e-cws-cspm-init]
-    - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=$ARGO_WORKFLOW
-
-.k8s_e2e_template_needs_main:
-  extends: .k8s_e2e_template
-  needs:
-    - dev_master-a7
-    - dca_dev_master
-
-.k8s_e2e_template_main_with_cws_cspm_init:
-  extends: .k8s_e2e_template_needs_main
-  script:
-    - !reference [.k8s-e2e-cws-cspm-init]
-    - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py3 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=$ARGO_WORKFLOW
-
-.k8s_e2e_template_main:
-  extends: .k8s_e2e_template_needs_main
-  script:
-    - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py3 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=$ARGO_WORKFLOW
-
-k8s-e2e-cspm-dev:
-  extends: .k8s_e2e_template_dev_with_cws_cspm_init
-  rules: !reference [.on_dev_branch_manual]
-  variables:
-    ARGO_WORKFLOW: cspm
-
-k8s-e2e-cspm-main:
-  extends: .k8s_e2e_template_main_with_cws_cspm_init
-  rules: !reference [.on_main]
-  retry: 1
-  variables:
-    ARGO_WORKFLOW: cspm
-
-k8s-e2e-otlp-dev:
-  extends: .k8s_e2e_template_dev
-  rules: !reference [.on_dev_branch_manual]
-  variables:
-    ARGO_WORKFLOW: otlp
-
-k8s-e2e-otlp-main:
-  extends: .k8s_e2e_template_main
-  rules: !reference [.on_main]
-  variables:
-    ARGO_WORKFLOW: otlp
+# Contains test jobs based on the new-e2e tests framework
 
 .new_e2e_template:
   stage: e2e
@@ -446,6 +368,32 @@ new-e2e-ndm-snmp:
     TARGETS: ./tests/ndm/snmp
     TEAM: network-device-monitoring
 
+new-e2e-windows-systemprobe:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_windows_systemprobe_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - !reference [.needs_new_e2e_template]
+    - deploy_windows_testing-a7
+    - tests_windows_sysprobe_x64
+  variables:
+    TARGETS: ./tests/sysprobe-functional
+    TEAM: windows-kernel-integrations
+
+new-e2e-windows-security-agent:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_windows_security_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - !reference [.needs_new_e2e_template]
+    - deploy_windows_testing-a7
+    - tests_windows_secagent_x64
+  variables:
+    TARGETS: ./tests/security-agent-functional
+    TEAM: windows-kernel-integrations
+
 new-e2e-otel:
   extends: .new_e2e_template
   rules:
@@ -460,6 +408,33 @@ new-e2e-otel:
     TARGETS: ./tests/otel
     TEAM: otel
 
+.new-e2e_package_signing:
+  variables:
+    TARGETS: ./tests/agent-platform/package-signing
+    TEAM: agent-delivery
+    EXTRA_PARAMS: --osversion $E2E_BRANCH_OSVERS
+
+new-e2e-package-signing-amazonlinux-a6-x86_64:
+  extends:
+    - .new_e2e_template
+    - .new-e2e_amazonlinux_a7_x86_64
+    - .new-e2e_package_signing
+  rules: !reference [.on_default_new_e2e_tests]
+
+new-e2e-package-signing-debian-a7-x86_64:
+  extends:
+    - .new_e2e_template
+    - .new-e2e_debian_a7_x86_64
+    - .new-e2e_package_signing
+  rules: !reference [.on_default_new_e2e_tests]
+
+new-e2e-package-signing-suse-a7-x86_64:
+  extends:
+    - .new_e2e_template
+    - .new-e2e_suse_a7_x86_64
+    - .new-e2e_package_signing
+  rules: !reference [.on_default_new_e2e_tests]
+
 generate-flakes-finder-pipeline:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   stage: e2e