[v2] Merge main into feature-arch-v2 as of Nov 8

.ci/Jenkinsfile

@@ -425,7 +425,7 @@ pipeline {
     cleanup {
       notifyBuildResult(prComment: true,
                         analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: (isPR() ? env.CHANGE_TARGET : env.BRANCH_NAME)),
-                        githubIssue: isBranch() && currentBuild.currentResult != "SUCCESS",
+                        githubIssue: false, // Disable creating gh issues for build failures while the E2E tests are stabilized.
                         githubLabels: 'Team:Elastic-Agent-Control-Plane')
     }
   }

.github/stale.yml

@@ -0,0 +1,91 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 90
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - flaky-test
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: true
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: true
+
+# Label to use when marking as stale
+staleLabel: Stalled
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  Hi!
+
+  We just realized that we haven't looked into this issue in a while. We're
+  sorry!
+
+
+  We're labeling this issue as `Stale` to make it hit our filters and
+  make sure we get back to it as soon as possible. In the meantime, it'd
+  be extremely helpful if you could take a look at it as well and confirm its
+  relevance. A simple comment with a nice emoji will be enough `:+1`.
+
+  Thank you for your contribution!
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+pulls:
+  daysUntilStale: 60
+  daysUntilClose: 30
+  markComment: >
+    Hi!
+
+    We just realized that we haven't looked into this PR in a while. We're
+    sorry!
+
+
+    We're labeling this issue as `Stale` to make it hit our filters and
+    make sure we get back to it as soon as possible. In the meantime, it'd
+    be extremely helpful if you could take a look at it as well and confirm its
+    relevance. A simple comment with a nice emoji will be enough `:+1`.
+
+    Thank you for your contribution!
+
+  closeComment: >
+    Hi!
+
+    This PR has been stale for a while and we're going to close it as part of
+    our cleanup procedure.
+
+    We appreciate your contribution and would like to apologize if we have not
+    been able to review it, due to the current heavy load of the team.
+
+    Feel free to re-open this PR if you think it should stay open and is worth rebasing.
+
+    Thank you for your contribution!
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/add-issues-to-ingest-board.yml

@@ -0,0 +1,53 @@
+name: Add issue to Platform Ingest project
+
+on:
+  issues:
+    types:
+      - labeled
+
+env:
+  INGEST_PROJECT_ID: 'PVT_kwDOAGc3Zs4AEzn4'
+  DATA_PLANE_LABEL: 'Team:Elastic-Agent-Data-Plane'
+  CONTROL_PLANE_LABEL: 'Team:Elastic-Agent-Control-Plane'
+  ELASTIC_AGENT_LABEL: 'Team:Elastic-Agent'
+  AREA_FIELD_ID: 'PVTSSF_lADOAGc3Zs4AEzn4zgEgZSo'
+  ELASTIC_AGENT_OPTION_ID: 'c1e1a30a'
+
+jobs:
+  add_to_ingest_project:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: octokit/[email protected]
+        id: add_to_project
+        if: github.event.label.name == env.DATA_PLANE_LABEL || github.event.label.name == env.CONTROL_PLANE_LABEL || github.event.label.name == env.ELASTIC_AGENT_LABEL
+        with:
+          query: |
+            # Variables have to be snake cased because of https://github.com/octokit/graphql-action/issues/164
+            mutation AddToIngestProject($project_id: ID!, $content_id: ID!) {
+              addProjectV2ItemById(input: { projectId: $project_id, contentId: $content_id }) {
+                  item {
+                    id
+                  }
+                }
+              }
+          project_id: ${{ env.INGEST_PROJECT_ID }}
+          content_id: ${{ github.event.issue.node_id }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.PROJECT_ASSIGNER_TOKEN }}
+      - uses: octokit/[email protected]
+        id: set_elastic_agent_area
+        if: github.event.label.name == env.DATA_PLANE_LABEL || github.event.label.name == env.CONTROL_PLANE_LABEL || github.event.label.name == env.ELASTIC_AGENT_LABEL
+        with:
+          query: |
+            mutation updateIngestArea($item_id: ID!, $project_id: ID!, $area_field_id: ID!, $area_id: String) {
+              updateProjectV2ItemFieldValue(
+                input: { itemId: $item_id, projectId: $project_id, fieldId: $area_field_id, value: { singleSelectOptionId: $area_id } }) {
+                  clientMutationId
+                }
+              }
+          item_id: ${{ fromJSON(steps.add_to_project.outputs.data).addProjectV2ItemById.item.id }}
+          project_id: ${{ env.INGEST_PROJECT_ID }}
+          area_field_id: ${{ env.AREA_FIELD_ID }}
+          area_id: ${{ env.ELASTIC_AGENT_OPTION_ID }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.PROJECT_ASSIGNER_TOKEN }}

.gitignore

@@ -45,6 +45,7 @@ fleet.enc.lock
 # Files generated with the bump version automations
 *.bck
 
+
 # agent
 build/
 elastic-agent

.go-version

@@ -1 +1 @@
-1.18.7
+1.18.8

README.md

@@ -5,6 +5,24 @@
 The source files for the general Elastic Agent documentation are currently stored
 in the [observability-docs](https://github.com/elastic/observability-docs) repo. The following docs are only focused on getting developers started building code for Elastic Agent.
 
+### Changelog
+
+The changelog for the Elastic Agent is generated and maintained using the [elastic-agent-changelog-tool](https://github.com/elastic/elastic-agent-changelog-tool). Read the [installation](https://github.com/elastic/elastic-agent-changelog-tool/blob/main/docs/install.md)
+and [usage](https://github.com/elastic/elastic-agent-changelog-tool/blob/main/docs/usage.md#im-a-developer) instructions to get started.
+
+The changelog tool produces fragement files that are consolidated to generate a changelog for each release. Each PR containing a change with user
+impact (new feature, bug fix, etc.) must contain a changelog fragement describing the change. There is a GitHub action in CI that will fail
+if a PR does not contain a changelog fragment. For PRs that should not have a changelog entry, use the "skip-changelog" label to bypass
+this check.
+
+A simple example of a changelog fragment is below for reference:
+
+```yml
+kind: bug-fix
+summary: Fix a panic caused by a race condition when installing the Elastic Agent.
+pr: https://github.com/elastic/elastic-agent/pull/823
+```
+
 ## Testing
 
 Prerequisites:
@@ -27,6 +45,13 @@ export COMPOSE_DOCKER_CLI_BUILD=0
 export DOCKER_DEFAULT_PLATFORM=linux/amd64
 ```
 
+If you are on a Mac with M1 chip, don't forget to export some docker variable to be able to build for AMD
+```
+export DOCKER_BUILDKIT=0
+export COMPOSE_DOCKER_CLI_BUILD=0
+export DOCKER_DEFAULT_PLATFORM=linux/amd64
+```
+
 In Linux operating systems that you can not run docker as a root user you need to follow [linux-postinstall steps](https://docs.docker.com/engine/install/linux-postinstall/)
 
 ### Testing docker container

changelog/8.5.0.yaml

@@ -0,0 +1,129 @@
+version: 8.5.0
+entries:
+    - kind: bug-fix
+      summary: Fix a panic caused by a race condition when installing the Elastic Agent.
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/823
+      issue:
+        - https://github.com/elastic/elastic-agent/issues/806
+      timestamp: 1660139385
+      file:
+        name: 1660139385-Fix-a-panic-caused-by-a-race-condition-when-installing-the-Elastic-Agent.yaml
+        checksum: be820e85b12290a895bfd37c2c2245bfae5a70c9
+    - kind: breaking-change
+      summary: Upgrade to Go 1.18. Certificates signed with SHA-1 are now rejected. See the Go 1.18 https//tip.golang.org/doc/go1.18#sha1[release notes] for details.
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/832
+      issue: []
+      timestamp: 1660158319
+      file:
+        name: 1660158319-Upgrade-to-Go-118-Certificates-signed-with-SHA-1-are-now-rejected-See-the-Go-118.yaml
+        checksum: 2b304d75a687ec7384f3011a55f243fef66e447b
+    - kind: feature
+      summary: Add `lumberjack` input type to the Filebeat spec.
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/959
+      issue: []
+      timestamp: 1661188787
+      file:
+        name: 1661188787-Add-lumberjack-input-type-to-the-Filebeat-spec.yaml
+        checksum: ab7e3af045affbda1522c029cc56a9c5403340c3
+    - kind: feature
+      summary: Add support for hints' based autodiscovery in kubernetes provider.
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/698
+      issue: []
+      timestamp: 1663143487
+      file:
+        name: 1663143487-Add-support-for-hints-based-autodiscovery-in-kubernetes-provider.yaml
+        checksum: 398470d14a8475e93aaf66f96407570653dde71e
+    - kind: bug-fix
+      summary: Fix unintended reset of source URI when downloading components
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/1252
+      issue: []
+      timestamp: 1664177394
+      file:
+        name: 1664177394-Fix-unintended-reset-of-source-URI-when-downloading-components.yaml
+        checksum: 1040aceb00b70182c0bba621e15cfe711e32f9fe
+    - kind: bug-fix
+      summary: Create separate status reporter for local only events so that degraded fleet-checkins no longer affect health on successful fleet-checkins.
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/1285
+      issue:
+        - https://github.com/elastic/elastic-agent/issues/1157
+      timestamp: 1664212969
+      file:
+        name: 1664212969-Create-separate-status-reporter-for-local-only-events-so-that-degraded-fleet-che.yaml
+        checksum: 929f1ada47aeaaf9c631091c0f2732f631b3539f
+    - kind: feature
+      summary: Improve logging during upgrades.
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/1287
+      issue:
+        - https://github.com/elastic/elastic-agent/issues/1253
+      timestamp: 1664230732
+      file:
+        name: 1664230732-Improve-logging-during-upgrades.yaml
+        checksum: 13ab968324d342118c5a257d6c6cc0c5db161b46
+    - kind: bug-fix
+      summary: Add success log message after previous checkin failures
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/1327
+      issue: []
+      timestamp: 1664360554
+      file:
+        name: 1664360554-Add-success-log-message-after-previous-checkin-failures.yaml
+        checksum: 819a22a452dddfa3b2976433d5cb4c8354a6ccc5
+    - kind: bug-fix
+      summary: Fix docker provider add_fields processors
+      description: ""
+      component: providers
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/1420
+      issue:
+        - https://github.com/elastic/elastic-agent/issues/29030
+      timestamp: 1664989867
+      file:
+        name: 1664989867-fix-docker-provider-processors.yaml
+        checksum: 45731791cd2e8f1d747abfc47bb256971a77d015
+    - kind: enhancement
+      summary: Improve logging of Fleet check-in errors.
+      description: Improve logging of Fleet check-in errors and only report the local state as degraded after two consecutive failed check-ins.
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/1477
+      issue:
+        - https://github.com/elastic/elastic-agent/issues/1154
+      timestamp: 1665517984
+      file:
+        name: 1665517984-improve-checkin-error-logging.yaml
+        checksum: 2d3dd39309def9a082f794eda815af459596c2e6
+    - kind: bug-fix
+      summary: Fix admin permission check on localized windows
+      description: ""
+      component: ""
+      pr:
+        - https://github.com/elastic/elastic-agent/pull/1552
+      issue:
+        - https://github.com/elastic/elastic-agent/issues/857
+      timestamp: 1666088774
+      file:
+        name: 1666088774-Fix-admin-permission-check-on-localized-windows.yaml
+        checksum: 16ee5909c319680b8d32045e74c38922eafc29ea

changelog/fragments/1665780486-heartbeat-es-output-only.yaml

@@ -0,0 +1,31 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Only support elasticsearch as an output for the beta synthetics integration.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description: 
+
+# Affected component; a word indicating the component this changeset affects.
+component: synthetics-integration
+
+# PR number; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: 1491
+
+# Issue number; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: 1234