Add apm trace sampling (#5492)

* Update elastic-agent-client dependency to 7.16.0

* Add support for APM sampling rate config

* fixup! Add support for APM sampling rate config

* Add unit tests for APM config triggering component changes

* make notice

NOTICE.txt

@@ -1031,11 +1031,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-client/v7
-Version: v7.15.0
+Version: v7.16.0
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.15.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.16.0/LICENSE.txt:
 
 ELASTIC LICENSE AGREEMENT
 

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/docker/go-units v0.5.0
 	github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5
 	github.com/elastic/elastic-agent-autodiscover v0.8.2
-	github.com/elastic/elastic-agent-client/v7 v7.15.0
+	github.com/elastic/elastic-agent-client/v7 v7.16.0
 	github.com/elastic/elastic-agent-libs v0.10.1
 	github.com/elastic/elastic-agent-system-metrics v0.11.2
 	github.com/elastic/elastic-transport-go/v8 v8.6.0

go.sum

@@ -239,8 +239,8 @@ github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5 h1:BzN9o4IS1Hj
 github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5/go.mod h1:cxc20xI7fOgsFHWgt+PenlDDnMcrvh7Ocuj5hEFIdEk=
 github.com/elastic/elastic-agent-autodiscover v0.8.2 h1:Fs2FhR33AMBPfm5/jz4drVzaEZaqOIHlDBvGtkUZdIk=
 github.com/elastic/elastic-agent-autodiscover v0.8.2/go.mod h1:VZnU53EVaFTxR8Xf6YsLN8FHD5DKQzHSPlKax9/4w+o=
-github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE=
-github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
+github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0=
+github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
 github.com/elastic/elastic-agent-libs v0.10.1 h1:4MMqNQVPoQGqPMM9bEO1Q6d48QPhW+rIdV/0zt82ta4=
 github.com/elastic/elastic-agent-libs v0.10.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
 github.com/elastic/elastic-agent-system-metrics v0.11.2 h1:UjSBcY6U3H3venB5zAPEFNjAb4Bb38EiVqaA4zALEnM=

internal/pkg/agent/application/apm_config_modifier_test.go

@@ -439,6 +439,7 @@ func TestPatchAPMConfig(t *testing.T) {
                       skip_verify: true
                       server_ca: ""
                       server_certificate: ""
+                    sampling_rate: null
               `,
 		},
 	}

internal/pkg/agent/application/coordinator/diagnostics_test.go

@@ -106,15 +106,14 @@ agent:
         - host1
         - host2
       environment: diag-unit-test
-      apikey: apikey
-      secrettoken: secret
-      globallabels:
+      api_key: apikey
+      secret_token: secret
+      global_labels:
         k1: v1
         k2: v2
       tls:
-        skipverify: false
-        servercertificate: "/path/to/server/cert"
-        serverca: "/path/to/server/ca"
+        server_certificate: "/path/to/server/cert"
+        server_ca: "/path/to/server/ca"
 fleet:
   enabled: true
   access_api_key: "test-key"
@@ -337,6 +336,7 @@ components:
             skipverify: true
             servercert: servercert
             serverca: serverca
+          samplingrate: null
 `
 
 	coord := &Coordinator{componentModel: components}
@@ -556,6 +556,7 @@ components:
               skipverify: true
               serverca: sca
               servercert: sc
+            samplingrate: null
         limits: null
       component_idx: 1
 `

internal/pkg/agent/cmd/run.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -611,6 +612,7 @@ func initTracer(agentName, version string, mcfg *monitoringCfg.MonitoringConfig)
 		envVerifyServerCert = "ELASTIC_APM_VERIFY_SERVER_CERT"
 		envServerCert       = "ELASTIC_APM_SERVER_CERT"
 		envCACert           = "ELASTIC_APM_SERVER_CA_CERT_FILE"
+		envSampleRate       = "ELASTIC_APM_TRANSACTION_SAMPLE_RATE"
 	)
 	if cfg.TLS.SkipVerify {
 		os.Setenv(envVerifyServerCert, "false")
@@ -624,6 +626,10 @@ func initTracer(agentName, version string, mcfg *monitoringCfg.MonitoringConfig)
 		os.Setenv(envCACert, cfg.TLS.ServerCA)
 		defer os.Unsetenv(envCACert)
 	}
+	if cfg.SamplingRate != nil {
+		os.Setenv(envSampleRate, strconv.FormatFloat(float64(*cfg.SamplingRate), 'b', -1, 32))
+		defer os.Unsetenv(envSampleRate)
+	}
 
 	opts := apmtransport.HTTPTransportOptions{}
 

internal/pkg/core/monitoring/config/config.go

@@ -125,20 +125,21 @@ func DefaultConfig() *MonitoringConfig {
 
 // APMConfig configures APM Tracing.
 type APMConfig struct {
-	Environment  string            `config:"environment"`
-	APIKey       string            `config:"api_key"`
-	SecretToken  string            `config:"secret_token"`
-	Hosts        []string          `config:"hosts"`
-	GlobalLabels map[string]string `config:"global_labels"`
-	TLS          APMTLS            `config:"tls"`
+	Environment  string            `config:"environment" yaml:"environment,omitempty"`
+	APIKey       string            `config:"api_key" yaml:"api_key,omitempty"`
+	SecretToken  string            `config:"secret_token" yaml:"secret_token,omitempty"`
+	Hosts        []string          `config:"hosts" yaml:"hosts,omitempty"`
+	GlobalLabels map[string]string `config:"global_labels" yaml:"global_labels,omitempty"`
+	TLS          APMTLS            `config:"tls" yaml:"tls,omitempty"`
+	SamplingRate *float32          `config:"sampling_rate" yaml:"sampling_rate,omitempty"`
 }
 
 // APMTLS contains the configuration options necessary for configuring TLS in
 // apm-agent-go.
 type APMTLS struct {
-	SkipVerify        bool   `config:"skip_verify"`
-	ServerCertificate string `config:"server_certificate"`
-	ServerCA          string `config:"server_ca"`
+	SkipVerify        bool   `config:"skip_verify" yaml:"skip_verify,omitempty"`
+	ServerCertificate string `config:"server_certificate" yaml:"server_certificate,omitempty"`
+	ServerCA          string `config:"server_ca" yaml:"server_ca,omitempty"`
 }
 
 func defaultAPMConfig() APMConfig {

internal/pkg/core/monitoring/config/config_test.go

@@ -160,6 +160,9 @@ http:
 }
 
 func TestAPMConfig(t *testing.T) {
+
+	tenPercentSamplingRate := float32(0.1)
+
 	tcs := map[string]struct {
 		in  map[string]interface{}
 		out APMConfig
@@ -193,6 +196,24 @@ func TestAPMConfig(t *testing.T) {
 				},
 			},
 		},
+		"sampling_rate 10%": {
+			in: map[string]interface{}{
+				"traces": true,
+				"apm": map[string]interface{}{
+					"api_key":       "abc123",
+					"environment":   "production",
+					"hosts":         []string{"https://abc.123.com"},
+					"sampling_rate": &tenPercentSamplingRate,
+				},
+			},
+			out: APMConfig{
+				APIKey:       "abc123",
+				Environment:  "production",
+				Hosts:        []string{"https://abc.123.com"},
+				TLS:          APMTLS{},
+				SamplingRate: &tenPercentSamplingRate,
+			},
+		},
 	}
 
 	for name, tc := range tcs {

pkg/component/runtime/apm_config_mapper.go

@@ -27,6 +27,7 @@ func MapAPMConfig(conf *config.APMConfig) *proto.APMConfig {
 		SecretToken:  conf.SecretToken,
 		Hosts:        conf.Hosts,
 		GlobalLabels: buildGlobalLabelsString(conf.GlobalLabels),
+		SamplingRate: conf.SamplingRate,
 	}
 
 	if conf.TLS != zeroElasticAPMTLS {

pkg/component/runtime/command_test.go

@@ -9,7 +9,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-agent-client/v7/pkg/client"
+	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
+	"github.com/elastic/elastic-agent/pkg/component"
 )
 
 func TestAddToBucket(t *testing.T) {
@@ -51,3 +56,151 @@ func TestAddToBucket(t *testing.T) {
 		})
 	}
 }
+
+// TestSyncExpected verifies that the command runtime correctly establish if we need to send a CheckinObserved after an
+// update in the model coming from the coordinator
+func TestSyncExpected(t *testing.T) {
+
+	tenPercentSamplingRate := float32(0.1)
+	anotherTenPercentSamplingRate := tenPercentSamplingRate
+	t.Run("TestAPMConfig", func(t *testing.T) {
+		testcases := []struct {
+			name          string
+			initialConfig *proto.APMConfig
+			updatedConfig *proto.APMConfig
+			syncExpected  bool
+		}{
+			{
+				name:          "No config (both nil)",
+				initialConfig: nil,
+				updatedConfig: nil,
+				syncExpected:  false,
+			},
+			{
+				name:          "Config added",
+				initialConfig: nil,
+				updatedConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment: "test",
+						ApiKey:      "apikey",
+						Hosts:       []string{"some.somedomain"},
+					},
+				},
+				syncExpected: true,
+			},
+			{
+				name: "Same config",
+				initialConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment: "test",
+						ApiKey:      "apikey",
+						Hosts:       []string{"some.somedomain"},
+					},
+				},
+				updatedConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment: "test",
+						ApiKey:      "apikey",
+						Hosts:       []string{"some.somedomain"},
+					},
+				},
+				syncExpected: false,
+			},
+			{
+				name: "Added sampling rate",
+				initialConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment: "test",
+						ApiKey:      "apikey",
+						Hosts:       []string{"some.somedomain"},
+					},
+				},
+				updatedConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment:  "test",
+						ApiKey:       "apikey",
+						Hosts:        []string{"some.somedomain"},
+						SamplingRate: &tenPercentSamplingRate,
+					},
+				},
+				syncExpected: true,
+			},
+			{
+				name: "Same sampling rate",
+				initialConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment:  "test",
+						ApiKey:       "apikey",
+						Hosts:        []string{"some.somedomain"},
+						SamplingRate: &tenPercentSamplingRate,
+					},
+				},
+				updatedConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment:  "test",
+						ApiKey:       "apikey",
+						Hosts:        []string{"some.somedomain"},
+						SamplingRate: &anotherTenPercentSamplingRate,
+					},
+				},
+				syncExpected: false,
+			},
+			{
+				name: "Remove sampling rate",
+				initialConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment:  "test",
+						ApiKey:       "apikey",
+						Hosts:        []string{"some.somedomain"},
+						SamplingRate: &tenPercentSamplingRate,
+					},
+				},
+				updatedConfig: &proto.APMConfig{
+					Elastic: &proto.ElasticAPM{
+						Environment: "test",
+						ApiKey:      "apikey",
+						Hosts:       []string{"some.somedomain"},
+					},
+				},
+				syncExpected: true,
+			},
+		}
+
+		for _, tt := range testcases {
+			t.Run(tt.name, func(t *testing.T) {
+				compState := ComponentState{
+					State:       client.UnitStateHealthy,
+					Message:     "fake component state running",
+					Features:    nil,
+					FeaturesIdx: 0,
+					Component: &proto.Component{
+						ApmConfig: tt.initialConfig,
+					},
+					ComponentIdx: 0,
+					VersionInfo: ComponentVersionInfo{
+						Name:      "fake component",
+						BuildHash: "abcdefgh",
+					},
+					Pid:                 123,
+					expectedUnits:       nil,
+					expectedFeatures:    nil,
+					expectedFeaturesIdx: 0,
+					expectedComponent: &proto.Component{
+						ApmConfig: tt.initialConfig,
+					},
+					expectedComponentIdx: 0,
+				}
+
+				actualSyncExpected := compState.syncExpected(&component.Component{
+					ID: "fakecomponent",
+					Component: &proto.Component{
+						ApmConfig: tt.updatedConfig,
+					},
+				})
+
+				assert.Equal(t, tt.syncExpected, actualSyncExpected)
+			})
+		}
+	})
+
+}