Add orchestrator for supporting multiple processes

.github/workflows/bare-metal.yaml

@@ -0,0 +1,50 @@
+name: e2e-tests-bare-metal
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+  pull_request:
+jobs:
+  bare-metal-test:
+    strategy:
+      matrix:
+        k8s-version: ["v1.26.0"]
+        library: ["gorillamux", "nethttp", "gin"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+      - name: Setup BATS
+        uses: mig4/setup-bats@v1
+      - name: run collector
+        run: |
+          docker run  -d -v ${PWD}/.github/workflows/e2e/bare-metal/collector-config.yml:/etc/config.yaml -v /tmp:/tmp -p 4317:4317  otel/opentelemetry-collector-contrib  --config /etc/config.yaml
+      - name: Build auto-instrumentation
+        run: |
+          IMG=otel-go-instrumentation:latest make docker-build
+      - name: run auto-instrumentation
+        run: |
+          docker run -d -e OTEL_EXPORTER_OTLP_ENDPOINT="http://172.17.0.1:4317"  --privileged --pid=host  otel-go-instrumentation
+      - name: Build sample app
+        run: |
+          cd test/e2e/${{ matrix.library }}
+          go build .
+          cd ../../..
+      - name: run app
+        run: |
+          export OTEL_SERVICE_NAME=sample-app
+          ./test/e2e/${{matrix.library}}/${{matrix.library}}
+      - name: copy traces
+        run: |
+          sudo chmod 666 /tmp/trace.json
+          cp /tmp/trace.json ./test/e2e/${{ matrix.library }}/traces-orig.json
+          rm -f ./test/e2e/${{ matrix.library }}/traces.json
+      - name: verify output and redact to traces.json
+        run: |
+          bats ./test/e2e/${{ matrix.library }}/verify.bats

.github/workflows/e2e/bare-metal/collector-config.yml

@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+exporters:
+  logging: {}
+  file/trace:
+    path: /tmp/trace.json
+    rotation:
+service:
+  telemetry:
+    logs:
+      level: "debug"
+  pipelines:
+    traces:
+      receivers:
+        - otlp
+      exporters:
+        - file/trace
+        - logging

.github/workflows/e2e/k8s/deployment/sample-job.yml

@@ -0,0 +1,63 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: sample-job
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        workload: job
+      labels:
+        app: sample
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      containers:
+        - name: sample-app
+          image: sample-app
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: OTEL_SERVICE_NAME
+              value: "sample-app"
+          command: ["/bin/sh", "-c"]
+          args: ["/sample-app/main"]
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "auto-instrumentation"
+  namespace: default
+  labels:
+    app: "auto-instrumentation"
+spec:
+  hostPID: true
+  containers:
+    - name: auto-instrumentation
+      image: otel-go-instrumentation
+      imagePullPolicy: IfNotPresent
+      env:
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: "http://test-opentelemetry-collector:4317"
+        - name: OTEL_PROPAGATORS
+          value: "tracecontext,baggage"
+      resources: {}
+      securityContext:
+        runAsUser: 0
+        runAsGroup: 0
+        capabilities:
+          add:
+            - SYS_PTRACE
+        privileged: true
+      volumeMounts:
+        - mountPath: /sys/kernel/debug
+          name: kernel-debug
+  volumes:
+    - name: kernel-debug
+      hostPath:
+        path: /sys/kernel/debug
+  restartPolicy: Always
+---

.github/workflows/e2e/k8s/sidecar/collector-helm-values.yml

@@ -0,0 +1,43 @@
+mode: "statefulset"
+
+config:
+  exporters:
+    logging: {}
+    file/trace:
+      path: /tmp/trace.json
+      rotation:
+
+  service:
+    telemetry:
+      logs:
+        level: "debug"
+    pipelines:
+      traces:
+        receivers:
+          - otlp
+        exporters:
+          - file/trace
+          - logging
+
+
+image:
+  repository: otel/opentelemetry-collector-contrib
+  tag: "latest"
+
+command:
+  name: otelcol-contrib
+
+extraVolumes:
+- name: filevolume
+  emptyDir: {}
+extraVolumeMounts: 
+- mountPath: /tmp
+  name: filevolume
+
+extraContainers: 
+- name: filecp
+  image: busybox
+  command: ["sh", "-c", "sleep 36000"]   
+  volumeMounts:
+  - name: filevolume
+    mountPath: /tmp

.github/workflows/kind.yml

@@ -1,19 +1,18 @@
 name: e2e-tests
-
 on:
   push:
     branches:
       - main
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
   pull_request:
-
 jobs:
   kubernetes-test:
     strategy:
       matrix:
         k8s-version: ["v1.26.0"]
         library: ["gorillamux", "nethttp", "gin"]
+        mode: ["sidecar", "deployment"]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repo
@@ -52,19 +51,19 @@ jobs:
       - name: Add Dependencies
         shell: bash
         run: |
-            helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+          helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
       - uses: actions/checkout@v3
         with:
           repository: 'open-telemetry/opentelemetry-helm-charts'
           path: opentelemetry-helm-charts
       - name: Helm install collector
-        run: helm install test -f .github/workflows/e2e/k8s/collector-helm-values.yml opentelemetry-helm-charts/charts/opentelemetry-collector
+        run: helm install test -f .github/workflows/e2e/k8s/${{ matrix.mode }}/collector-helm-values.yml opentelemetry-helm-charts/charts/opentelemetry-collector
       - name: check collector status
         run: |
           kubectl wait --for=condition=Ready --timeout=60s pod/test-opentelemetry-collector-0
       - name: start sample job
         run: |
-          kubectl -n default create -f .github/workflows/e2e/k8s/sample-job.yml
+          kubectl -n default create -f .github/workflows/e2e/k8s/${{ matrix.mode }}/sample-job.yml
       - name: check job status
         run: |
           kubectl wait --for=condition=Complete --timeout=60s job/sample-job

CHANGELOG.md

@@ -8,6 +8,8 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http
 
 ## [Unreleased]
 
+- Add ability to monitor multiple processes. ([#197](https://github.com/open-telemetry/opentelemetry-go-instrumentation/issues/197))
+
 ## [v0.2.2-alpha] - 2023-07-12
 
 ### Added

Makefile

@@ -14,6 +14,9 @@ BPF_INCLUDE += -I${REPODIR}/include
 
 .DEFAULT_GOAL := precommit
 
+# Generate fixtures using "sidecar" or "deployment" method
+FIXTURE_MODE ?= "sidecar"
+
 .PHONY: precommit
 precommit: license-header-check go-mod-tidy golangci-lint-fix
 
@@ -132,9 +135,10 @@ fixtures/%:
 	if [ ! -d "opentelemetry-helm-charts" ]; then \
 		git clone https://github.com/open-telemetry/opentelemetry-helm-charts.git; \
 	fi
-	helm install test -f .github/workflows/e2e/k8s/collector-helm-values.yml opentelemetry-helm-charts/charts/opentelemetry-collector
+	helm install test -f .github/workflows/e2e/k8s/$(FIXTURE_MODE)/collector-helm-values.yml opentelemetry-helm-charts/charts/opentelemetry-collector
+	sleep 5
 	kubectl wait --for=condition=Ready --timeout=60s pod/test-opentelemetry-collector-0
-	kubectl -n default create -f .github/workflows/e2e/k8s/sample-job.yml
+	kubectl -n default create -f .github/workflows/e2e/k8s/$(FIXTURE_MODE)/sample-job.yml
 	kubectl wait --for=condition=Complete --timeout=60s job/sample-job
 	kubectl cp -c filecp default/test-opentelemetry-collector-0:tmp/trace.json ./test/e2e/$(LIBRARY)/traces-orig.json
 	rm -f ./test/e2e/$(LIBRARY)/traces.json

cli/main.go

@@ -15,18 +15,41 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/signal"
+	"runtime"
+	"strings"
 	"syscall"
 
-	"go.opentelemetry.io/auto/pkg/errors"
-	"go.opentelemetry.io/auto/pkg/instrumentors"
+	"google.golang.org/grpc"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+
+	"go.opentelemetry.io/auto"
 	"go.opentelemetry.io/auto/pkg/log"
-	"go.opentelemetry.io/auto/pkg/opentelemetry"
+	"go.opentelemetry.io/auto/pkg/orchestrator"
 	"go.opentelemetry.io/auto/pkg/process"
 )
 
+var (
+	// Controller-local reference to the auto-instrumentation release version.
+	releaseVersion = auto.Version()
+	// Start of this auto-instrumentation's exporter User-Agent header, e.g. ""OTel-Go-Auto-Instrumentation/1.2.3".
+	baseUserAgent = fmt.Sprintf("OTel-Go-Auto-Instrumentation/%s", releaseVersion)
+	// Information about the runtime environment for inclusion in User-Agent, e.g. "go/1.18.2 (linux/amd64)".
+	runtimeInfo = fmt.Sprintf(
+		"%s (%s/%s)",
+		strings.Replace(runtime.Version(), "go", "go/", 1),
+		runtime.GOOS,
+		runtime.GOARCH,
+	)
+	// Combined User-Agent identifying this auto-instrumentation and its runtime environment, see RFC7231 for format considerations.
+	autoinstUserAgent = fmt.Sprintf("%s %s", baseUserAgent, runtimeInfo)
+)
+
 func main() {
 	err := log.Init()
 	if err != nil {
@@ -35,56 +58,49 @@ func main() {
 	}
 
 	log.Logger.V(0).Info("starting Go OpenTelemetry Agent ...")
-	target := process.ParseTargetArgs()
-	if err = target.Validate(); err != nil {
-		log.Logger.Error(err, "invalid target args")
-		return
-	}
-
-	processAnalyzer := process.NewAnalyzer()
-	otelController, err := opentelemetry.NewController()
+	ctx := contextWithSigterm(context.Background())
+	log.Logger.V(0).Info("Establishing connection to OTLP receiver ...")
+	otlpTraceClient := otlptracegrpc.NewClient(
+		otlptracegrpc.WithDialOption(grpc.WithUserAgent(autoinstUserAgent)),
+	)
+	traceExporter, err := otlptrace.New(ctx, otlpTraceClient)
 	if err != nil {
-		log.Logger.Error(err, "unable to create OpenTelemetry controller")
+		log.Logger.Error(err, "unable to connect to OTLP endpoint")
 		return
 	}
-
-	instManager, err := instrumentors.NewManager(otelController)
+	targetArgs := process.ParseTargetArgs()
+	if targetArgs != nil {
+		if err := targetArgs.Validate(); err != nil {
+			log.Logger.Error(err, "invalid target args")
+			return
+		}
+	}
+	r, err := orchestrator.New(ctx, targetArgs, traceExporter)
 	if err != nil {
-		log.Logger.Error(err, "error creating instrumetors manager")
-		return
+		log.Logger.V(0).Error(err, "creating orchestrator")
+	}
+	if err = r.Run(); err != nil {
+		log.Logger.Error(err, "running orchestrator")
 	}
+}
 
-	stopper := make(chan os.Signal, 1)
-	signal.Notify(stopper, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-stopper
-		log.Logger.V(0).Info("Got SIGTERM, cleaning up..")
-		processAnalyzer.Close()
-		instManager.Close()
-	}()
+func contextWithSigterm(parent context.Context) context.Context {
+	ctx, cancel := context.WithCancel(parent)
 
-	pid, err := processAnalyzer.DiscoverProcessID(target)
-	if err != nil {
-		if err != errors.ErrInterrupted {
-			log.Logger.Error(err, "error while discovering process id")
-		}
-		return
-	}
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
 
-	targetDetails, err := processAnalyzer.Analyze(pid, instManager.GetRelevantFuncs())
-	if err != nil {
-		log.Logger.Error(err, "error while analyzing target process")
-		return
-	}
-	log.Logger.V(0).Info("target process analysis completed", "pid", targetDetails.PID,
-		"go_version", targetDetails.GoVersion, "dependencies", targetDetails.Libraries,
-		"total_functions_found", len(targetDetails.Functions))
+	go func() {
+		defer close(ch)
+		defer signal.Stop(ch)
 
-	instManager.FilterUnusedInstrumentors(targetDetails)
+		select {
+		case <-parent.Done(): // if parent is cancelled, return
+			return
+		case <-ch: // if SIGTERM is received, cancel this context
+			cancel()
+		}
+	}()
 
-	log.Logger.V(0).Info("invoking instrumentors")
-	err = instManager.Run(targetDetails)
-	if err != nil && err != errors.ErrInterrupted {
-		log.Logger.Error(err, "error while running instrumentors")
-	}
+	return ctx
 }