[test ]Test the pytest OTEL with the withOtelEnv step

.ci/Jenkinsfile

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-@Library('apm@master') _
+@Library('apm@feature/open-telemetry-with-helper') _
 
 pipeline {
   agent { label 'linux && immutable' }
@@ -94,6 +94,17 @@ pipeline {
         }
       }
     }
+    stage('Test OTEL') {
+      steps {
+        withGithubNotify(context: 'Test OTEL') {
+          dir("${BASE_DIR}"){
+            withOtelEnv() {
+              sh(script: '.ci/scripts/pytest-otel.sh')
+            }
+          }
+        }
+      }
+    }
     /**
      Checkout the code and stash it, to use it on other stages.
     */

.ci/scripts/pytest-otel.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+cd resources/scripts/pytest_otel || exit
+
+OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} \
+OTEL_EXPORTER_OTLP_HEADERS="authorization=Bearer ${OTEL_TOKEN_ID}" \
+TRACEPARENT="00-${TRACE_ID}-${SPAN_ID}-01" \
+make -n test || true
+
+OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} \
+OTEL_EXPORTER_OTLP_HEADERS="authorization=Bearer ${OTEL_TOKEN_ID}" \
+TRACEPARENT="00-${TRACE_ID}-${SPAN_ID}-01" \
+make test || true

resources/scripts/pytest_apm/Makefile

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 PYTEST_APM_VERSION = 0.0.1
 PYTEST_APM_PKG = pytest-apm-$(PYTEST_APM_VERSION)
 VENV ?= .venv

resources/scripts/pytest_otel/.editorconfig

@@ -0,0 +1,11 @@
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+max_line_length = 120
+tab_width = 4
+
+[Makefile]
+indent_style = tab

resources/scripts/pytest_otel/.gitignore

@@ -0,0 +1,9 @@
+__pycache__
+.coverage
+.idea
+.pytest_cache
+.venv
+*.pyc
+**/*.egg-info
+**/build
+**/dist

resources/scripts/pytest_otel/Makefile

@@ -0,0 +1,126 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+SHELL := /bin/bash
+PYTEST_OTEL_VERSION = 0.0.1
+PYTEST_OTEL_PKG = pytest-otel-$(PYTEST_OTEL_VERSION)
+OTEL_EXPORTER_OTLP_ENDPOINT ?= 127.0.0.1:4317
+OTEL_SERVICE_NAME ?= "pytest_otel_test"
+OTEL_EXPORTER_OTLP_INSECURE ?= true
+OTEL_EXPORTER_OTLP_HEADERS ?=
+TRACEPARENT ?=
+
+VENV ?= .venv
+PYTHON ?= python3
+PIP ?= pip3
+GH_VERSION = 1.0.0
+
+OTEL_EXPORTER_OTLP_ENDPOINT ?= 127.0.0.1:4317
+OTEL_SERVICE_NAME ?= pytest_otel_test
+OTEL_EXPORTER_OTLP_INSECURE ?= True
+OTEL_EXPORTER_OTLP_HEADERS ?=
+
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+	OS_FLAG := linux
+endif
+ifeq ($(UNAME_S),Darwin)
+	OS_FLAG := macOS
+endif
+UNAME_P := $(shell uname -m)
+ifeq ($(UNAME_P),x86_64)
+	ARCH_FLAG := amd64
+endif
+ifneq ($(filter %86,$(UNAME_P)),)
+	ARCH_FLAG := i386
+endif
+GH_BINARY = gh_$(GH_VERSION)_$(OS_FLAG)_$(ARCH_FLAG)
+GH = $(CURDIR)/bin/gh
+
+.SILENT:
+
+.PHONY: help
+help:
+	@echo "Targets:"
+	@echo ""
+	@grep '^## @help' Makefile|cut -d ":" -f 2-3|( (sort|column -s ":" -t) || (sort|tr ":" "\t") || (tr ":" "\t"))
+
+bin:
+	mkdir bin
+
+bin/gh: bin
+	curl -sSfL https://github.com/cli/cli/releases/download/v$(GH_VERSION)/$(GH_BINARY).tar.gz|tar xz
+	mv $(GH_BINARY)/bin/gh bin/gh
+	rm -fr $(GH_BINARY)
+
+## @help:virtualenv:Create a Python virtual environment.
+.PHONY: virtualenv
+virtualenv:
+	$(PYTHON) --version
+	test -d $(VENV) || virtualenv -q --python=$(PYTHON) $(VENV);\
+	source $(VENV)/bin/activate;\
+	$(PIP) install -q -r requirements.txt;
+
+## @help:install:Install APM CLI in a Python virtual environment.
+.PHONY: install
+install: virtualenv
+	source $(VENV)/bin/activate;\
+	$(PIP) install .;
+
+## @help:test:Run the test.
+.PHONY: test
+test: virtualenv install
+	OTEL_EXPORTER_OTLP_ENDPOINT="$(OTEL_EXPORTER_OTLP_ENDPOINT)" \
+	OTEL_SERVICE_NAME="$(OTEL_SERVICE_NAME)" \
+	OTEL_EXPORTER_OTLP_INSECURE="$(OTEL_EXPORTER_OTLP_INSECURE)" \
+	OTEL_EXPORTER_OTLP_HEADERS="$(OTEL_EXPORTER_OTLP_HEADERS)" \
+	TRACEPARENT="$(TRACEPARENT)" \
+	source $(VENV)/bin/activate;\
+	$(VENV)/bin/pytest --capture=no
+
+## @help:coverage:Report coverage.
+.PHONY: coverage
+coverage: virtualenv
+	source $(VENV)/bin/activate;\
+	coverage run --source=otel -m pytest; \
+	coverage report -m;
+
+## @help:clean:Remove Python file artifacts.
+.PHONY: clean
+clean:
+	@echo "+ $@"
+	@find . -type f -name "*.py[co]" -delete
+	@find . -type d -name "__pycache__" -delete
+	@find . -name '*~' -delete
+	-@rm -fr *.egg-info build dist $(VENV) bin
+
+package: virtualenv
+	source $(VENV)/bin/activate;\
+	$(PYTHON) setup.py sdist bdist_wheel
+
+release: bin/gh package
+	$(GH) release create $(PYTEST_OTEL_PKG) dist/$(PYTEST_OTEL_PKG).tar.gz \
+		--repo elastic/apm-pipeline-library \
+		--title $(PYTEST_OTEL_PKG) \
+		--notes "$(PYTEST_OTEL_PKG)"
+
+## @help:run-otel-collector:Run OpenTelemetry collector in debug mode.
+.PHONY: run-otel-collector
+run-otel-collector:
+	docker run --rm -p 4317:4317 \
+	-v "$(CURDIR)/otel-collector.yaml":/otel-config.yaml \
+	--name otelcol otel/opentelemetry-collector \
+	--config otel-config.yaml; \

resources/scripts/pytest_otel/README.md

@@ -0,0 +1,65 @@
+
+Features
+--------
+
+pytest-otel plugin for reporting APM traces of tests executed.
+
+
+Requirements
+------------
+
+* opentelemetry-api == 1.2
+* opentelemetry-exporter-otlp == 1.2.0
+* opentelemetry-sdk == 1.2.0
+
+
+Installation
+------------
+
+You can install "pytest-otel" via `pip` or using the `setup.py` script.
+
+```
+git checkout https://github.com/elastic/apm-pipeline-library
+cd apm-pipeline-library/resources/scripts
+pip install ./pytest_otel
+```
+
+Usage
+-----
+
+`pytest_otel` is configured by adding some parameters to the pytest command line. Below are the descriptions:
+
+* --endpoint: URL for the OpenTelemetry server. (Required). Env variable: `OTEL_EXPORTER_OTLP_ENDPOINT`
+* --headers: Additional headers to send (i.e.: key1=value1,key2=value2). Env variable: `OTEL_EXPORTER_OTLP_HEADERS`
+* --service-name: Name of the service. Env variable: `OTEL_SERVICE_NAME`
+* --session-name: Name for the main span.
+* --traceparent: Trace parent ID. Env variable: `TRACEPARENT`. See https://www.w3.org/TR/trace-context-1/#trace-context-http-headers-format
+* --insecure: Disables TLS. Env variable: `OTEL_EXPORTER_OTLP_INSECURE`
+
+```bash
+cd pytest_otel
+pytest --endpoint https://otelcollector.example.com:4317 \
+       --headers "authorization=Bearer ASWDCcCRFfr" \
+       --service-name pytest_otel \
+       --session-name='My_Test_cases' \
+       --traceparent=00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01 \
+       --insecure=False
+```
+
+**IMPORTANT**: If you use `--headers` the transaction metadata might expose those arguments
+with their values. In order to avoid any credentials to be exposed, it's recommended to use the environment variables.
+For instance, given the above example, a similar one with environment variables can be seen below:
+
+```bash
+OTEL_EXPORTER_OTLP_ENDPOINT=https://apm.example.com:8200 \
+OTEL_EXPORTER_OTLP_HEADERS="authorization=Bearer ASWDCcCRFfr" \
+OTEL_SERVICE_NAME=pytest_otel \
+TRACEPARENT=00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01 \
+OTEL_EXPORTER_OTLP_INSECURE=False \
+pytest --apm-session-name='My_Test_cases'
+```
+
+License
+-------
+
+Distributed under the terms of the `Apache License Version 2.0`_ license, "pytest-otel" is free and open source software

resources/scripts/pytest_otel/otel-collector.yaml

@@ -0,0 +1,23 @@
+---
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+exporters:
+  logging:
+    logLevel: debug
+
+processors:
+  batch:
+
+service:
+  pipelines:
+    traces:
+      receivers:
+        - otlp
+      processors:
+        - batch
+      exporters:
+        - logging