Add auto instrumentation support to salt (#1686)

.github/workflows/salt-test.yml

@@ -19,7 +19,6 @@ concurrency:
 
 env:
   PYTHON_VERSION: '3.10'
-  PIP_VERSION: '22.0.4'
   REQUIREMENTS_PATH: "internal/buildscripts/packaging/tests/requirements.txt"
   RESULT_PATH: "~/testresults"
 
@@ -42,6 +41,7 @@ jobs:
     strategy:
       matrix:
         PACKAGE_TYPE: [ "deb", "rpm" ]
+        TEST_CASE: [ "with_fluentd", "without_fluentd", "with_instrumentation" ]
     steps:
       - name: Check out the codebase.
         uses: actions/checkout@v3
@@ -59,20 +59,14 @@ jobs:
           key: v1-pytest-${{ env.PYTHON_VERSION }}-${{ hashFiles(env.REQUIREMENTS_PATH) }}
 
       - name: Install pytest
-        run: |
-          if which pip; then
-              pip install --upgrade 'pip==${{ env.PIP_VERSION }}'
-          else
-              curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-              python get-pip.py 'pip==${{ env.PIP_VERSION }}'
-          fi
-          pip install -r "${{ env.REQUIREMENTS_PATH }}"
+        run: pip install -r "${{ env.REQUIREMENTS_PATH }}"
 
       - name: Test salt deployment
         timeout-minutes: 45
         run: |
           mkdir -p ${{ env.RESULT_PATH }}
           pytest -n2 --verbose -m ${{ matrix.PACKAGE_TYPE }} \
+            -k ${{ matrix.TEST_CASE }} \
             --junitxml=${{ env.RESULT_PATH }}/results.xml \
             --html=${{ env.RESULT_PATH }}/results.html \
             --self-contained-html \
@@ -81,5 +75,5 @@ jobs:
       - name: Uploading test result
         uses: actions/upload-artifact@v3
         with:
-          name: salt-test-${{ matrix.PACKAGE_TYPE }}-result
+          name: salt-test-${{ matrix.TEST_CASE}}-${{ matrix.PACKAGE_TYPE }}-result
           path: ${{ env.RESULT_PATH }}

deployments/salt/README.md

@@ -119,3 +119,57 @@ splunk-otel-collector:
   remote hosts. To use custom fluentd config add the config file into salt dir, 
   e.g. `salt://templates/td_agent.conf` (**default:** `""` meaning 
   that nothing will be copied and existing `splunk_fluentd_config` will be used)
+
+- `install_auto_instrumentation`: Whether to install/manage [Splunk
+  OpenTelemetry Auto Instrumentation for Java](
+  https://github.com/signalfx/splunk-otel-collector/tree/main/instrumentation).
+  When set to `True`, the `splunk-otel-auto-instrumentation` deb/rpm package
+  will be downloaded and installed from the Collector repository. **Note:** The
+  Java application on the node needs to be started/restarted separately after
+  installation in order for auto instrumentation to take effect. (**default:**
+  `False`)
+
+- `auto_instrumentation_version`: Version of the
+  `splunk-otel-auto-instrumentation` package to install, e.g. `0.50.0`. The
+  minimum supported version is `0.48.0`. **Note:** The Java application on the
+  node needs to be restarted separately in order for any change to take effect.
+  (**default:** `latest`)
+
+- `auto_instrumentation_ld_so_preload`: By default, the `/etc/ld.so.preload`
+  file on the node will be configured for the
+  `/usr/lib/splunk-instrumentation/libsplunk.so` [shared object library](
+  https://github.com/signalfx/splunk-otel-collector/tree/main/instrumentation#operation)
+  provided by the `splunk-otel-auto-instrumentation` package and is required
+  for auto instrumentation. Configure this variable to include additional
+  library paths, e.g. `/path/to/my.library.so`. **Note:** The Java application
+  on the node needs to be restarted separately in order for any change to take
+  effect. (**default:** `None`)
+
+- `auto_instrumentation_java_agent_path`: Path to the [Splunk OpenTelemetry
+  Java agent](https://github.com/signalfx/splunk-otel-java). The default path
+  is provided by the `splunk-otel-auto-instrumentation` package. If the path is
+  changed from the default value, the path should be an existing file on the
+  node. The specified path will be added to the
+  `/usr/lib/splunk-instrumentation/instrumentation.conf` config file on the
+  node. **Note:** The Java application on the node needs to be restarted
+  separately in order for any change to take effect. (**default:**
+  `/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar`)
+
+- `auto_instrumentation_resource_attributes`: Configure the OpenTelemetry
+  instrumentation [resource attributes](
+  https://github.com/signalfx/splunk-otel-collector/tree/main/instrumentation#configuration-file),
+  e.g. `deployment.environment=prod`. The specified resource attribute(s) will
+  be added to the `/usr/lib/splunk-instrumentation/instrumentation.conf` config
+  file on the node. **Note:** The Java application on the node needs to be
+  restarted separately in order for any change to take effect. (**default:**
+  `None`)
+
+- `auto_instrumentation_service_name`: Explicitly set the [service name](
+  https://github.com/signalfx/splunk-otel-collector/tree/main/instrumentation#configuration-file)
+  for the instrumented Java application, e.g. `my.service`. By default, the
+  service name is automatically derived from the arguments of the Java
+  executable on the node. However, if this variable is set to a non-empty
+  value, the value will override the derived service name and be added to the
+  `/usr/lib/splunk-instrumentation/instrumentation.conf` config file on the
+  node. **Note:** The Java application on the node needs to be restarted
+  separately in order for any change to take effect. (**default:** `None`)

deployments/salt/splunk-otel-collector/auto_instrumentation.sls

@@ -0,0 +1,37 @@
+{% set auto_instrumentation_version = salt['pillar.get']('splunk-otel-collector:auto_instrumentation_version', 'latest') %}
+{% set auto_instrumentation_java_agent_path = salt['pillar.get']('splunk-otel-collector:auto_instrumentation_java_agent_path', '/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar') %}
+{% set auto_instrumentation_ld_so_preload = salt['pillar.get']('splunk-otel-collector:auto_instrumentation_ld_so_preload') %}
+{% set auto_instrumentation_resource_attributes = salt['pillar.get']('splunk-otel-collector:auto_instrumentation_resource_attributes') %}
+{% set auto_instrumentation_service_name = salt['pillar.get']('splunk-otel-collector:auto_instrumentation_service_name') %}
+
+Install Splunk OpenTelemetry Auto Instrumentation:
+  pkg.installed:
+    - name: splunk-otel-auto-instrumentation
+    - version: {{ auto_instrumentation_version }}
+    - require:
+      - pkg: splunk-otel-collector
+
+/etc/ld.so.preload:
+  file.managed:
+    - contents:
+        - /usr/lib/splunk-instrumentation/libsplunk.so
+{% if auto_instrumentation_ld_so_preload %}
+        - {{ auto_instrumentation_ld_so_preload }}
+{% endif %}
+    - makedirs: True
+    - require:
+      - pkg: splunk-otel-auto-instrumentation
+
+/usr/lib/splunk-instrumentation/instrumentation.conf:
+  file.managed:
+    - contents:
+        - java_agent_jar={{ auto_instrumentation_java_agent_path }}
+{% if auto_instrumentation_resource_attributes %}
+        - resource_attributes={{ auto_instrumentation_resource_attributes }}
+{% endif %}
+{% if auto_instrumentation_service_name %}
+        - service_name={{ auto_instrumentation_service_name }}
+{% endif %}
+    - makedirs: True
+    - require:
+      - pkg: splunk-otel-auto-instrumentation

deployments/salt/splunk-otel-collector/init.sls

@@ -1,18 +1,23 @@
-{% if not salt['pillar.get']('splunk-otel-collector:splunk_access_token') %}
+{% if grains['os_family'] not in ['Debian', 'RedHat', 'Suse'] %}
+Check OS family:
+  test.fail_without_changes:
+    - name: "OS family ({{ grains['os_family'] }}) is not supported!"
+    - failhard: True
+{% elif not salt['pillar.get']('splunk-otel-collector:splunk_access_token') %}
 Check splunk_access_token:
-  cmd.run:
-    - name: |
-        echo "splunk_access_token is is not specified";
-        exit 1
+  test.fail_without_changes:
+    - name: "splunk_access_token is not specified!"
+    - failhard: True
 {% elif not salt['pillar.get']('splunk-otel-collector:splunk_realm') %}
 Check splunk_realm:
-  cmd.run:
-    - name: |
-        echo "splunk_realm is is not specified";
-        exit 1
-{% else %}
+  test.fail_without_changes:
+    - name: "splunk_realm is not specified!"
+    - failhard: True
+{% endif %}
 
 {% set install_fluentd = salt['pillar.get']('splunk-otel-collector:install_fluentd', True) | to_bool %}
+{% set install_auto_instrumentation = salt['pillar.get']('splunk-otel-collector:install_auto_instrumentation', False) | to_bool %}
+
 include:
 {% if grains['os_family'] == 'Suse' or grains['oscodename'] == 'jammy' or install_fluentd == False %}
   - splunk-otel-collector.install
@@ -30,5 +35,6 @@ include:
   - splunk-otel-collector.fluentd_config
   - splunk-otel-collector.fluentd_service
 {% endif %}
-
+{% if install_auto_instrumentation %}
+  - splunk-otel-collector.auto_instrumentation
 {% endif %}

deployments/salt/splunk-otel-collector/install.sls

@@ -1,16 +1,8 @@
-{% set os_family = salt['grains.get']('os_family') %}
-
-# Check if the OS is in supported types.
-
-{% if os_family not in ['Debian', 'RedHat', 'Suse'] %}
-
-{{ "This deploy is supported on ['Debian', 'Ubuntu'], ['CentOS', 'Red Hat Enterprise Linux', 'Amazon'], ['Suse'] " }}
-
-{% else %}
-
+{% set os_family = grains['os_family'] %}
 {% set splunk_repo_base_url = salt['pillar.get']('splunk-otel-collector:repo_base_url', 'https://splunk.jfrog.io/splunk') %}
-
 {% set package_stage = salt['pillar.get']('splunk-otel-collector:package_stage', 'release') %}
+{% set collector_version = salt['pillar.get']('splunk-otel-collector:collector_version', 'latest') %}
+
 # Repository configuration.
 
 {% if os_family == 'RedHat' %}
@@ -79,8 +71,4 @@ Add Splunk OpenTelemetry Collector repo to zypper source list:
 Install Splunk OpenTelemetry Collector:
   pkg.installed:
     - name: splunk-otel-collector
-{% if salt['pillar.get']('splunk-otel-collector:collector_version') is not none and salt['pillar.get']('splunk-otel-collector:collector_version') != 'latest' %}
-    - version: {{ salt['pillar.get']('splunk-otel-collector:collector_version') }}
-{% endif %}
-
-{% endif %}
+    - version: {{ collector_version }}

internal/buildscripts/packaging/tests/deployments/salt/salt_test.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import glob
-import os
 import string
 import tempfile
 
@@ -26,6 +25,7 @@
     run_container_cmd,
     run_distro_container,
     service_is_running,
+    verify_package_version,
     wait_for,
     REPO_DIR,
     SERVICE_NAME,
@@ -65,6 +65,7 @@
   """
 )
 
+
 def run_salt_apply(container, config):
     with tempfile.NamedTemporaryFile(mode="w+") as fd:
         print(config)
@@ -88,7 +89,8 @@ def verify_env_file(container):
     run_container_cmd(container, f"grep '^SPLUNK_BUNDLE_DIR={SPLUNK_BUNDLE_DIR}$' {SPLUNK_ENV_PATH}")
     run_container_cmd(container, f"grep '^SPLUNK_COLLECTD_DIR={SPLUNK_COLLECTD_DIR}$' {SPLUNK_ENV_PATH}")
 
-@pytest.mark.installer
+
+@pytest.mark.salt
 @pytest.mark.parametrize(
     "distro",
     [pytest.param(distro, marks=pytest.mark.deb) for distro in DEB_DISTROS]
@@ -112,14 +114,19 @@ def test_salt_with_fluentd(distro):
                 assert wait_for(lambda: service_is_running(container))
                 if "opensuse" not in distro and distro != "ubuntu-jammy":
                     assert container.exec_run("systemctl status td-agent").exit_code == 0
+                if collector_version == "latest":
+                    verify_package_version(container, "splunk-otel-collector", collector_version, "0.34.0")
+                else:
+                    verify_package_version(container, "splunk-otel-collector", collector_version)
         finally:
             run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")
             if "opensuse" not in distro and distro != "ubuntu-jammy":
                 run_container_cmd(container, "journalctl -u td-agent --no-pager")
                 if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0:
                     run_container_cmd(container, "cat /var/log/td-agent/td-agent.log")
 
-@pytest.mark.installer
+
+@pytest.mark.salt
 @pytest.mark.parametrize(
     "distro",
     [pytest.param(distro, marks=pytest.mark.deb) for distro in DEB_DISTROS]
@@ -138,5 +145,72 @@ def test_salt_without_fluentd(distro):
             verify_env_file(container)
             assert wait_for(lambda: service_is_running(container))
             assert container.exec_run("systemctl status td-agent").exit_code != 0
+            if distro in DEB_DISTROS:
+                assert container.exec_run("dpkg -s splunk-otel-auto-instrumentation").exit_code != 0
+            else:
+                assert container.exec_run("rpm -q splunk-otel-auto-instrumentation").exit_code != 0
+        finally:
+            run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")
+
+
+INSTRUMENTATION_CONFIG = string.Template(f"""
+splunk-otel-collector:
+  splunk_access_token: '{SPLUNK_ACCESS_TOKEN}'
+  splunk_realm: '{SPLUNK_REALM}'
+  collector_version: '$version'
+  install_auto_instrumentation: 'True'
+  auto_instrumentation_version: '$version'
+  auto_instrumentation_resource_attributes: 'deployment.environment=test'
+  auto_instrumentation_service_name: 'test'
+  """
+)
+
+
+def verify_instrumentation_config(container):
+    config_path = "/usr/lib/splunk-instrumentation/instrumentation.conf"
+    libsplunk_path = "/usr/lib/splunk-instrumentation/libsplunk.so"
+    java_agent_path = "/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar"
+
+    try:
+        run_container_cmd(container, f"grep '^{libsplunk_path}' /etc/ld.so.preload")
+        run_container_cmd(container, f"grep '^java_agent_jar={java_agent_path}' {config_path}")
+        run_container_cmd(container, f"grep '^resource_attributes=deployment.environment=test' {config_path}")
+        run_container_cmd(container, f"grep '^service_name=test' {config_path}")
+    finally:
+        run_container_cmd(container, "cat /etc/ld.so.preload")
+        run_container_cmd(container, f"cat {config_path}")
+
+
+@pytest.mark.salt
+@pytest.mark.instrumentation
+@pytest.mark.parametrize(
+    "distro",
+    [pytest.param(distro, marks=pytest.mark.deb) for distro in DEB_DISTROS]
+    + [pytest.param(distro, marks=pytest.mark.rpm) for distro in RPM_DISTROS],
+    )
+def test_salt_with_instrumentation(distro):
+    if distro in DEB_DISTROS:
+        dockerfile = IMAGES_DIR / "deb" / f"Dockerfile.{distro}"
+    else:
+        dockerfile = IMAGES_DIR / "rpm" / f"Dockerfile.{distro}"
+
+    with run_distro_container(distro, dockerfile=dockerfile, path=REPO_DIR) as container:
+        try:
+            for version in ["0.48.0", "latest"]:
+                config = INSTRUMENTATION_CONFIG.substitute(version=version)
+                run_salt_apply(container, config)
+                verify_env_file(container)
+                assert wait_for(lambda: service_is_running(container))
+                if "opensuse" not in distro and distro != "ubuntu-jammy":
+                    assert container.exec_run("systemctl status td-agent").exit_code == 0
+                if version == "latest":
+                    verify_package_version(container, "splunk-otel-auto-instrumentation", version, "0.48.0")
+                else:
+                    verify_package_version(container, "splunk-otel-auto-instrumentation", version)
+                verify_instrumentation_config(container)
         finally:
             run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")
+            if "opensuse" not in distro and distro != "ubuntu-jammy":
+                run_container_cmd(container, "journalctl -u td-agent --no-pager")
+                if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0:
+                    run_container_cmd(container, "cat /var/log/td-agent/td-agent.log")

internal/buildscripts/packaging/tests/pytest.ini

@@ -5,3 +5,4 @@ markers =
     instrumentation
     puppet
     rpm
+    salt