Config to allow subset of processors and exporters

This adds two new job properties that let's an operator restrict the
kinds of OTel processors and exporters that can be used:
`allow_list.processors` and `allow_list.exporters`. These properties
are empty by default ([]), which keeps the existing behavior of
allowing all processors and exporters that are included in our
distribution. If not empty, then only the included components are
allowed.

Signed-off-by: Ivan Protsiuk <[email protected]>
Signed-off-by: Matthew Kocher <[email protected]>
Signed-off-by: Glenn Oppegard <[email protected]>

jobs/otel-collector-windows/spec

@@ -40,6 +40,14 @@ properties:
             receivers: [otlp/placeholder]
             processors: [batch]
             exporters: [otlp]
+  allow_list.processors:
+    description: "Processors allowed for use in otel-collector config. Must be a subset of list included in otel-collector builder config. Empty list means allow all possible."
+    default: []
+    example: ['batch']
+  allow_list.exporters:
+    description: "Exporters allowed for use in otel-collector config. Must be a subset of list included in otel-collector builder config. Empty list means allow all possible."
+    default: []
+    example: ['otlp', 'prometheus', 'prometheusremotewrite']
   ingress.grpc.address:
     description: "Address to listen on to receive OTLP over gRPC"
     default: 127.0.0.1

jobs/otel-collector-windows/templates/config.yml.erb

@@ -15,7 +15,7 @@ def check_for_new_and_old_properties!
 end
 
 def retrieve_property(name)
-  if p(name).respond_to?(:keys)
+  if p(name).respond_to?(:each)
     p(name)
   else
     YAML.safe_load(p(name))
@@ -90,8 +90,8 @@ def set_internal_receiver_as_only_receiver
 end
 
 def set_internal_receiver_on_all_pipelines
-  config['service']['pipelines'].each_value do |p|
-    p['receivers'] = ['otlp/cf-internal-local']
+  config['service']['pipelines'].each_value do |pipeline|
+    pipeline['receivers'] = ['otlp/cf-internal-local']
   end
 end
 
@@ -104,11 +104,39 @@ def expose_internal_telemetry
   }
 end
 
+# Hardcoded list of exporters included in this otelcol distribution at `src/otel-collector-builder/config.yaml`
+def included_exporters
+  %w[otlp file prometheus prometheusremotewrite].sort
+end
+
+# Hardcoded list of processors included in this otelcol distribution at `src/otel-collector-builder/config.yaml`
+def included_processors
+  %w[batch memory_limiter].sort
+end
+
+def check_for_use_of_valid_components!(component_kind, included_components, allowed_components)
+  user_components = config.fetch(component_kind, {}).map {|key, _| key.split('/')[0]}
+  if allowed_components.empty?
+    allowed_components = included_components # Allow all if none are provided
+  end
+
+  unrecognized_user_components = user_components - included_components
+  raise "The following configured #{component_kind} are not included in this OpenTelemetry Collector distribution: #{unrecognized_user_components}. Available: #{included_components}." unless unrecognized_user_components.empty?
+
+  unrecognized_allowed_components = allowed_components - included_components
+  raise "The following #{component_kind} specified in the allow list are not included in this OpenTelemetry Collector distribution: #{unrecognized_allowed_components}. Available: #{included_components}." unless unrecognized_allowed_components.empty?
+
+  disallowed_components = user_components - allowed_components
+  raise "The following configured #{component_kind} are not allowed: #{disallowed_components}. Allowed: #{allowed_components}." unless disallowed_components.empty?
+end
+
 check_for_new_and_old_properties!
 check_for_no_exporters!
 check_for_no_service_config!
 check_for_use_of_reserved_prefix!
 check_for_use_of_reserved_bbs_api_port!
+check_for_use_of_valid_components!('processors', included_processors, retrieve_property('allow_list.processors'))
+check_for_use_of_valid_components!('exporters', included_exporters, retrieve_property('allow_list.exporters'))
 set_internal_receiver_as_only_receiver
 set_internal_receiver_on_all_pipelines
 expose_internal_telemetry

jobs/otel-collector/spec

@@ -17,6 +17,14 @@ properties:
   enabled:
     description: "Enable OTel Collector"
     default: true
+  allow_list.exporters:
+    description: "Exporters allowed for use in otel-collector config. Must be a subset of list included in otel-collector builder config. Empty list means allow all possible."
+    default: []
+    example: ['otlp', 'prometheus', 'prometheusremotewrite']
+  allow_list.processors:
+    description: "Processors allowed for use in otel-collector config. Must be a subset of list included in otel-collector builder config. Empty list means allow all possible."
+    default: []
+    example: ['batch']
   config:
     description: "Collector configuration"
     default: {}

jobs/otel-collector/templates/config.yml.erb

@@ -15,7 +15,7 @@ def check_for_new_and_old_properties!
 end
 
 def retrieve_property(name)
-  if p(name).respond_to?(:keys)
+  if p(name).respond_to?(:each)
     p(name)
   else
     YAML.safe_load(p(name))
@@ -90,8 +90,8 @@ def set_internal_receiver_as_only_receiver
 end
 
 def set_internal_receiver_on_all_pipelines
-  config['service']['pipelines'].each_value do |p|
-    p['receivers'] = ['otlp/cf-internal-local']
+  config['service']['pipelines'].each_value do |pipeline|
+    pipeline['receivers'] = ['otlp/cf-internal-local']
   end
 end
 
@@ -104,11 +104,39 @@ def expose_internal_telemetry
   }
 end
 
+# Hardcoded list of exporters included in this otelcol distribution at `src/otel-collector-builder/config.yaml`
+def included_exporters
+  %w[otlp file prometheus prometheusremotewrite].sort
+end
+
+# Hardcoded list of processors included in this otelcol distribution at `src/otel-collector-builder/config.yaml`
+def included_processors
+  %w[batch memory_limiter].sort
+end
+
+def check_for_use_of_valid_components!(component_kind, included_components, allowed_components)
+  user_components = config.fetch(component_kind, {}).map {|key, _| key.split('/')[0]}
+  if allowed_components.empty?
+    allowed_components = included_components # Allow all if none are provided
+  end
+
+  unrecognized_user_components = user_components - included_components
+  raise "The following configured #{component_kind} are not included in this OpenTelemetry Collector distribution: #{unrecognized_user_components}. Available: #{included_components}." unless unrecognized_user_components.empty?
+
+  unrecognized_allowed_components = allowed_components - included_components
+  raise "The following #{component_kind} specified in the allow list are not included in this OpenTelemetry Collector distribution: #{unrecognized_allowed_components}. Available: #{included_components}." unless unrecognized_allowed_components.empty?
+
+  disallowed_components = user_components - allowed_components
+  raise "The following configured #{component_kind} are not allowed: #{disallowed_components}. Allowed: #{allowed_components}." unless disallowed_components.empty?
+end
+
 check_for_new_and_old_properties!
 check_for_no_exporters!
 check_for_no_service_config!
 check_for_use_of_reserved_prefix!
 check_for_use_of_reserved_bbs_api_port!
+check_for_use_of_valid_components!('processors', included_processors, retrieve_property('allow_list.processors'))
+check_for_use_of_valid_components!('exporters', included_exporters, retrieve_property('allow_list.exporters'))
 set_internal_receiver_as_only_receiver
 set_internal_receiver_on_all_pipelines
 expose_internal_telemetry

spec/jobs/otel-collector_spec.rb

@@ -5,7 +5,8 @@
 require 'support/shared_examples_for_otel_collector'
 
 describe 'otel-collector' do
-  let(:release) { Bosh::Template::Test::ReleaseDir.new(File.join(File.dirname(__FILE__), '../..')) }
+  let(:release_dir) { File.join(File.dirname(__FILE__), '../..') }
+  let(:release) { Bosh::Template::Test::ReleaseDir.new(release_dir) }
   let(:job) { release.job('otel-collector') }
   let(:config_path) { '/var/vcap/jobs/otel-collector/config' }
 

spec/support/shared_examples_for_otel_collector.rb

@@ -182,26 +182,113 @@ def without_internal_telemetry(cfg)
     end
 
     describe 'processors' do
+      it 'list of available processors matches builder source of truth' do
+        config['processors']['unavailable'] = nil
+
+        builder_config = YAML.load_file(File.join(release_dir, "src/otel-collector-builder/config.yaml"))
+        processor_gomods = builder_config.fetch('processors').map {|entry| entry.fetch('gomod').split(" ")[0]}
+        processor_names = processor_gomods.map do |gomod|
+          YAML.load_file(File.join(release_dir, "src/otel-collector/vendor", gomod, "metadata.yaml")).fetch('type')
+        end
+        formatted_names = processor_names.sort.map {|name| "\"#{name}\"" }.join(", ")
+
+        expect { rendered }.to raise_error do |error|
+          expect(error.message).to include("Available: [#{formatted_names}]")
+        end
+      end
+
       it 'includes the configured processors in the config' do
         expect(rendered.keys).to include 'processors'
         expect(rendered['processors']).to eq(config['processors'])
       end
+
+      it 'includes the configured processors even if their names contain `/`' do
+        config['processors']['batch/bar'] = nil
+        expect(rendered.keys).to include 'processors'
+        expect(rendered['processors']).to eq(config['processors'])
+      end
+
       context 'when a processor uses the reserved namespace' do
         before do
           config['processors']['batch/cf-internal-foo'] = nil
         end
+
         it 'raises an error' do
           expect { rendered }.to raise_error(/Processors cannot be defined under cf-internal namespace/)
         end
       end
+
+      it 'errors when a configured processor is not allowed' do
+        properties['allow_list'] = {'processors' => ['memory_limiter']}
+        expect { rendered }.to raise_error(/The following configured processors are not allowed: \["batch"\]/)
+      end
+
+      it 'allows all processors with empty allow list' do
+        properties['allow_list'] = {'processors' => [] }
+        expect(rendered.keys).to include 'processors'
+        expect(rendered['processors']).to eq(config['processors'])
+      end
+
+      it 'errors when an unrecognized processor is in allow list' do
+        properties['allow_list'] = {'processors' => ['memory_limiter', 'unrecognized-processor']}
+        expect { rendered }.to raise_error(/The following processors specified in the allow list are not included in this OpenTelemetry Collector distribution: \["unrecognized-processor"\]/)
+      end
+
+      it 'errors when an unavailable processor is configured' do
+        config['processors']['unavailable'] = nil
+        expect { rendered }.to raise_error(/The following configured processors are not included in this OpenTelemetry Collector distribution: \["unavailable"\]/)
+      end
     end
 
     describe 'exporters' do
+      it 'list of available exporters matches builder source of truth' do
+        config['exporters']['unavailable'] = nil
+
+        builder_config = YAML.load_file(File.join(release_dir, "src/otel-collector-builder/config.yaml"))
+        exporter_gomods = builder_config.fetch('exporters').map {|entry| entry.fetch('gomod').split(" ")[0]}
+        exporter_names = exporter_gomods.map do |gomod|
+          YAML.load_file(File.join(release_dir, "src/otel-collector/vendor", gomod, "metadata.yaml")).fetch('type')
+        end
+        formatted_names = exporter_names.sort.map {|name| "\"#{name}\"" }.join(", ")
+
+        expect { rendered }.to raise_error do |error|
+          expect(error.message).to include("Available: [#{formatted_names}]")
+        end
+      end
+
       it 'includes the configured exporters in the config' do
         expect(rendered.keys).to include 'exporters'
         expect(rendered['exporters']).to eq(config['exporters'])
       end
 
+      it 'errors when a configured exporters is not allowed' do
+        properties['allow_list'] = {'exporters' => ['prometheus']}
+        expect { rendered }.to raise_error(/The following configured exporters are not allowed: \["otlp"\]/)
+      end
+
+      it 'allows all exporters with empty allow list' do
+        properties['allow_list'] = {'exporters' => []}
+        expect(rendered.keys).to include 'exporters'
+        expect(rendered['exporters']).to eq(config['exporters'])
+      end
+
+      it 'includes the configured exporters even if their names contain `/`' do
+        config['exporters']['otlp/bar'] = nil
+        expect(rendered.keys).to include 'exporters'
+        expect(rendered['exporters']).to eq(config['exporters'])
+      end
+
+      context 'when unsupported exporter is provided' do
+        it 'raises unrecognized exporter error' do
+          properties['allow_list'] = {'exporters' => ['unrecognized-exporter']}
+          expect { rendered }.to raise_error(/The following exporters specified in the allow list are not included in this OpenTelemetry Collector distribution/)
+        end
+        it 'raises not allowed error' do
+          config['exporters']['another-unrecognized-exporter/bar'] = nil
+          expect { rendered }.to raise_error(/The following configured exporters are not included in this OpenTelemetry Collector distribution/)
+        end
+      end
+
       context 'when there is a prometheus exporter listening on 8889' do
         before do
           config['exporters']['prometheus/tls'] = {

src/otel-collector-builder/config.yaml

@@ -12,5 +12,6 @@ exporters:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.104.0
 processors:
   - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.104.0
+  - gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.104.0
 receivers:
   - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.104.0

src/otel-collector/go.mod

@@ -23,6 +23,7 @@ require (
 	go.opentelemetry.io/collector/otelcol v0.104.0
 	go.opentelemetry.io/collector/processor v0.104.0
 	go.opentelemetry.io/collector/processor/batchprocessor v0.104.0
+	go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.104.0
 	go.opentelemetry.io/collector/receiver v0.104.0
 	go.opentelemetry.io/collector/receiver/otlpreceiver v0.104.0
 	golang.org/x/sys v0.21.0

src/otel-collector/go.sum

@@ -409,6 +409,8 @@ go.opentelemetry.io/collector/processor v0.104.0 h1:KSvMDu4DWmK1/k2z2rOzMtTvAa00
 go.opentelemetry.io/collector/processor v0.104.0/go.mod h1:qU2/xCCYdvVORkN6aq0H/WUWkvo505VGYg2eOwPvaTg=
 go.opentelemetry.io/collector/processor/batchprocessor v0.104.0 h1:6xXvHYkPjwM1zdzliDM2H/omTGgIOkY96JTCln7CFZQ=
 go.opentelemetry.io/collector/processor/batchprocessor v0.104.0/go.mod h1:f1VfVdiOlqtJDAvQy8YONEee19nJ3haxNeiMPy59w8M=
+go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.104.0 h1:bfxUNxP2i41Dpdp5cXwVuh4ZIQ8g6e4NDnu5HakWQw4=
+go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.104.0/go.mod h1:2HtP0f+EBu99Uq07JF20fa2FKAsjnIieOZ4f9Jysfpc=
 go.opentelemetry.io/collector/receiver v0.104.0 h1:URL1ExkYYd+qbndm7CdGvI2mxzsv/pNfmwJ+1QSQ9/o=
 go.opentelemetry.io/collector/receiver v0.104.0/go.mod h1:+enTCZQLf6dRRANWvykXEzrlRw2JDppXJtoYWd/Dd54=
 go.opentelemetry.io/collector/receiver/otlpreceiver v0.104.0 h1:t9cACuSc7kY09guws7VyB/z9QnG7/zWLC1NQ29WH4+o=