Use internal pipelines for collector prometheus metrics (#469)

Author: rmfitzpatrick

cmd/otelcol/config/collector/agent_config.yaml

@@ -65,7 +65,7 @@ receivers:
         endpoint: 0.0.0.0:55681
   # This section is used to collect the OpenTelemetry Collector metrics
   # Even if just a Splunk APM customer, these metrics are included
-  prometheus:
+  prometheus/internal:
     config:
       scrape_configs:
       - job_name: 'otel-collector'
@@ -100,12 +100,17 @@ processors:
     ballast_size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     check_interval: 2s
     limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
-  ## detect if the collector is running on a cloud system
-  ## important for creating unique cloud provider dimensions
+  # detect if the collector is running on a cloud system
+  # important for creating unique cloud provider dimensions
   resourcedetection:
     detectors: [system, gce, ecs, ec2, azure]
     override: false
 
+  # Same as above but overrides resource attributes set by receivers
+  resourcedetection/internal:
+    detectors: [system, gce, ecs, ec2, azure]
+    override: true
+
   # Optional: The following processor can be used to add a default "deployment.environment" attribute to the logs and 
   # traces when it's not populated by instrumentation libraries.
   # If enabled, make sure to enable this processor in the pipeline below.
@@ -157,11 +162,17 @@ service:
       # Use instead when sending to gateway
       #exporters: [otlp, signalfx]
     metrics:
-      receivers: [hostmetrics, otlp, prometheus, signalfx, smartagent/signalfx-forwarder]
+      receivers: [hostmetrics, otlp, signalfx, smartagent/signalfx-forwarder]
       processors: [memory_limiter, batch, resourcedetection]
       exporters: [signalfx]
       # Use instead when sending to gateway
       #exporters: [otlp]
+    metrics/internal:
+      receivers: [prometheus/internal]
+      processors: [memory_limiter, batch, resourcedetection/internal]
+      exporters: [signalfx]
+      # Use instead when sending to gateway
+      #exporters: [otlp]
     logs/signalfx:
       receivers: [signalfx]
       processors: [memory_limiter, batch]

cmd/otelcol/config/collector/full_config_linux.yaml

@@ -127,7 +127,7 @@ receivers:
   # Enables the prometheus receiver -- should only be used for local metric collection
   # This section is used to collect the OpenTelemetry Collector metrics
   # Even if just a Splunk APM customer, these metrics are included
-  prometheus:
+  prometheus/internal:
     config:
       scrape_configs:
       - job_name: 'otel-collector'
@@ -209,6 +209,12 @@ processors:
     # # determines if existing resource attributes should be overridden or preserved, defaults to true
     #override: <bool>
 
+  # detect if the collector is running on a cloud system.
+  # overrides resource attributes set by receivers
+  resourcedetection/internal:
+    detectors: [system, gce, ecs, ec2, azure]
+    override: true
+
   # Enables the memory limiter processor with default settings
   # Full configuration here: https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/memorylimiter
   # Enabling the memory_limiter is strongly recommended for every pipeline.
@@ -554,9 +560,13 @@ service:
       #- resource/add_environment
       exporters: [sapm, signalfx]
     metrics:
-      receivers: [otlp, prometheus, signalfx, smartagent/signalfx-forwarder]
+      receivers: [otlp, signalfx, smartagent/signalfx-forwarder]
       processors: [memory_limiter, batch]
       exporters: [signalfx]
+    metrics/internal:
+      receivers: [prometheus/internal]
+      processors: [memory_limiter, batch, resourcedetection/internal]
+      exporters: [signalfx]
     logs:
       receivers: [signalfx]
       processors: [memory_limiter, batch]

cmd/otelcol/config/collector/gateway_config.yaml

@@ -31,7 +31,7 @@ receivers:
         endpoint: 0.0.0.0:55681
   # This section is used to collect the OpenTelemetry Collector metrics
   # Even if just a Splunk APM customer, these metrics are included
-  prometheus:
+  prometheus/internal:
     config:
       scrape_configs:
       - job_name: 'otel-collector'
@@ -73,6 +73,12 @@ processors:
         #value: staging/production/...
         #key: deployment.environment
 
+  # detect if the collector is running on a cloud system.
+  # overrides resource attributes set by receivers
+  resourcedetection/internal:
+    detectors: [system, gce, ecs, ec2, azure]
+    override: true
+
 exporters:
   # Traces
   sapm:
@@ -101,9 +107,13 @@ service:
       #- resource/add_environment
       exporters: [sapm, signalfx]
     metrics:
-      receivers: [otlp, signalfx, prometheus]
+      receivers: [otlp, signalfx]
       processors: [memory_limiter, batch]
       exporters: [signalfx]
+    metrics/internal:
+      receivers: [prometheus/internal]
+      processors: [memory_limiter, batch, resourcedetection/internal]
+      exporters: [signalfx]
     logs:
       receivers: [signalfx]
       processors: [memory_limiter, batch]

cmd/otelcol/config/collector/otlp_config_linux.yaml

@@ -20,7 +20,7 @@ receivers:
         endpoint: 0.0.0.0:55681
   # This section is used to collect the OpenTelemetry Collector metrics
   # Even if just a Splunk APM customer, these metrics are included
-  prometheus:
+  prometheus/internal:
     config:
       scrape_configs:
       - job_name: 'otel-collector'
@@ -65,6 +65,12 @@ processors:
         #value: staging/production/...
         #key: deployment.environment
 
+  # detect if the collector is running on a cloud system.
+  # overrides resource attributes set by receivers
+  resourcedetection/internal:
+    detectors: [system, gce, ecs, ec2, azure]
+    override: true
+
 exporters:
   # Traces
   otlphttp:
@@ -102,9 +108,13 @@ service:
       #- resource/add_environment
       exporters: [otlphttp, signalfx]
     metrics:
-      receivers: [otlp, prometheus, signalfx, smartagent/signalfx-forwarder]
+      receivers: [otlp, signalfx, smartagent/signalfx-forwarder]
       processors: [memory_limiter, batch]
       exporters: [signalfx]
+    metrics/internal:
+      receivers: [prometheus/internal]
+      processors: [memory_limiter, batch, resourcedetection/internal]
+      exporters: [signalfx]
     logs:
       receivers: [signalfx]
       processors: [memory_limiter, batch]