Replace the SAPM exporter with OTLP HTTP in default configs

.github/workflows/scripts/win-test-services.ps1

@@ -44,7 +44,8 @@ $expected_svc_env_vars = @{
   "SPLUNK_REALM"            = "$realm";
   "SPLUNK_API_URL"          = "$api_url";
   "SPLUNK_INGEST_URL"       = "$ingest_url";
-  "SPLUNK_TRACE_URL"        = "${ingest_url}/v2/trace";
+  # We no longer test for this as we have moved to OTLP and it breaks upgrade tests.
+  #"SPLUNK_TRACE_URL"        = "${ingest_url}/v2/trace/otlp";
   "SPLUNK_HEC_URL"          = "${ingest_url}/v1/log";
   "SPLUNK_HEC_TOKEN"        = "$access_token";
   "SPLUNK_BUNDLE_DIR"       = "${env:PROGRAMFILES}\Splunk\OpenTelemetry Collector\agent-bundle";

cmd/otelcol/config/collector/agent_config.yaml

@@ -10,7 +10,7 @@
 # - SPLUNK_HEC_URL: The Splunk HEC endpoint URL, e.g. https://ingest.us0.signalfx.com/v1/log
 # - SPLUNK_INGEST_URL: The Splunk ingest URL, e.g. https://ingest.us0.signalfx.com
 # - SPLUNK_LISTEN_INTERFACE: The network interface the agent receivers listen on.
-# - SPLUNK_TRACE_URL: The Splunk trace endpoint URL, e.g. https://ingest.us0.signalfx.com/v2/trace
+# - SPLUNK_TRACE_URL: The Splunk trace endpoint URL, e.g. https://ingest.us0.signalfx.com/v2/trace/otlp
 
 extensions:
   health_check:
@@ -129,7 +129,7 @@ exporters:
   # Traces
   sapm:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
-    endpoint: "${SPLUNK_TRACE_URL}"
+    endpoint: "${SPLUNK_INGEST_URL}/v2/trace"
   # Metrics + Events
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
@@ -157,8 +157,13 @@ exporters:
     token: "${SPLUNK_ACCESS_TOKEN}"
     endpoint: "${SPLUNK_INGEST_URL}/v1/log"
     log_data_enabled: false
+  otlphttp:
+    metrics_endpoint: "${SPLUNK_INGEST_URL}/v2/datapoint/otlp"
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
   # Send to gateway
-  otlp:
+  otlp/gateway:
     endpoint: "${SPLUNK_GATEWAY_URL}:4317"
     tls:
       insecure: true
@@ -179,15 +184,15 @@ service:
       - batch
       - resourcedetection
       #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp, signalfx]
+      #exporters: [otlp/gateway, signalfx]
     metrics:
       receivers: [hostmetrics, otlp, signalfx]
       processors: [memory_limiter, batch, resourcedetection]
       exporters: [signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]
     metrics/internal:
       receivers: [prometheus/internal]
       processors: [memory_limiter, batch, resourcedetection]
@@ -212,4 +217,4 @@ service:
       #- resource/add_environment
       exporters: [splunk_hec, splunk_hec/profiling]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]

cmd/otelcol/config/collector/ecs_ec2_config.yaml

@@ -129,6 +129,11 @@ exporters:
     token: "${SPLUNK_ACCESS_TOKEN}"
     endpoint: "${SPLUNK_INGEST_URL}/v1/log"
     log_data_enabled: false
+  otlphttp:
+    metrics_endpoint: "${SPLUNK_INGEST_URL}/v2/datapoint/otlp"
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 
 service:
   extensions: [health_check, http_forwarder, zpages]
@@ -140,7 +145,7 @@ service:
         - batch
         - resourcedetection
         #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
     metrics:
       receivers: [hostmetrics, otlp, signalfx, awsecscontainermetrics]
       processors: [memory_limiter, batch, filter, resourcedetection]

cmd/otelcol/config/collector/fargate_config.yaml

@@ -112,6 +112,11 @@ exporters:
     token: "${SPLUNK_ACCESS_TOKEN}"
     endpoint: "${SPLUNK_INGEST_URL}/v1/log"
     log_data_enabled: false
+  otlphttp:
+    metrics_endpoint: "${SPLUNK_INGEST_URL}/v2/datapoint/otlp"
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 
 service:
   extensions: [health_check, http_forwarder, zpages]
@@ -123,7 +128,7 @@ service:
         - batch
         - resourcedetection
         #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
     metrics:
       receivers: [otlp, signalfx, awsecscontainermetrics, prometheus/internal]
       processors: [memory_limiter, batch, resourcedetection]

cmd/otelcol/config/collector/full_config_linux.yaml

@@ -774,7 +774,7 @@ service:
       - memory_limiter
       - batch
       #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
     metrics:
       receivers: [otlp, signalfx]
       processors: [memory_limiter, batch]

cmd/otelcol/config/collector/gateway_config.yaml

@@ -1,4 +1,4 @@
-# Configuration file that uses the Splunk exporters (SAPM, SignalFx) to push
+# Configuration file that uses the Splunk exporters (OTLPHTTP, SignalFx) to push
 # data to Splunk products.
 
 extensions:
@@ -125,6 +125,11 @@ exporters:
     token: "${SPLUNK_ACCESS_TOKEN}"
     endpoint: "${SPLUNK_INGEST_URL}/v1/log"
     log_data_enabled: false
+  otlphttp:
+    metrics_endpoint: "${SPLUNK_INGEST_URL}/v2/datapoint/otlp"
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 
 service:
   telemetry:
@@ -138,7 +143,7 @@ service:
       - memory_limiter
       - batch
       #- resource/add_environment
-      exporters: [sapm]
+      exporters: [otlphttp]
     metrics:
       receivers: [otlp, signalfx]
       processors: [memory_limiter, batch]

cmd/otelcol/config/collector/upstream_agent_config.yaml

@@ -12,7 +12,7 @@
 # - SPLUNK_HEC_TOKEN: The Splunk HEC authentication token
 # - SPLUNK_HEC_URL: The Splunk HEC endpoint URL, e.g. https://ingest.us0.signalfx.com/v1/log
 # - SPLUNK_INGEST_URL: The Splunk ingest URL, e.g. https://ingest.us0.signalfx.com
-# - SPLUNK_TRACE_URL: The Splunk trace endpoint URL, e.g. https://ingest.us0.signalfx.com/v2/trace
+# - SPLUNK_TRACE_URL: The Splunk trace endpoint URL, e.g. https://ingest.us0.signalfx.com/v2/trace/otlp
 
 extensions:
   health_check:
@@ -143,8 +143,13 @@ exporters:
     token: "${SPLUNK_ACCESS_TOKEN}"
     endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v1/log"
     log_data_enabled: false
+  otlphttp:
+    metrics_endpoint: "${SPLUNK_INGEST_URL}/v2/datapoint/otlp"
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
   # Send to gateway
-  otlp:
+  otlp/gateway:
     endpoint: "${SPLUNK_GATEWAY_URL}:4317"
     tls:
       insecure: true
@@ -163,30 +168,30 @@ service:
       - batch
       - resourcedetection
       #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp, signalfx]
+      #exporters: [otlp/gateway, signalfx]
     # Required for Splunk Infrastructure Monitoring
     metrics:
       receivers: [hostmetrics, otlp, signalfx]
       processors: [memory_limiter, batch, resourcedetection]
       exporters: [signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]
     # Required for Splunk APM and/or Infrastructure Monitoring
     metrics/internal:
       receivers: [prometheus/internal]
       processors: [memory_limiter, batch, resourcedetection]
       exporters: [signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]
     # Required for Splunk Infrastructure Monitoring
     logs/signalfx:
       receivers: [signalfx]
       processors: [memory_limiter, batch]
       exporters: [signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]
     logs/entities:
       # Receivers are dynamically added if discovery mode is enabled
       receivers: [nop]
@@ -202,4 +207,4 @@ service:
       #- resource/add_environment
       exporters: [splunk_hec/profiling]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]

cmd/otelcol/fips/config/agent_config.yaml

@@ -119,7 +119,7 @@ exporters:
   # Traces
   sapm:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
-    endpoint: "${SPLUNK_TRACE_URL}"
+    endpoint: "${SPLUNK_INGEST_URL}/v2/trace"
   # Metrics + Events
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"

deployments/ansible/molecule/custom_vars/verify.yml

@@ -39,7 +39,7 @@
 
     - name: Assert SPLUNK_TRACE_URL env var is set
       ansible.builtin.lineinfile:
-        line: SPLUNK_TRACE_URL=https://fake-splunk-ingest.com/v2/trace
+        line: SPLUNK_TRACE_URL=https://fake-splunk-ingest.com/v2/trace/otlp
         dest: /etc/otel/collector/splunk-otel-collector.conf
         state: present
       check_mode: yes

deployments/ansible/molecule/custom_vars/windows-verify.yml

@@ -8,7 +8,7 @@
       SPLUNK_CONFIG: '{{ ansible_env.ProgramData }}\Splunk\OpenTelemetry Collector\custom_config.yml'
       SPLUNK_INGEST_URL: https://fake-splunk-ingest.com
       SPLUNK_API_URL: https://fake-splunk-api.com
-      SPLUNK_TRACE_URL: https://fake-splunk-ingest.com/v2/trace
+      SPLUNK_TRACE_URL: https://fake-splunk-ingest.com/v2/trace/otlp
       SPLUNK_HEC_URL: https://fake-splunk-hec.com
       SPLUNK_HEC_TOKEN: fake-hec-token
       SPLUNK_MEMORY_TOTAL_MIB: "256"

deployments/ansible/molecule/default/Dockerfile.j2

@@ -28,7 +28,7 @@ RUN zypper -n clean && zypper -n refresh
 RUN zypper -n install -l ansible dbus-1 rpm-python sudo systemd-sysvinit
 {% else %}
 FROM opensuse/leap:15
-RUN sed -i 's|download.opensuse.org|provo-mirror.opensuse.org|' /etc/zypp/repos.d/*.repo
+#RUN sed -i 's|download.opensuse.org|provo-mirror.opensuse.org|' /etc/zypp/repos.d/*.repo
 RUN zypper -n install -l ansible dbus-1 python3-rpm sudo systemd-sysvinit
 {% endif %}
 

deployments/ansible/molecule/default/verify.yml

@@ -45,7 +45,7 @@
 
     - name: Assert SPLUNK_TRACE_URL env var is set
       ansible.builtin.lineinfile:
-        line: SPLUNK_TRACE_URL=https://ingest.fake-realm.signalfx.com/v2/trace
+        line: SPLUNK_TRACE_URL=https://ingest.fake-realm.signalfx.com/v2/trace/otlp
         dest: /etc/otel/collector/splunk-otel-collector.conf
         state: present
       check_mode: yes

deployments/ansible/molecule/default/windows-verify.yml

@@ -12,7 +12,7 @@
       SPLUNK_HEC_TOKEN: fake-token
       SPLUNK_HEC_URL: https://ingest.fake-realm.signalfx.com/v1/log
       SPLUNK_INGEST_URL: https://ingest.fake-realm.signalfx.com
-      SPLUNK_TRACE_URL: https://ingest.fake-realm.signalfx.com/v2/trace
+      SPLUNK_TRACE_URL: https://ingest.fake-realm.signalfx.com/v2/trace/otlp
   tasks:
     - name: Check splunk-otel-collector service
       ansible.windows.win_service:

deployments/ansible/molecule/with_instrumentation/windows-verify.yml

@@ -12,7 +12,7 @@
       SPLUNK_HEC_TOKEN: fake-token
       SPLUNK_HEC_URL: https://ingest.fake-realm.signalfx.com/v1/log
       SPLUNK_INGEST_URL: https://ingest.fake-realm.signalfx.com
-      SPLUNK_TRACE_URL: https://ingest.fake-realm.signalfx.com/v2/trace
+      SPLUNK_TRACE_URL: https://ingest.fake-realm.signalfx.com/v2/trace/otlp
     iis_reg_values:
       COR_ENABLE_PROFILING: "1"
       COR_PROFILER: "{B4C89B0F-9908-4F73-9F59-0D77C5A06874}"

deployments/ansible/roles/collector/README.md

@@ -65,9 +65,9 @@ $> ansible-playbook playbook.yaml -e start_service=false
   collector service. (**default:** `https://api.{{ splunk_realm }}.signalfx.com`)
 
 - `splunk_trace_url`: The Splunk trace endpoint URL, e.g.
-  `https://ingest.us0.signalfx.com/v2/trace`. The `SPLUNK_TRACE_URL` environment
+  `https://ingest.us0.signalfx.com/v2/trace/otlp`. The `SPLUNK_TRACE_URL` environment
   variable will be set with this value for the collector service. (**default:**
-  `{{ splunk_ingest_url }}/v2/trace`)
+  `{{ splunk_ingest_url }}/v2/trace/otlp`)
 
 - `splunk_hec_url`: The Splunk HEC endpoint URL, e.g.
   `https://ingest.us0.signalfx.com/v1/log`. The `SPLUNK_HEC_URL` environment

deployments/ansible/roles/collector/tasks/win_configurable_vars.yml

@@ -11,7 +11,7 @@
 
 - name: Set default trace url
   set_fact:
-    splunk_trace_url: "{{splunk_ingest_url}}/v2/trace"
+    splunk_trace_url: "{{splunk_ingest_url}}/v2/trace/otlp"
   when: splunk_trace_url is not defined or (splunk_trace_url | trim) == ""
 
 - name: Set default hec url

deployments/ansible/roles/collector/templates/splunk-otel-collector.conf.j2

@@ -6,7 +6,7 @@ SPLUNK_INGEST_URL={{ splunk_ingest_url | default("https://ingest." + splunk_real
 {% if splunk_trace_url is defined and splunk_trace_url %}
 SPLUNK_TRACE_URL={{ splunk_trace_url }}
 {% elif splunk_ingest_url is defined and splunk_ingest_url %}
-SPLUNK_TRACE_URL={{ splunk_ingest_url + "/v2/trace" }}
+SPLUNK_TRACE_URL={{ splunk_ingest_url + "/v2/trace/otlp" }}
 {% else %}
 SPLUNK_TRACE_URL={{ "https://ingest." + splunk_realm + ".signalfx.com/v2/trace" }}
 {% endif %}

deployments/chef/attributes/default.rb

@@ -16,7 +16,7 @@
 
 default['splunk_otel_collector']['splunk_api_url'] = "https://api.#{node['splunk_otel_collector']['splunk_realm']}.signalfx.com"
 default['splunk_otel_collector']['splunk_ingest_url'] = "https://ingest.#{node['splunk_otel_collector']['splunk_realm']}.signalfx.com"
-default['splunk_otel_collector']['splunk_trace_url'] = "#{node['splunk_otel_collector']['splunk_ingest_url']}/v2/trace"
+default['splunk_otel_collector']['splunk_trace_url'] = "#{node['splunk_otel_collector']['splunk_ingest_url']}/v2/trace/otlp"
 default['splunk_otel_collector']['splunk_hec_url'] = "#{node['splunk_otel_collector']['splunk_ingest_url']}/v1/log"
 default['splunk_otel_collector']['splunk_hec_token'] = "#{node['splunk_otel_collector']['splunk_access_token']}"
 default['splunk_otel_collector']['splunk_memory_total_mib'] = '512'

deployments/chef/test/integration/custom_vars/test.rb

@@ -2,7 +2,7 @@
 splunk_realm = 'test'
 splunk_api_url = 'https://fake-splunk-api.com'
 splunk_ingest_url = 'https://fake-splunk-ingest.com'
-splunk_trace_url = "#{splunk_ingest_url}/v2/trace"
+splunk_trace_url = "#{splunk_ingest_url}/v2/trace/otlp"
 splunk_hec_url = "#{splunk_ingest_url}/v1/log"
 splunk_hec_token = 'fake-hec-token'
 splunk_memory_total = '256'

deployments/chef/test/integration/default/test.rb

@@ -2,7 +2,7 @@
 splunk_realm = 'test'
 splunk_api_url = "https://api.#{splunk_realm}.signalfx.com"
 splunk_ingest_url = "https://ingest.#{splunk_realm}.signalfx.com"
-splunk_trace_url = "#{splunk_ingest_url}/v2/trace"
+splunk_trace_url = "#{splunk_ingest_url}/v2/trace/otlp"
 splunk_hec_url = "#{splunk_ingest_url}/v1/log"
 splunk_hec_token = splunk_access_token
 splunk_memory_total = '512'

deployments/nomad/otel-agent.nomad

@@ -212,6 +212,11 @@ exporters:
     sync_host_metadata: true
   debug:
     verbosity: detailed
+  otlphttp:
+    metrics_endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/datapoint/otlp"
+    traces_endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 service:
   extensions:
   - health_check

deployments/nomad/otel-gateway.nomad

@@ -194,6 +194,11 @@ exporters:
     ingest_url: https://ingest.${SPLUNK_REALM}.signalfx.com
   debug:
     verbosity: detailed
+  otlphttp:
+    metrics_endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/datapoint/otlp"
+    traces_endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 service:
   extensions:
   - health_check

deployments/puppet/manifests/init.pp

@@ -4,7 +4,7 @@
   $splunk_realm            = '',  # required
   $splunk_ingest_url       = "https://ingest.${splunk_realm}.signalfx.com",
   $splunk_api_url          = "https://api.${splunk_realm}.signalfx.com",
-  $splunk_trace_url        = "${splunk_ingest_url}/v2/trace",
+  $splunk_trace_url        = "${splunk_ingest_url}/v2/trace/otlp",
   $splunk_hec_url          = "${splunk_ingest_url}/v1/log",
   $splunk_hec_token        = $splunk_access_token,
   $splunk_bundle_dir       = $splunk_otel_collector::params::splunk_bundle_dir,

deployments/salt/splunk-otel-collector/collector_config.sls

@@ -10,7 +10,7 @@
 
 {% set splunk_ingest_url = salt['pillar.get']('splunk-otel-collector:splunk_ingest_url', 'https://ingest.' + splunk_realm + '.signalfx.com') %}
 
-{% set splunk_trace_url = salt['pillar.get']('splunk-otel-collector:splunk_trace_url', splunk_ingest_url + '/v2/trace') %}
+{% set splunk_trace_url = salt['pillar.get']('splunk-otel-collector:splunk_trace_url', splunk_ingest_url + '/v2/trace/otlp') %}
 
 {% set splunk_hec_url = salt['pillar.get']('splunk-otel-collector:splunk_hec_url', splunk_ingest_url + '/v1/log') %}
 

internal/settings/settings.go

@@ -390,7 +390,7 @@ func setDefaultEnvVars(s *Settings) error {
 		envVars = append(envVars,
 			ev{e: APIURLEnvVar, v: fmt.Sprintf("https://api.%s.signalfx.com", realm)},
 			ev{e: IngestURLEnvVar, v: fmt.Sprintf("https://ingest.%s.signalfx.com", realm)},
-			ev{e: TraceIngestURLEnvVar, v: fmt.Sprintf("https://ingest.%s.signalfx.com/v2/trace", realm)},
+			ev{e: TraceIngestURLEnvVar, v: fmt.Sprintf("https://ingest.%s.signalfx.com/v2/trace/otlp", realm)},
 			ev{e: HecLogIngestURLEnvVar, v: fmt.Sprintf("https://ingest.%s.signalfx.com/v1/log", realm)},
 		)
 	}