[receiver/splunkenterprisereceiver] Add splunk.indexer.rollingrestart.status metric (#39023)

…eceiver

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description
Add `splunk.indexer.rollingrestart.status` to splunkenterprisereceiver

<!-- Issue number (e.g. #1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes

<!--Describe what testing was performed and which tests were added.-->
#### Testing

<!--Describe the documentation added.-->
#### Documentation

<!--Please delete paragraphs that you did not use before submitting.-->

---------

Co-authored-by: Antoine Toulme <[email protected]>

Author: juanhotencoding

.chloggen/add-splunk-rolling-restart-metric.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'enhancement'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: splunkenterprisereceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "adds `splunk.indexer.rollingrestart.status` metric"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [39023]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

receiver/splunkenterprisereceiver/documentation.md

@@ -251,6 +251,23 @@ Gauge tracking the number of raw write seconds per instance
 | splunk.splunkd.build | The build number for this Splunk instance version | Any Str |
 | splunk.splunkd.version | The splunkd version number | Any Str |
 
+### splunk.indexer.rollingrestart.status
+
+The status of a rolling restart.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| {status} | Gauge | Int |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| splunk.searchable.restart | Boolean that indicates if a searchable rolling restart/upgrade in progress.t | Any Bool |
+| splunk.rollingorrestart | Boolean that indicates if there is a rolling restart or rolling upgrade in progress. | Any Bool |
+| splunk.splunkd.build | The build number for this Splunk instance version | Any Str |
+| splunk.splunkd.version | The splunkd version number | Any Str |
+
 ### splunk.indexer.throughput
 
 Gauge tracking average bytes per second throughput of indexer. *Note:** Must be pointed at specific indexer `endpoint` and gathers metrics from only that indexer.

receiver/splunkenterprisereceiver/internal/metadata/generated_config.go

@@ -29,6 +29,8 @@ all_set:
       enabled: true
     splunk.indexer.raw.write.time:
       enabled: true
+    splunk.indexer.rollingrestart.status:
+      enabled: true
     splunk.indexer.throughput:
       enabled: true
     splunk.indexes.avg.size:
@@ -121,6 +123,8 @@ none_set:
       enabled: false
     splunk.indexer.raw.write.time:
       enabled: false
+    splunk.indexer.rollingrestart.status:
+      enabled: false
     splunk.indexer.throughput:
       enabled: false
     splunk.indexes.avg.size:

receiver/splunkenterprisereceiver/internal/metadata/generated_config_test.go

@@ -54,6 +54,12 @@ attributes:
   splunk.search.state:
     description: The dispatch state of the search
     type: string
+  splunk.searchable.restart:
+    description: Boolean that indicates if a searchable rolling restart/upgrade in progress.t
+    type: bool
+  splunk.rollingorrestart:
+    description: Boolean that indicates if there is a rolling restart or rolling upgrade in progress.
+    type: bool
 
 metrics:
   splunk.license.index.usage:
@@ -368,6 +374,14 @@ metrics:
     gauge:
       value_type: int
     attributes: [splunk.feature, splunk.feature.health, splunk.splunkd.build, splunk.splunkd.version]
+  # `services/cluster/manager/status`
+  splunk.indexer.rollingrestart.status:
+    enabled: false
+    description: The status of a rolling restart.
+    unit: "{status}"
+    gauge:
+      value_type: int
+    attributes: [splunk.searchable.restart, splunk.rollingorrestart, splunk.splunkd.build, splunk.splunkd.version]
 
   # Search Metrics
   splunk.search.duration:

receiver/splunkenterprisereceiver/internal/metadata/generated_metrics.go

@@ -106,6 +106,7 @@ func (s *splunkScraper) scrape(ctx context.Context) (pmetric.Metrics, error) {
 		s.scrapeSearchArtifacts,
 		s.scrapeHealth,
 		s.scrapeSearch,
+		s.scrapeIndexerClusterManagerStatus,
 	}
 	errChan := make(chan error, len(metricScrapes))
 
@@ -2060,3 +2061,39 @@ func (s *splunkScraper) setSearchJobTTLByID(sid string) error {
 
 	return nil
 }
+
+// Scrape Indexer Cluster Manger Status Endpoint
+func (s *splunkScraper) scrapeIndexerClusterManagerStatus(_ context.Context, now pcommon.Timestamp, info infoDict, errs chan error) {
+	if !s.conf.Metrics.SplunkIndexerRollingrestartStatus.Enabled {
+		return
+	}
+
+	i := info[typeCm].Entries[0].Content
+
+	ept := apiDict[`SplunkIndexerClusterManagerStatus`]
+	var icms indexersClusterManagerStatus
+
+	req, err := s.splunkClient.createAPIRequest(typeCm, ept)
+	if err != nil {
+		errs <- err
+		return
+	}
+	res, err := s.splunkClient.makeRequest(req)
+	if err != nil {
+		errs <- err
+		return
+	}
+	defer res.Body.Close()
+
+	if err := json.NewDecoder(res.Body).Decode(&icms); err != nil {
+		errs <- err
+		return
+	}
+
+	for _, ic := range icms.Entries {
+		if ic.Content.RollingRestartOrUpgrade {
+			s.mb.RecordSplunkIndexerRollingrestartStatusDataPoint(now, 1, ic.Content.SearchableRolling, ic.Content.RollingRestartFlag, i.Build, i.Version)
+		}
+		s.mb.RecordSplunkIndexerRollingrestartStatusDataPoint(now, 0, ic.Content.SearchableRolling, ic.Content.RollingRestartFlag, i.Build, i.Version)
+	}
+}