Log when dropping metrics due to missing process_start_time_seconds (#1921)

Report error via obsreport.EndMetricsReceiveOp and return error in transaction Commit()

Fixes #969

Author: nilebox

receiver/prometheusreceiver/internal/transaction.go

@@ -47,6 +47,7 @@ const (
 var errMetricNameNotFound = errors.New("metricName not found from labels")
 var errTransactionAborted = errors.New("transaction aborted")
 var errNoJobInstance = errors.New("job or instance cannot be found from labels")
+var errNoStartTimeMetrics = errors.New("process_start_time_seconds metric is missing")
 
 // A transaction is corresponding to an individual scrape operation or stale report.
 // That said, whenever prometheus receiver scrapped a target metric endpoint a page of raw metrics is returned,
@@ -157,12 +158,17 @@ func (tr *transaction) Commit() error {
 	}
 
 	if tr.useStartTimeMetric {
-		// AdjustStartTime - startTime has to be non-zero in this case.
+		// startTime is mandatory in this case, but may be zero when the
+		// process_start_time_seconds metric is missing from the target endpoint.
 		if tr.metricBuilder.startTime == 0.0 {
-			metrics = []*metricspb.Metric{}
-		} else {
-			adjustStartTime(tr.metricBuilder.startTime, metrics)
+			// Since we are unable to adjust metrics properly, we will drop them
+			// and return an error.
+			err = errNoStartTimeMetrics
+			obsreport.EndMetricsReceiveOp(ctx, dataformat, 0, 0, err)
+			return err
 		}
+
+		adjustStartTime(tr.metricBuilder.startTime, metrics)
 	} else {
 		// AdjustMetrics - jobsMap has to be non-nil in this case.
 		// Note: metrics could be empty after adjustment, which needs to be checked before passing it on to ConsumeMetricsData()

receiver/prometheusreceiver/internal/transaction_test.go

@@ -131,6 +131,21 @@ func Test_transaction(t *testing.T) {
 		// assert.Len(t, ocmds[0].Metrics, 1)
 	})
 
+	t.Run("Error when start time is zero", func(t *testing.T) {
+		sink := new(exportertest.SinkMetricsExporter)
+		tr := newTransaction(context.Background(), nil, true, "", rn, ms, sink, testLogger)
+		if _, got := tr.Add(goodLabels, time.Now().Unix()*1000, 1.0); got != nil {
+			t.Errorf("expecting error == nil from Add() but got: %v\n", got)
+		}
+		tr.metricBuilder.startTime = 0 // zero value means the start time metric is missing
+		got := tr.Commit()
+		if got == nil {
+			t.Error("expecting error from Commit() but got nil")
+		} else if got.Error() != errNoStartTimeMetrics.Error() {
+			t.Errorf("expected error %q but got %q", errNoStartTimeMetrics, got)
+		}
+	})
+
 	t.Run("Drop NaN value", func(t *testing.T) {
 		sink := new(exportertest.SinkMetricsExporter)
 		tr := newTransaction(context.Background(), nil, true, "", rn, ms, sink, testLogger)