Added support for capturing top most failing span + improved no_capture_reason communication

Author: GreenMatan

tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs

@@ -143,7 +143,7 @@ internal static class Debugger
             /// Configuration key for the maximum number of frames in a call stack we would like to capture values for.
             /// </summary>
             /// <seealso cref="ExceptionReplaySettings.MaximumFramesToCapture"/>
-            public const string ExceptionReplayMaxFramesToCapture = "DD_EXCEPTION_REPLAY_MAX_FRAMES_TO_CAPTURE";
+            public const string ExceptionReplayCaptureMaxFrames = "DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES";
 
             /// <summary>
             /// Configuration key to enable capturing the variables of all the frames in exception call stack.

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayDiagnosticTagNames.cs

@@ -20,7 +20,6 @@ internal static class ExceptionReplayDiagnosticTagNames
         public const string Eligible = nameof(Eligible);
         public const string NotEligible = nameof(NotEligible);
         public const string ExceptionTrackManagerNotInitialized = nameof(ExceptionTrackManagerNotInitialized);
-        public const string NotRootSpan = nameof(NotRootSpan);
         public const string ExceptionObjectIsNull = nameof(ExceptionObjectIsNull);
         public const string NonSupportedExceptionType = nameof(NonSupportedExceptionType);
         public const string CachedDoneExceptionCase = nameof(CachedDoneExceptionCase);

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySettings.cs

@@ -33,7 +33,7 @@ public ExceptionReplaySettings(IConfigurationSource? source, IConfigurationTelem
             CaptureFullCallStack = config.WithKeys(ConfigurationKeys.Debugger.ExceptionReplayCaptureFullCallStackEnabled).AsBool(false);
 
             var maximumFramesToCapture = config
-                                        .WithKeys(ConfigurationKeys.Debugger.ExceptionReplayMaxFramesToCapture)
+                                        .WithKeys(ConfigurationKeys.Debugger.ExceptionReplayCaptureMaxFrames)
                                         .AsInt32(DefaultMaxFramesToCapture, maxDepth => maxDepth > 0)
                                         .Value;
 

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs

@@ -79,15 +79,11 @@ public static void Report(Span span, Exception? exception)
                 return;
             }
 
-            // For V1 of Exception Debugging, we only care about exceptions propagating up the stack
-            // and marked as error by the service entry/root span.
-            if (span.IsRootSpan == false || exception == null || !IsSupportedExceptionType(exception))
+            if (exception == null || !IsSupportedExceptionType(exception))
             {
                 Log.Information(exception, "Skipping the processing of the exception. Exception = {Exception}, Span = {Span}", exception?.ToString(), span.ToString());
 
-                var failureReason =
-                    span.IsRootSpan == false ? ExceptionReplayDiagnosticTagNames.NotRootSpan :
-                                    exception == null ? ExceptionReplayDiagnosticTagNames.ExceptionObjectIsNull : ExceptionReplayDiagnosticTagNames.NonSupportedExceptionType;
+                var failureReason = exception == null ? ExceptionReplayDiagnosticTagNames.ExceptionObjectIsNull : ExceptionReplayDiagnosticTagNames.NonSupportedExceptionType;
                 SetDiagnosticTag(span, failureReason, 0);
                 return;
             }
@@ -323,6 +319,8 @@ private static void ProcessException(Exception exception, int normalizedExHash,
                     }
                     else
                     {
+                        EvaluateWithRootSpanCases.Add(normalizedExHash);
+
                         if (rootSpan != null)
                         {
                             SetDiagnosticTag(rootSpan, ExceptionReplayDiagnosticTagNames.EmptyCallStackTreeWhileCollecting, normalizedExHash);
@@ -759,6 +757,40 @@ private static bool ShouldSkip(MethodBase method)
 
         private static void SetDiagnosticTag(Span span, string exceptionPhase, int exceptionHash)
         {
+            if (exceptionPhase is ExceptionReplayDiagnosticTagNames.CachedInvalidatedExceptionCase or ExceptionReplayDiagnosticTagNames.CachedDoneExceptionCase or ExceptionReplayDiagnosticTagNames.NonCachedDoneExceptionCase)
+            {
+                return;
+            }
+
+            var noCaptureReason = string.Empty;
+
+            switch (exceptionPhase)
+            {
+                case ExceptionReplayDiagnosticTagNames.Eligible:
+                    break;
+                case ExceptionReplayDiagnosticTagNames.NoCustomerFrames or ExceptionReplayDiagnosticTagNames.NoFramesToInstrument:
+                    noCaptureReason = NoCaptureReason.OnlyThirdPartyCode;
+                    break;
+                case ExceptionReplayDiagnosticTagNames.InvalidatedCase or ExceptionReplayDiagnosticTagNames.InvalidatedExceptionCase:
+                    noCaptureReason = NoCaptureReason.InstrumentationFailure;
+                    break;
+                case ExceptionReplayDiagnosticTagNames.NonSupportedExceptionType:
+                    noCaptureReason = NoCaptureReason.NonSupportedExceptionType;
+                    break;
+                case ExceptionReplayDiagnosticTagNames.NotEligible or ExceptionReplayDiagnosticTagNames.NewCase:
+                    noCaptureReason = NoCaptureReason.FirstOccurrence;
+                    break;
+                default:
+                    noCaptureReason = NoCaptureReason.GeneralError;
+                    break;
+            }
+
+            if (noCaptureReason != string.Empty)
+            {
+                span.Tags.SetTag("error.debug_info_captured", "true");
+                span.Tags.SetTag("_dd.debug.error.no_capture_reason", noCaptureReason);
+            }
+
             span.Tags.SetTag("_dd.di._er", exceptionPhase);
             span.Tags.SetTag("_dd.di._eh", exceptionHash.ToString());
         }

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/NoCaptureReason.cs

@@ -0,0 +1,26 @@
+// <copyright file="NoCaptureReason.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+#nullable enable
+namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation
+{
+    /// <summary>
+    /// Assigned as a tag on incoming spans by <see cref="ExceptionTrackManager"/> to designate why certain errors were not captured
+    /// </summary>
+    internal static class NoCaptureReason
+    {
+        public const string OnlyThirdPartyCode = nameof(OnlyThirdPartyCode);
+        public const string InstrumentationFailure = nameof(InstrumentationFailure);
+        public const string NonSupportedExceptionType = nameof(NonSupportedExceptionType);
+        public const string FirstOccurrence = nameof(FirstOccurrence);
+        public const string GeneralError = nameof(GeneralError);
+    }
+}

tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json

@@ -423,6 +423,7 @@
   "DD_TRACE_COMPUTE_STATS": "dd_trace_compute_stats",
   "DD_EXCEPTION_DEBUGGING_ENABLED": "dd_exception_debugging_enabled",
   "DD_EXCEPTION_DEBUGGING_MAX_FRAMES_TO_CAPTURE": "dd_exception_debugging_max_frames_to_capture",
+  "DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES": "dd_exception_replay_capture_max_frames",
   "DD_EXCEPTION_DEBUGGING_CAPTURE_FULL_CALLSTACK_ENABLED": "dd_exception_debugging_capture_full_callstack_enabled",
   "DD_EXCEPTION_DEBUGGING_RATE_LIMIT_SECONDS": "dd_exception_debugging_rate_limit_seconds",
   "DD_EXCEPTION_DEBUGGING_MAX_EXCEPTION_ANALYSIS_LIMIT": "dd_exception_debugging_max_exception_analysis_limit",