From 8ba7805d48157f5eecef68adf1c7390cb04d1b8d Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Mon, 16 Dec 2024 18:24:03 +0100 Subject: [PATCH 1/2] Change pprof labels to be prometheus compatible Enables scrapping pprof endpoint for continuous profiling --- modules/graceful/manager.go | 6 +++--- modules/graceful/manager_common.go | 12 ++++++++---- modules/process/context.go | 2 +- modules/process/manager.go | 4 ++-- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/modules/graceful/manager.go b/modules/graceful/manager.go index 991b2f2b7af1e..cac6ce62d63cb 100644 --- a/modules/graceful/manager.go +++ b/modules/graceful/manager.go @@ -136,7 +136,7 @@ func (g *Manager) doShutdown() { } g.lock.Lock() g.shutdownCtxCancel() - atShutdownCtx := pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "post-shutdown")) + atShutdownCtx := pprof.WithLabels(g.hammerCtx, pprof.Labels(LifecyclePProfLabel, "post-shutdown")) pprof.SetGoroutineLabels(atShutdownCtx) for _, fn := range g.toRunAtShutdown { go fn() @@ -167,7 +167,7 @@ func (g *Manager) doHammerTime(d time.Duration) { default: log.Warn("Setting Hammer condition") g.hammerCtxCancel() - atHammerCtx := pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "post-hammer")) + atHammerCtx := pprof.WithLabels(g.terminateCtx, pprof.Labels(LifecyclePProfLabel, "post-hammer")) pprof.SetGoroutineLabels(atHammerCtx) } g.lock.Unlock() @@ -183,7 +183,7 @@ func (g *Manager) doTerminate() { default: log.Warn("Terminating") g.terminateCtxCancel() - atTerminateCtx := pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "post-terminate")) + atTerminateCtx := pprof.WithLabels(g.managerCtx, pprof.Labels(LifecyclePProfLabel, "post-terminate")) pprof.SetGoroutineLabels(atTerminateCtx) for _, fn := range g.toRunAtTerminate { diff --git a/modules/graceful/manager_common.go b/modules/graceful/manager_common.go index f6dbcc748dc80..de303eba7ce29 100644 --- a/modules/graceful/manager_common.go +++ b/modules/graceful/manager_common.go @@ -22,6 +22,10 @@ const ( watchdogMsg systemdNotifyMsg = "WATCHDOG=1" ) +// LifecyclePProfLabel is a label marking manager lifecycle phase +// Compliant with prometheus key regex +const LifecyclePProfLabel = "graceful_lifecycle" + func statusMsg(msg string) systemdNotifyMsg { return systemdNotifyMsg("STATUS=" + msg) } @@ -65,10 +69,10 @@ func (g *Manager) prepare(ctx context.Context) { g.hammerCtx, g.hammerCtxCancel = context.WithCancel(ctx) g.managerCtx, g.managerCtxCancel = context.WithCancel(ctx) - g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "with-terminate")) - g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels("graceful-lifecycle", "with-shutdown")) - g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "with-hammer")) - g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "with-manager")) + g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels(LifecyclePProfLabel, "with-terminate")) + g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels(LifecyclePProfLabel, "with-shutdown")) + g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels(LifecyclePProfLabel, "with-hammer")) + g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels(LifecyclePProfLabel, "with-manager")) if !g.setStateTransition(stateInit, stateRunning) { panic("invalid graceful manager state: transition from init to running failed") diff --git a/modules/process/context.go b/modules/process/context.go index 26a80ebd6207c..1854988bce2ba 100644 --- a/modules/process/context.go +++ b/modules/process/context.go @@ -32,7 +32,7 @@ func (c *Context) Value(key any) any { } // ProcessContextKey is the key under which process contexts are stored -var ProcessContextKey any = "process-context" +var ProcessContextKey any = "process_context" // GetContext will return a process context if one exists func GetContext(ctx context.Context) *Context { diff --git a/modules/process/manager.go b/modules/process/manager.go index bdc4931810d1c..7b8ada786ea42 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -26,7 +26,7 @@ var ( ) // DescriptionPProfLabel is a label set on goroutines that have a process attached -const DescriptionPProfLabel = "process-description" +const DescriptionPProfLabel = "process_description" // PIDPProfLabel is a label set on goroutines that have a process attached const PIDPProfLabel = "pid" @@ -35,7 +35,7 @@ const PIDPProfLabel = "pid" const PPIDPProfLabel = "ppid" // ProcessTypePProfLabel is a label set on goroutines that have a process attached -const ProcessTypePProfLabel = "process-type" +const ProcessTypePProfLabel = "process_type" // IDType is a pid type type IDType string From 60ae54f119eed8f1caff180a26b524da3f189a8d Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 17 Dec 2024 09:54:24 +0100 Subject: [PATCH 2/2] Reword comment --- modules/graceful/manager_common.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/graceful/manager_common.go b/modules/graceful/manager_common.go index de303eba7ce29..15dd4406d532a 100644 --- a/modules/graceful/manager_common.go +++ b/modules/graceful/manager_common.go @@ -23,7 +23,9 @@ const ( ) // LifecyclePProfLabel is a label marking manager lifecycle phase -// Compliant with prometheus key regex +// Making it compliant with prometheus key regex https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels +// would enable someone interested to be able to to continuously gather profiles into pyroscope. +// Other labels for pprof (in "modules/process" package) should also follow this rule. const LifecyclePProfLabel = "graceful_lifecycle" func statusMsg(msg string) systemdNotifyMsg {