Skip to content

Support SIGHUP configuration reloading #6000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 10, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .chloggen/sighup-configuration-reloading.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
component: service/collector

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: "Support SIGHUP configuration reloading"

# One or more tracking issues or pull requests related to the change
issues: [5966]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:
37 changes: 28 additions & 9 deletions service/collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,21 @@ func (col *Collector) setupConfigurationComponents(ctx context.Context) error {
return nil
}

func (col *Collector) reloadConfiguration(ctx context.Context) error {
col.service.telemetrySettings.Logger.Warn("Config updated, restart service")
col.setCollectorState(Closing)

if err := col.service.Shutdown(ctx); err != nil {
return fmt.Errorf("failed to shutdown the retiring config: %w", err)
}

if err := col.setupConfigurationComponents(ctx); err != nil {
return fmt.Errorf("failed to setup configuration components: %w", err)
}

return nil
}

// Run starts the collector according to the given configuration, and waits for it to complete.
// Consecutive calls to Run are not allowed, Run shouldn't be called once a collector is shut down.
func (col *Collector) Run(ctx context.Context) error {
Expand All @@ -163,6 +178,9 @@ func (col *Collector) Run(ctx context.Context) error {
return err
}

// Always notify with SIGHUP for configuration reloading.
signal.Notify(col.signalsChannel, syscall.SIGHUP)

// Only notify with SIGTERM and SIGINT if graceful shutdown is enabled.
if !col.set.DisableGracefulShutdown {
signal.Notify(col.signalsChannel, os.Interrupt, syscall.SIGTERM)
Expand All @@ -177,21 +195,22 @@ LOOP:
break LOOP
}

col.service.telemetrySettings.Logger.Warn("Config updated, restart service")
col.setCollectorState(Closing)

if err = col.service.Shutdown(ctx); err != nil {
return fmt.Errorf("failed to shutdown the retiring config: %w", err)
}
if err = col.setupConfigurationComponents(ctx); err != nil {
return fmt.Errorf("failed to setup configuration components: %w", err)
if err = col.reloadConfiguration(ctx); err != nil {
return err
}
case err := <-col.asyncErrorChannel:
col.service.telemetrySettings.Logger.Error("Asynchronous error received, terminating process", zap.Error(err))
break LOOP
case s := <-col.signalsChannel:
col.service.telemetrySettings.Logger.Info("Received signal from OS", zap.String("signal", s.String()))
break LOOP
switch s {
case syscall.SIGHUP:
if err := col.reloadConfiguration(ctx); err != nil {
return err
}
default:
break LOOP
}
Comment on lines +206 to +213
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a simple if correct?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I was considering future support for other signals.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So please use if for the moment :)

case <-col.shutdownChan:
col.service.telemetrySettings.Logger.Info("Received shutdown request")
break LOOP
Expand Down
6 changes: 6 additions & 0 deletions service/collector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,12 @@ func TestCollectorSendSignal(t *testing.T) {
return Running == col.GetState()
}, 2*time.Second, 200*time.Millisecond)

col.signalsChannel <- syscall.SIGHUP

assert.Eventually(t, func() bool {
return Running == col.GetState()
}, 2*time.Second, 200*time.Millisecond)

col.signalsChannel <- syscall.SIGTERM

wg.Wait()
Expand Down