From beeebb4bbd9eafdd849ec511ba12fa20fbe708b4 Mon Sep 17 00:00:00 2001 From: Justin Geibel Date: Sat, 16 Mar 2019 16:53:49 -0400 Subject: [PATCH] Add support for graceful shutdown of the server The server process now intercepts SIGINT and SIGTERM to initiate a graceful shutdown of the server. This is useful when tracking memory leaks locally, as both `hyper` and `civet` are given a chance to return memory and shutdown cleanly. However, this will not improve things in production as we also run an instance of `nginx`, which will close all connections after receiving a SIGTERM from Heroku. From my preliminary investigation, it appears we may need to customize the buildpack to change this behavior. Additionally, the server will now briefly sleep before notifying Heroku that it is ready to receive connections. Also, when using `hyper` the `Runtime` is configured to use 4 background threads. This overrides the default, which is one thread per CPU and provides consistency between differently sized dynos. The number of `conduit` background threads are still configured via `SERVER_THREADS` and defaults to 50 in production. --- Cargo.lock | 25 +++++++++++++++ Cargo.toml | 7 +++-- src/bin/server.rs | 78 +++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 96 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ba04e3cede..f2d4f0de3fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,6 +184,7 @@ dependencies = [ "conduit-static 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "conduit-test 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "cookie 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ctrlc 3.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "derive_deref 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "diesel 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "diesel_full_text_search 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -217,6 +218,7 @@ dependencies = [ "swirl 0.1.0 (git+https://github.com/sgrif/swirl.git?rev=de5d8bb)", "tar 0.4.21 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -481,6 +483,15 @@ dependencies = [ "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ctrlc" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nix 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "curl" version = "0.4.12" @@ -1419,6 +1430,18 @@ name = "new_debug_unreachable" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "nix" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nodrop" version = "0.1.13" @@ -2821,6 +2844,7 @@ dependencies = [ "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" "checksum crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c" "checksum ctor 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9a43db2bba5cafdc6aa068c892a518e477ee0df3705e53ec70247a9ff93546d5" +"checksum ctrlc 3.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5531b7f0698d9220b4729f8811931dbe0e91a05be2f7b3245fdc50dd856bae26" "checksum curl 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "aaf20bbe084f285f215eef2165feed70d6b75ba29cad24469badb853a4a287d0" "checksum curl-sys 0.4.16 (registry+https://github.com/rust-lang/crates.io-index)" = "ca79238a79fb294be6173b4057c95b22a718c94c4e38475d5faa82b8383f3502" "checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" @@ -2923,6 +2947,7 @@ dependencies = [ "checksum native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8b0a7bd714e83db15676d31caf968ad7318e9cc35f93c85a90231c8f22867549" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30" +"checksum nix 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46f0f3210768d796e8fa79ec70ee6af172dacbe7147f5e69be5240a47778302b" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9c349f68f25f596b9f44cf0e7c69752a5c633b0550c3ff849518bfba0233774a" "checksum num 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "a311b77ebdc5dd4cf6449d81e4135d9f0e3b153839ac90e648a8ef538f923525" diff --git a/Cargo.toml b/Cargo.toml index 27d99e897b8..75f06cd409f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,11 +77,14 @@ conduit-git-http-backend = "0.8" civet = "0.9" conduit-hyper = "0.1.3" +futures = "0.1" +tokio = "0.1" +hyper = "0.12" +ctrlc = { version = "3.0", features = ["termination"] } + [dev-dependencies] conduit-test = "0.8" -hyper = "0.12" hyper-tls = "0.3" -futures = "0.1" lazy_static = "1.0" tokio-core = "0.1" diff --git a/src/bin/server.rs b/src/bin/server.rs index f74adb5b615..73bfd7d422c 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -1,19 +1,22 @@ #![deny(warnings, clippy::all, rust_2018_idioms)] use cargo_registry::{boot, App, Env}; -use jemalloc_ctl; use std::{ fs::File, - sync::{mpsc::channel, Arc}, + sync::{mpsc::channel, Arc, Mutex}, + thread, + time::Duration, }; use civet::Server as CivetServer; use conduit_hyper::Service as HyperService; +use futures::Future; +use jemalloc_ctl; use reqwest::Client; enum Server { Civet(CivetServer), - Hyper(HyperService), + Hyper(tokio::runtime::Runtime), } use Server::*; @@ -56,7 +59,34 @@ fn main() { let server = if dotenv::var("USE_HYPER").is_ok() { println!("Booting with a hyper based server"); - Hyper(HyperService::new(app, threads as usize)) + let addr = ([127, 0, 0, 1], port).into(); + let service = HyperService::new(app, threads as usize); + let server = hyper::Server::bind(&addr).serve(service); + + let (tx, rx) = futures::sync::oneshot::channel::<()>(); + let server = server + .with_graceful_shutdown(rx) + .map_err(|e| log::error!("Server error: {}", e)); + + ctrlc_handler(move || tx.send(()).unwrap_or(())); + + let mut rt = tokio::runtime::Builder::new() + .core_threads(4) + .name_prefix("hyper-server-worker-") + .after_start(|| { + log::debug!("Stared thread {}", thread::current().name().unwrap_or("?")) + }) + .before_stop(|| { + log::debug!( + "Stopping thread {}", + thread::current().name().unwrap_or("?") + ) + }) + .build() + .unwrap(); + rt.spawn(server); + + Hyper(rt) } else { println!("Booting with a civet based server"); let mut cfg = civet::Config::new(); @@ -66,19 +96,43 @@ fn main() { println!("listening on port {}", port); + // Give tokio a chance to spawn the first worker thread + thread::sleep(Duration::from_millis(10)); + // Creating this file tells heroku to tell nginx that the application is ready // to receive traffic. if heroku { + println!("Writing to /tmp/app-initialized"); File::create("/tmp/app-initialized").unwrap(); } - if let Hyper(server) = server { - let addr = ([127, 0, 0, 1], port).into(); - server.run(addr); - } else { - // Civet server is already running, but we need to block the main thread forever - // TODO: handle a graceful shutdown by just waiting for a SIG{INT,TERM} - let (_tx, rx) = channel::<()>(); - rx.recv().unwrap(); + // Block the main thread until the server has shutdown + match server { + Hyper(rt) => rt.shutdown_on_idle().wait().unwrap(), + Civet(server) => { + let (tx, rx) = channel::<()>(); + ctrlc_handler(move || tx.send(()).unwrap_or(())); + rx.recv().unwrap(); + drop(server); + } } + + println!("Server has gracefully shutdown!"); +} + +fn ctrlc_handler(f: F) +where + F: FnOnce() + Send + 'static, +{ + let call_once = Mutex::new(Some(f)); + + ctrlc::set_handler(move || { + if let Some(f) = call_once.lock().unwrap().take() { + println!("Starting graceful shutdown"); + f(); + } else { + println!("Already sent signal to start graceful shutdown"); + } + }) + .unwrap(); }