From f6eefd1690517f54964871c28d5674414e5a2396 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 2 Dec 2022 09:07:08 -0500 Subject: [PATCH 1/6] Close HTTP service on shutdown --- cmd/rqlited/main.go | 2 ++ http/service.go | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/rqlited/main.go b/cmd/rqlited/main.go index ed5adaa9..a65cc76a 100644 --- a/cmd/rqlited/main.go +++ b/cmd/rqlited/main.go @@ -170,6 +170,8 @@ func main() { // Perform a stepdown, ignore any errors. str.Stepdown(true) } + + httpServ.Close() if err := str.Close(true); err != nil { log.Printf("failed to close store: %s", err.Error()) } diff --git a/http/service.go b/http/service.go index b15f9d3b..70925089 100644 --- a/http/service.go +++ b/http/service.go @@ -3,6 +3,7 @@ package http import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" @@ -245,9 +246,10 @@ func ResetStats() { // Service provides HTTP service. type Service struct { - closeCh chan struct{} - addr string // Bind address of the HTTP service. - ln net.Listener // Service listener + httpServer http.Server + closeCh chan struct{} + addr string // Bind address of the HTTP service. + ln net.Listener // Service listener store Store // The Raft-backed database store. @@ -304,7 +306,7 @@ func New(addr string, store Store, cluster Cluster, credentials CredentialStore) // Start starts the service. func (s *Service) Start() error { - server := http.Server{ + s.httpServer = http.Server{ Handler: s, } @@ -337,9 +339,9 @@ func (s *Service) Start() error { s.DefaultQueueCap, s.DefaultQueueBatchSz, s.DefaultQueueTimeout.String()) go func() { - err := server.Serve(s.ln) + err := s.httpServer.Serve(s.ln) if err != nil { - s.logger.Println("HTTP service Serve() returned:", err.Error()) + s.logger.Println("HTTP service stopped:", err.Error()) } }() s.logger.Println("service listening on", s.Addr()) @@ -349,8 +351,9 @@ func (s *Service) Start() error { // Close closes the service. func (s *Service) Close() { - s.stmtQueue.Close() + s.httpServer.Shutdown(context.Background()) + s.stmtQueue.Close() select { case <-s.queueDone: default: From e3a3d67e9ff96440dd7d6003b45312e656ddc907 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 2 Dec 2022 09:22:05 -0500 Subject: [PATCH 2/6] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc34380a..e5347379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 7.12.1 (unreleased) +### Implementation changes and bug fixes +- [PR #1136](https://github.com/rqlite/rqlite/pull/1136): Stop HTTP server gracefully on node shutdown. + ## 7.12.0 (December 1st 2022) ### New features - [PR #1121](https://github.com/rqlite/rqlite/pull/1121): Transparently forward node-removal requests to Leader. From 54d61c78660dc3ab7e676f434bf65d7a02915ad2 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 2 Dec 2022 10:28:17 -0500 Subject: [PATCH 3/6] Update store.go --- store/store.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/store/store.go b/store/store.go index 7bcafa9f..ece651a2 100644 --- a/store/store.go +++ b/store/store.go @@ -418,7 +418,8 @@ func (s *Store) Bootstrap(servers ...*Server) error { } // Stepdown forces this node to relinquish leadership to another node in -// the cluster. If this node is not the leader an error will be returned. +// the cluster. If this node is not the leader, and 'wait' is true, an error +// will be returned. func (s *Store) Stepdown(wait bool) error { f := s.raft.LeadershipTransfer() if !wait { From f6c4b17a727809696f952a018b2262681932f521 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 2 Dec 2022 14:41:45 -0500 Subject: [PATCH 4/6] Enable "stepdown on shutdown" by default https://github.com/rqlite/rqlite/issues/1132 --- cmd/rqlited/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/rqlited/flags.go b/cmd/rqlited/flags.go index d761b01e..9b040eba 100644 --- a/cmd/rqlited/flags.go +++ b/cmd/rqlited/flags.go @@ -396,7 +396,7 @@ func ParseFlags(name, desc string, build *BuildInfo) (*Config, error) { flag.Uint64Var(&config.RaftSnapThreshold, "raft-snap", 8192, "Number of outstanding log entries that trigger snapshot") flag.DurationVar(&config.RaftSnapInterval, "raft-snap-int", 30*time.Second, "Snapshot threshold check interval") flag.DurationVar(&config.RaftLeaderLeaseTimeout, "raft-leader-lease-timeout", 0, "Raft leader lease timeout. Use 0s for Raft default") - flag.BoolVar(&config.RaftStepdownOnShutdown, "raft-shutdown-stepdown", false, "Stepdown as leader before shutting down") + flag.BoolVar(&config.RaftStepdownOnShutdown, "raft-shutdown-stepdown", true, "Stepdown as leader before shutting down. Enabled by default") flag.BoolVar(&config.RaftShutdownOnRemove, "raft-remove-shutdown", false, "Shutdown Raft if node removed") flag.BoolVar(&config.RaftNoFreelistSync, "raft-no-freelist-sync", false, "Do not sync Raft log database freelist to disk") flag.StringVar(&config.RaftLogLevel, "raft-log-level", "INFO", "Minimum log level for Raft module") From 4dace6f88882ad0ca6026f45cc7a05032e7bf8fb Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 2 Dec 2022 16:05:23 -0500 Subject: [PATCH 5/6] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5347379..c4878fe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 7.12.1 (unreleased) ### Implementation changes and bug fixes - [PR #1136](https://github.com/rqlite/rqlite/pull/1136): Stop HTTP server gracefully on node shutdown. +- [f6c4b17](https://github.com/rqlite/rqlite/commit/f6c4b17a727809696f952a018b2262681932f521): By default, Leader node will stepdown if node is shutting down. ## 7.12.0 (December 1st 2022) ### New features From f967754fcb9631466f1aaa7baa7f2df97df68b0d Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 3 Dec 2022 08:34:15 -0500 Subject: [PATCH 6/6] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4878fe2..4991c84a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ## 7.12.1 (unreleased) ### Implementation changes and bug fixes - [PR #1136](https://github.com/rqlite/rqlite/pull/1136): Stop HTTP server gracefully on node shutdown. -- [f6c4b17](https://github.com/rqlite/rqlite/commit/f6c4b17a727809696f952a018b2262681932f521): By default, Leader node will stepdown if node is shutting down. +- [f6c4b17](https://github.com/rqlite/rqlite/commit/f6c4b17a727809696f952a018b2262681932f521): By default, Leader node will stepdown if that node is shutting down. ## 7.12.0 (December 1st 2022) ### New features