feat: graceful shutdown fixes

Signed-off-by: m.nabokikh <maksim.nabokikh@flant.com>
This commit is contained in:
Maksim Nabokikh 2021-01-26 14:30:35 +04:00 committed by m.nabokikh
parent f82c217e12
commit 65a8bf2af3

View File

@ -10,7 +10,6 @@ import (
"net" "net"
"net/http" "net/http"
"os" "os"
"os/signal"
"strings" "strings"
"syscall" "syscall"
"time" "time"
@ -81,12 +80,11 @@ func listenAndShutdownGracefully(logger log.Logger, gr *run.Group, srv *http.Ser
logger.Infof("listening (%s) on %s", name, srv.Addr) logger.Infof("listening (%s) on %s", name, srv.Addr)
return srv.Serve(l) return srv.Serve(l)
}, func(err error) { }, func(err error) {
logger.Debugf("starting gracefully shutdown (%s)", name) ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
if err := l.Close(); err != nil { defer cancel()
logger.Errorf("gracefully close (%s) listener: %v", name, err)
}
if err := srv.Shutdown(context.Background()); err != nil { logger.Debugf("starting gracefully shutdown (%s)", name)
if err := srv.Shutdown(ctx); err != nil {
logger.Errorf("gracefully shutdown (%s): %v", name, err) logger.Errorf("gracefully shutdown (%s): %v", name, err)
} }
}) })
@ -330,6 +328,8 @@ func runServe(options serveOptions) error {
var gr run.Group var gr run.Group
if c.Telemetry.HTTP != "" { if c.Telemetry.HTTP != "" {
telemetrySrv := &http.Server{Addr: c.Telemetry.HTTP, Handler: telemetryServ} telemetrySrv := &http.Server{Addr: c.Telemetry.HTTP, Handler: telemetryServ}
defer telemetrySrv.Close()
if err := listenAndShutdownGracefully(logger, &gr, telemetrySrv, "http/telemetry"); err != nil { if err := listenAndShutdownGracefully(logger, &gr, telemetrySrv, "http/telemetry"); err != nil {
return err return err
} }
@ -337,6 +337,8 @@ func runServe(options serveOptions) error {
if c.Web.HTTP != "" { if c.Web.HTTP != "" {
httpSrv := &http.Server{Addr: c.Web.HTTP, Handler: serv} httpSrv := &http.Server{Addr: c.Web.HTTP, Handler: serv}
defer httpSrv.Close()
if err := listenAndShutdownGracefully(logger, &gr, httpSrv, "http"); err != nil { if err := listenAndShutdownGracefully(logger, &gr, httpSrv, "http"); err != nil {
return err return err
} }
@ -352,6 +354,8 @@ func runServe(options serveOptions) error {
MinVersion: tls.VersionTLS12, MinVersion: tls.VersionTLS12,
}, },
} }
defer httpsSrv.Close()
if err := listenAndShutdownGracefully(logger, &gr, httpsSrv, "https"); err != nil { if err := listenAndShutdownGracefully(logger, &gr, httpsSrv, "https"); err != nil {
return err return err
} }
@ -365,6 +369,7 @@ func runServe(options serveOptions) error {
grpcSrv := grpc.NewServer(grpcOptions...) grpcSrv := grpc.NewServer(grpcOptions...)
api.RegisterDexServer(grpcSrv, server.NewAPI(serverConfig.Storage, logger)) api.RegisterDexServer(grpcSrv, server.NewAPI(serverConfig.Storage, logger))
grpcMetrics.InitializeMetrics(grpcSrv) grpcMetrics.InitializeMetrics(grpcSrv)
if c.GRPC.Reflection { if c.GRPC.Reflection {
logger.Info("enabling reflection in grpc service") logger.Info("enabling reflection in grpc service")
@ -376,25 +381,16 @@ func runServe(options serveOptions) error {
return grpcSrv.Serve(grpcListener) return grpcSrv.Serve(grpcListener)
}, func(err error) { }, func(err error) {
logger.Debugf("starting gracefully shutdown (grpc)") logger.Debugf("starting gracefully shutdown (grpc)")
if err := grpcListener.Close(); err != nil {
logger.Errorf("failed to gracefully close (grpc) listener: %v", err)
}
grpcSrv.GracefulStop() grpcSrv.GracefulStop()
}) })
} }
sig := make(chan os.Signal) gr.Add(run.SignalHandler(context.Background(), os.Interrupt, syscall.SIGTERM))
gr.Add(func() error {
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
receivedSig := <-sig
logger.Infof("received %s signal, shutting down", receivedSig)
return nil
}, func(err error) {
close(sig)
})
if err := gr.Run(); err != nil { if err := gr.Run(); err != nil {
return fmt.Errorf("run groups: %w", err) if _, ok := err.(run.SignalError); !ok {
return fmt.Errorf("run groups: %w", err)
}
logger.Infof("%v, shutdown now", err)
} }
return nil return nil
} }