feat: graceful shutdown

Signed-off-by: m.nabokikh <maksim.nabokikh@flant.com>
This commit is contained in:
m.nabokikh 2021-01-26 12:16:30 +04:00
parent c41f970b16
commit f82c217e12
3 changed files with 80 additions and 36 deletions

View File

@ -10,11 +10,14 @@ import (
"net" "net"
"net/http" "net/http"
"os" "os"
"os/signal"
"strings" "strings"
"syscall"
"time" "time"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
"github.com/oklog/run"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
@ -68,6 +71,28 @@ func commandServe() *cobra.Command {
return cmd return cmd
} }
func listenAndShutdownGracefully(logger log.Logger, gr *run.Group, srv *http.Server, name string) error {
l, err := net.Listen("tcp", srv.Addr)
if err != nil {
return fmt.Errorf("listening (%s) on %s: %v", name, srv.Addr, err)
}
gr.Add(func() error {
logger.Infof("listening (%s) on %s", name, srv.Addr)
return srv.Serve(l)
}, func(err error) {
logger.Debugf("starting gracefully shutdown (%s)", name)
if err := l.Close(); err != nil {
logger.Errorf("gracefully close (%s) listener: %v", name, err)
}
if err := srv.Shutdown(context.Background()); err != nil {
logger.Errorf("gracefully shutdown (%s): %v", name, err)
}
})
return nil
}
func runServe(options serveOptions) error { func runServe(options serveOptions) error {
configFile := options.config configFile := options.config
configData, err := ioutil.ReadFile(configFile) configData, err := ioutil.ReadFile(configFile)
@ -302,21 +327,21 @@ func runServe(options serveOptions) error {
telemetryServ := http.NewServeMux() telemetryServ := http.NewServeMux()
telemetryServ.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{})) telemetryServ.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{}))
errc := make(chan error, 3) var gr run.Group
if c.Telemetry.HTTP != "" { if c.Telemetry.HTTP != "" {
logger.Infof("listening (http/telemetry) on %s", c.Telemetry.HTTP) telemetrySrv := &http.Server{Addr: c.Telemetry.HTTP, Handler: telemetryServ}
go func() { if err := listenAndShutdownGracefully(logger, &gr, telemetrySrv, "http/telemetry"); err != nil {
err := http.ListenAndServe(c.Telemetry.HTTP, telemetryServ) return err
errc <- fmt.Errorf("listening on %s failed: %v", c.Telemetry.HTTP, err) }
}()
} }
if c.Web.HTTP != "" { if c.Web.HTTP != "" {
logger.Infof("listening (http) on %s", c.Web.HTTP) httpSrv := &http.Server{Addr: c.Web.HTTP, Handler: serv}
go func() { if err := listenAndShutdownGracefully(logger, &gr, httpSrv, "http"); err != nil {
err := http.ListenAndServe(c.Web.HTTP, serv) return err
errc <- fmt.Errorf("listening on %s failed: %v", c.Web.HTTP, err) }
}()
} }
if c.Web.HTTPS != "" { if c.Web.HTTPS != "" {
httpsSrv := &http.Server{ httpsSrv := &http.Server{
Addr: c.Web.HTTPS, Addr: c.Web.HTTPS,
@ -327,35 +352,51 @@ func runServe(options serveOptions) error {
MinVersion: tls.VersionTLS12, MinVersion: tls.VersionTLS12,
}, },
} }
if err := listenAndShutdownGracefully(logger, &gr, httpsSrv, "https"); err != nil {
logger.Infof("listening (https) on %s", c.Web.HTTPS) return err
go func() { }
err = httpsSrv.ListenAndServeTLS(c.Web.TLSCert, c.Web.TLSKey)
errc <- fmt.Errorf("listening on %s failed: %v", c.Web.HTTPS, err)
}()
} }
if c.GRPC.Addr != "" { if c.GRPC.Addr != "" {
logger.Infof("listening (grpc) on %s", c.GRPC.Addr) grpcListener, err := net.Listen("tcp", c.GRPC.Addr)
go func() { if err != nil {
errc <- func() error { return fmt.Errorf("listening (grcp) on %s: %w", c.GRPC.Addr, err)
list, err := net.Listen("tcp", c.GRPC.Addr) }
if err != nil {
return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err) grpcSrv := grpc.NewServer(grpcOptions...)
} api.RegisterDexServer(grpcSrv, server.NewAPI(serverConfig.Storage, logger))
s := grpc.NewServer(grpcOptions...) grpcMetrics.InitializeMetrics(grpcSrv)
api.RegisterDexServer(s, server.NewAPI(serverConfig.Storage, logger)) if c.GRPC.Reflection {
grpcMetrics.InitializeMetrics(s) logger.Info("enabling reflection in grpc service")
if c.GRPC.Reflection { reflection.Register(grpcSrv)
logger.Info("enabling reflection in grpc service") }
reflection.Register(s)
} gr.Add(func() error {
err = s.Serve(list) logger.Infof("listening (grpc) on %s", c.GRPC.Addr)
return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err) return grpcSrv.Serve(grpcListener)
}() }, func(err error) {
}() logger.Debugf("starting gracefully shutdown (grpc)")
if err := grpcListener.Close(); err != nil {
logger.Errorf("failed to gracefully close (grpc) listener: %v", err)
}
grpcSrv.GracefulStop()
})
} }
return <-errc sig := make(chan os.Signal)
gr.Add(func() error {
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
receivedSig := <-sig
logger.Infof("received %s signal, shutting down", receivedSig)
return nil
}, func(err error) {
close(sig)
})
if err := gr.Run(); err != nil {
return fmt.Errorf("run groups: %w", err)
}
return nil
} }
var ( var (

1
go.mod
View File

@ -18,6 +18,7 @@ require (
github.com/lib/pq v1.9.0 github.com/lib/pq v1.9.0
github.com/mattermost/xml-roundtrip-validator v0.0.0-20201219040909-8fd2afad43d1 github.com/mattermost/xml-roundtrip-validator v0.0.0-20201219040909-8fd2afad43d1
github.com/mattn/go-sqlite3 v1.14.6 github.com/mattn/go-sqlite3 v1.14.6
github.com/oklog/run v1.1.0
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.4.0 github.com/prometheus/client_golang v1.4.0
github.com/russellhaering/goxmldsig v1.1.0 github.com/russellhaering/goxmldsig v1.1.0

2
go.sum
View File

@ -253,6 +253,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=