2017-12-21 20:23:45 +00:00
|
|
|
package grpc_prometheus
|
|
|
|
|
|
|
|
import (
|
|
|
|
"io"
|
|
|
|
|
|
|
|
prom "github.com/prometheus/client_golang/prometheus"
|
|
|
|
"golang.org/x/net/context"
|
|
|
|
"google.golang.org/grpc"
|
|
|
|
"google.golang.org/grpc/codes"
|
2019-07-31 06:09:38 +00:00
|
|
|
"google.golang.org/grpc/status"
|
2017-12-21 20:23:45 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// ClientMetrics represents a collection of metrics to be registered on a
|
|
|
|
// Prometheus metrics registry for a gRPC client.
|
|
|
|
type ClientMetrics struct {
|
|
|
|
clientStartedCounter *prom.CounterVec
|
|
|
|
clientHandledCounter *prom.CounterVec
|
|
|
|
clientStreamMsgReceived *prom.CounterVec
|
|
|
|
clientStreamMsgSent *prom.CounterVec
|
|
|
|
clientHandledHistogramEnabled bool
|
|
|
|
clientHandledHistogramOpts prom.HistogramOpts
|
|
|
|
clientHandledHistogram *prom.HistogramVec
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewClientMetrics returns a ClientMetrics object. Use a new instance of
|
|
|
|
// ClientMetrics when not using the default Prometheus metrics registry, for
|
|
|
|
// example when wanting to control which metrics are added to a registry as
|
|
|
|
// opposed to automatically adding metrics via init functions.
|
2019-07-31 06:09:38 +00:00
|
|
|
func NewClientMetrics(counterOpts ...CounterOption) *ClientMetrics {
|
|
|
|
opts := counterOptions(counterOpts)
|
2017-12-21 20:23:45 +00:00
|
|
|
return &ClientMetrics{
|
|
|
|
clientStartedCounter: prom.NewCounterVec(
|
2019-07-31 06:09:38 +00:00
|
|
|
opts.apply(prom.CounterOpts{
|
2017-12-21 20:23:45 +00:00
|
|
|
Name: "grpc_client_started_total",
|
|
|
|
Help: "Total number of RPCs started on the client.",
|
2019-07-31 06:09:38 +00:00
|
|
|
}), []string{"grpc_type", "grpc_service", "grpc_method"}),
|
2017-12-21 20:23:45 +00:00
|
|
|
|
|
|
|
clientHandledCounter: prom.NewCounterVec(
|
2019-07-31 06:09:38 +00:00
|
|
|
opts.apply(prom.CounterOpts{
|
2017-12-21 20:23:45 +00:00
|
|
|
Name: "grpc_client_handled_total",
|
|
|
|
Help: "Total number of RPCs completed by the client, regardless of success or failure.",
|
2019-07-31 06:09:38 +00:00
|
|
|
}), []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"}),
|
2017-12-21 20:23:45 +00:00
|
|
|
|
|
|
|
clientStreamMsgReceived: prom.NewCounterVec(
|
2019-07-31 06:09:38 +00:00
|
|
|
opts.apply(prom.CounterOpts{
|
2017-12-21 20:23:45 +00:00
|
|
|
Name: "grpc_client_msg_received_total",
|
|
|
|
Help: "Total number of RPC stream messages received by the client.",
|
2019-07-31 06:09:38 +00:00
|
|
|
}), []string{"grpc_type", "grpc_service", "grpc_method"}),
|
2017-12-21 20:23:45 +00:00
|
|
|
|
|
|
|
clientStreamMsgSent: prom.NewCounterVec(
|
2019-07-31 06:09:38 +00:00
|
|
|
opts.apply(prom.CounterOpts{
|
2017-12-21 20:23:45 +00:00
|
|
|
Name: "grpc_client_msg_sent_total",
|
|
|
|
Help: "Total number of gRPC stream messages sent by the client.",
|
2019-07-31 06:09:38 +00:00
|
|
|
}), []string{"grpc_type", "grpc_service", "grpc_method"}),
|
2017-12-21 20:23:45 +00:00
|
|
|
|
|
|
|
clientHandledHistogramEnabled: false,
|
|
|
|
clientHandledHistogramOpts: prom.HistogramOpts{
|
|
|
|
Name: "grpc_client_handling_seconds",
|
|
|
|
Help: "Histogram of response latency (seconds) of the gRPC until it is finished by the application.",
|
|
|
|
Buckets: prom.DefBuckets,
|
|
|
|
},
|
|
|
|
clientHandledHistogram: nil,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Describe sends the super-set of all possible descriptors of metrics
|
|
|
|
// collected by this Collector to the provided channel and returns once
|
|
|
|
// the last descriptor has been sent.
|
|
|
|
func (m *ClientMetrics) Describe(ch chan<- *prom.Desc) {
|
|
|
|
m.clientStartedCounter.Describe(ch)
|
|
|
|
m.clientHandledCounter.Describe(ch)
|
|
|
|
m.clientStreamMsgReceived.Describe(ch)
|
|
|
|
m.clientStreamMsgSent.Describe(ch)
|
|
|
|
if m.clientHandledHistogramEnabled {
|
|
|
|
m.clientHandledHistogram.Describe(ch)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Collect is called by the Prometheus registry when collecting
|
|
|
|
// metrics. The implementation sends each collected metric via the
|
|
|
|
// provided channel and returns once the last metric has been sent.
|
|
|
|
func (m *ClientMetrics) Collect(ch chan<- prom.Metric) {
|
|
|
|
m.clientStartedCounter.Collect(ch)
|
|
|
|
m.clientHandledCounter.Collect(ch)
|
|
|
|
m.clientStreamMsgReceived.Collect(ch)
|
|
|
|
m.clientStreamMsgSent.Collect(ch)
|
|
|
|
if m.clientHandledHistogramEnabled {
|
|
|
|
m.clientHandledHistogram.Collect(ch)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// EnableClientHandlingTimeHistogram turns on recording of handling time of RPCs.
|
|
|
|
// Histogram metrics can be very expensive for Prometheus to retain and query.
|
|
|
|
func (m *ClientMetrics) EnableClientHandlingTimeHistogram(opts ...HistogramOption) {
|
|
|
|
for _, o := range opts {
|
|
|
|
o(&m.clientHandledHistogramOpts)
|
|
|
|
}
|
|
|
|
if !m.clientHandledHistogramEnabled {
|
|
|
|
m.clientHandledHistogram = prom.NewHistogramVec(
|
|
|
|
m.clientHandledHistogramOpts,
|
|
|
|
[]string{"grpc_type", "grpc_service", "grpc_method"},
|
|
|
|
)
|
|
|
|
}
|
|
|
|
m.clientHandledHistogramEnabled = true
|
|
|
|
}
|
|
|
|
|
|
|
|
// UnaryClientInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Unary RPCs.
|
|
|
|
func (m *ClientMetrics) UnaryClientInterceptor() func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
|
|
|
|
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
|
|
|
|
monitor := newClientReporter(m, Unary, method)
|
|
|
|
monitor.SentMessage()
|
|
|
|
err := invoker(ctx, method, req, reply, cc, opts...)
|
|
|
|
if err != nil {
|
|
|
|
monitor.ReceivedMessage()
|
|
|
|
}
|
2019-07-31 06:09:38 +00:00
|
|
|
st, _ := status.FromError(err)
|
|
|
|
monitor.Handled(st.Code())
|
2017-12-21 20:23:45 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-31 06:09:38 +00:00
|
|
|
// StreamClientInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Streaming RPCs.
|
2017-12-21 20:23:45 +00:00
|
|
|
func (m *ClientMetrics) StreamClientInterceptor() func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
|
|
|
|
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
|
|
|
|
monitor := newClientReporter(m, clientStreamType(desc), method)
|
|
|
|
clientStream, err := streamer(ctx, desc, cc, method, opts...)
|
|
|
|
if err != nil {
|
2019-07-31 06:09:38 +00:00
|
|
|
st, _ := status.FromError(err)
|
|
|
|
monitor.Handled(st.Code())
|
2017-12-21 20:23:45 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return &monitoredClientStream{clientStream, monitor}, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func clientStreamType(desc *grpc.StreamDesc) grpcType {
|
|
|
|
if desc.ClientStreams && !desc.ServerStreams {
|
|
|
|
return ClientStream
|
|
|
|
} else if !desc.ClientStreams && desc.ServerStreams {
|
|
|
|
return ServerStream
|
|
|
|
}
|
|
|
|
return BidiStream
|
|
|
|
}
|
|
|
|
|
|
|
|
// monitoredClientStream wraps grpc.ClientStream allowing each Sent/Recv of message to increment counters.
|
|
|
|
type monitoredClientStream struct {
|
|
|
|
grpc.ClientStream
|
|
|
|
monitor *clientReporter
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *monitoredClientStream) SendMsg(m interface{}) error {
|
|
|
|
err := s.ClientStream.SendMsg(m)
|
|
|
|
if err == nil {
|
|
|
|
s.monitor.SentMessage()
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *monitoredClientStream) RecvMsg(m interface{}) error {
|
|
|
|
err := s.ClientStream.RecvMsg(m)
|
|
|
|
if err == nil {
|
|
|
|
s.monitor.ReceivedMessage()
|
|
|
|
} else if err == io.EOF {
|
|
|
|
s.monitor.Handled(codes.OK)
|
|
|
|
} else {
|
2019-07-31 06:09:38 +00:00
|
|
|
st, _ := status.FromError(err)
|
|
|
|
s.monitor.Handled(st.Code())
|
2017-12-21 20:23:45 +00:00
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|