package logmower import ( "time" "github.com/jtagcat/util" prom "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "go.mongodb.org/mongo-driver/bson" "go.uber.org/zap" ) var ( promShipperMongoSent = promauto.NewCounter(prom.CounterOpts{ Subsystem: "shipper", Name: "sent_count", Help: "Log items successfully committed to mongo", }) promShipperMongoSentError = promauto.NewCounter(prom.CounterOpts{ Subsystem: "shipper", Name: "mongo_errors", Help: "Errors while submitting to mongo", // TODO: }) promShipperDropped = promauto.NewCounterVec(prom.CounterOpts{ Subsystem: "shipper", Name: "queue_dropped", Help: "Items ready to be sent to mongo, but dropped due to full queue", }, []string{"filename"}) ) func init() { promauto.NewGaugeFunc(prom.GaugeOpts{ Subsystem: "shipper", Name: "queue_size", Help: "Submit queue size", }, func() float64 { return float64(SendQueueLimit) }) } func (s *submitter) sender() { promauto.NewGaugeFunc(prom.GaugeOpts{ Subsystem: "shipper", Name: "queue_items", Help: "Items in queue to be submitted in batch to mongo", }, func() float64 { return float64(len(s.sendQueue)) }) batched := make(chan []mLog) go func() { util.Batch(4, MaxBatchTime, s.sendQueue, batched) }() for { batch, ok := <-batched if !ok { return } var batchBson []interface{} // mongo does not like typing for _, b := range batch { batchBson = append(batchBson, b.toBson()) } result, err := s.db.InsertMany(mongoTimeoutCtx(s.ctx), batchBson, nil) promShipperMongoSent.Add(float64( len(result.InsertedIDs))) if err != nil { s.l.Error("mongo send returned error; TODO: add some selective retry here or something", zap.Error(err)) // TODO: } } } // when editing, also edit toBson(); all bson.D (and bson.M) uses type mLog struct { HostInfo HostInfo Filename string Offset int64 // byte offset where log entry ends at Content string // TODO: Time time.Time } // not using marshal, since it is <0.1x performance func (l *mLog) toBson() bson.M { return bson.M{ "host_info": l.HostInfo, "filename": l.Filename, "offset": l.Offset, "content": l.Content, "time": l.Time, } }