logmower-shipper/pkg/sender/sender.go

74 lines
1.7 KiB
Go
Raw Normal View History

2022-11-09 16:07:28 +00:00
package sender
import (
"context"
"log"
"time"
"git.k-space.ee/k-space/logmower-shipper/pkg/globals"
m "git.k-space.ee/k-space/logmower-shipper/pkg/mongo"
"github.com/jtagcat/util"
"go.mongodb.org/mongo-driver/mongo"
)
const (
MaxBatchItems = 10000
MaxBatchTime = 5 * time.Second
)
type Queue <-chan m.Record
func (queue Queue) Sender(db *mongo.Collection, metricsFilename string) {
batched := make(chan []m.Record)
// metrics for batcher and queue
go func() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
for {
promShipperQueued.WithLabelValues(metricsFilename).Set(float64(
len(queue)))
timer := time.NewTimer(time.Second)
select {
case <-ctx.Done():
return
case <-timer.C:
}
}
}()
util.Batch(MaxBatchItems, MaxBatchTime, queue, batched)
// returns when sendQueue is closed
}()
for {
promShipperSynced.WithLabelValues(metricsFilename).Set(1)
batch, ok := <-batched
if !ok {
return
}
promShipperBatchSizeResult.Observe(float64(len(batch)))
promShipperSynced.WithLabelValues(metricsFilename).Set(0)
var batchBson []interface{} // mongo does not like typing
for _, b := range batch {
batchBson = append(batchBson, b.ToBson())
}
result, err := db.InsertMany(globals.MongoTimeout(context.Background()), batchBson, nil)
if err != nil {
promShipperDbSendError.WithLabelValues(metricsFilename).Add(1)
log.Printf("failure in batch submit to database: %e", err) // TODO: add some selective retry here or something, better error handling
continue
}
promShipperDbSent.WithLabelValues(metricsFilename).Add(float64(
len(result.InsertedIDs)))
}
}