k-space/logmower-shipper
10
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Activity

retry within mongo sender

Browse Source
This commit is contained in:
rasmus 2022-11-11 17:52:30 +02:00
parent a50c1446c3
commit d08f314cf1
3 changed files with 34 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/prometheus/client_golang v1.14.0
github.com/urfave/cli/v2 v2.23.5
go.mongodb.org/mongo-driver v1.11.0
k8s.io/apimachinery v0.25.3
k8s.io/apimachinery v0.25.4
)
require (

48
pkg/sender/sender.go
View File

@ -10,6 +10,7 @@ import (
"github.com/jtagcat/util"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"k8s.io/apimachinery/pkg/util/wait"
)
var (
@ -21,6 +22,16 @@ const (
MaxBatchTime = 5 * time.Second
)
// wrapper to force copying before use
func backoff() wait.Backoff {
return wait.Backoff{
Duration: 2 * time.Second,
Factor: 4,
Jitter: 0.2,
Cap: 2 * time.Minute,
}
}
type Queue <-chan m.Record
func (queue Queue) Sender(db *mongo.Collection, metricsFilename string, cancelOnError func()) {
@ -60,31 +71,36 @@ func (queue Queue) Sender(db *mongo.Collection, metricsFilename string, cancelOn
promShipperSynced.WithLabelValues(metricsFilename).Set(0)
result, err := insertManyWithSimulate(db, batch)
err := util.RetryOnError(backoff(), func() (_ bool, _ error) {
result, err := insertManyWithSimulate(db, batch)
var succeedCount int
if result != nil {
succeedCount = len(result.InsertedIDs)
}
promShipperDbSent.WithLabelValues(metricsFilename).Add(float64(succeedCount))
var succeedCount int
if result != nil {
succeedCount = len(result.InsertedIDs)
}
promShipperDbSent.WithLabelValues(metricsFilename).Add(float64(succeedCount))
if err != nil {
promShipperDbSendError.WithLabelValues(metricsFilename).Add(1)
if succeedCount == len(batch) {
log.Printf("all insertions in batch were successful, yet failure in database: %e", err)
cancelOnError()
if err == nil {
return
}
promShipperDbSendError.WithLabelValues(metricsFilename).Add(1)
firstFailed := &batch[succeedCount] // (len-1)+1
log.Printf("failure in inserting %q record with offset %d to database: %e",
firstFailed.Path, firstFailed.Offset, err)
batch = batch[succeedCount:]
if succeedCount == len(batch) {
return true, fmt.Errorf("all insertions in batch were successful, yet failure in database: %w", err)
}
return true, fmt.Errorf("insert record with offset %d to database: %w",
batch[0].Offset, err)
})
if err != nil {
log.Printf("batch insert %q to mongo: %e", metricsFilename, err)
cancelOnError()
return
}
}
}

2
vendor/modules.txt vendored
View File

@ -167,7 +167,7 @@ google.golang.org/protobuf/runtime/protoiface
google.golang.org/protobuf/runtime/protoimpl
google.golang.org/protobuf/types/descriptorpb
google.golang.org/protobuf/types/known/timestamppb
# k8s.io/apimachinery v0.25.3 => github.com/jtagcat/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20221027124836-581f57977fff
# k8s.io/apimachinery v0.25.4 => github.com/jtagcat/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20221027124836-581f57977fff
## explicit; go 1.19
k8s.io/apimachinery/pkg/util/runtime
k8s.io/apimachinery/pkg/util/wait