retry within mongo sender

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/prometheus/client_golang v1.14.0
 	github.com/urfave/cli/v2 v2.23.5
 	go.mongodb.org/mongo-driver v1.11.0
-	k8s.io/apimachinery v0.25.3
+	k8s.io/apimachinery v0.25.4
 )
 
 require (

pkg/sender/sender.go

@@ -10,6 +10,7 @@ import (
 	"github.com/jtagcat/util"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 var (
@@ -21,6 +22,16 @@ const (
 	MaxBatchTime = 5 * time.Second
 )
 
+// wrapper to force copying before use
+func backoff() wait.Backoff {
+	return wait.Backoff{
+		Duration: 2 * time.Second,
+		Factor:   4,
+		Jitter:   0.2,
+		Cap:      2 * time.Minute,
+	}
+}
+
 type Queue <-chan m.Record
 
 func (queue Queue) Sender(db *mongo.Collection, metricsFilename string, cancelOnError func()) {
@@ -60,6 +71,7 @@ func (queue Queue) Sender(db *mongo.Collection, metricsFilename string, cancelOn
 
 		promShipperSynced.WithLabelValues(metricsFilename).Set(0)
 
+		err := util.RetryOnError(backoff(), func() (_ bool, _ error) {
 			result, err := insertManyWithSimulate(db, batch)
 
 			var succeedCount int
@@ -68,23 +80,27 @@ func (queue Queue) Sender(db *mongo.Collection, metricsFilename string, cancelOn
 			}
 			promShipperDbSent.WithLabelValues(metricsFilename).Add(float64(succeedCount))
 
-		if err != nil {
+			if err == nil {
+				return
+			}
 			promShipperDbSendError.WithLabelValues(metricsFilename).Add(1)
 
+			batch = batch[succeedCount:]
+
 			if succeedCount == len(batch) {
-				log.Printf("all insertions in batch were successful, yet failure in database: %e", err)
+				return true, fmt.Errorf("all insertions in batch were successful, yet failure in database: %w", err)
+			}
+
+			return true, fmt.Errorf("insert record with offset %d to database: %w",
+				batch[0].Offset, err)
+		})
+		if err != nil {
+			log.Printf("batch insert %q to mongo: %e", metricsFilename, err)
 
 			cancelOnError()
 			return
 		}
 
-			firstFailed := &batch[succeedCount] // (len-1)+1
-			log.Printf("failure in inserting %q record with offset %d to database: %e",
-				firstFailed.Path, firstFailed.Offset, err)
-
-			cancelOnError()
-			return
-		}
 	}
 }
 

vendor/modules.txt

@@ -167,7 +167,7 @@ google.golang.org/protobuf/runtime/protoiface
 google.golang.org/protobuf/runtime/protoimpl
 google.golang.org/protobuf/types/descriptorpb
 google.golang.org/protobuf/types/known/timestamppb
-# k8s.io/apimachinery v0.25.3 => github.com/jtagcat/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20221027124836-581f57977fff
+# k8s.io/apimachinery v0.25.4 => github.com/jtagcat/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20221027124836-581f57977fff
 ## explicit; go 1.19
 k8s.io/apimachinery/pkg/util/runtime
 k8s.io/apimachinery/pkg/util/wait