This commit is contained in:
@@ -12,7 +12,8 @@ import (
|
||||
"git.k-space.ee/k-space/logmower-shipper/pkg/lines"
|
||||
m "git.k-space.ee/k-space/logmower-shipper/pkg/mongo"
|
||||
"git.k-space.ee/k-space/logmower-shipper/pkg/sender"
|
||||
"github.com/jtagcat/util"
|
||||
"github.com/jtagcat/util/std"
|
||||
"github.com/jtagcat/util/tail"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
mongoOpt "go.mongodb.org/mongo-driver/mongo/options"
|
||||
@@ -64,7 +65,7 @@ func (f File) launchChannels(cancel func(), db *mongo.Collection) (_ chan<- line
|
||||
dbQueue := make(chan m.Record, SendQueueLimit)
|
||||
go lines.RawC(lineOut).Process(sctx, dbQueue)
|
||||
|
||||
waitBatchSend := util.GoWg(func() {
|
||||
waitBatchSend := std.GoWg(func() {
|
||||
sender.Queue(dbQueue).Sender(db, f.MetricsName, cancelAll)
|
||||
})
|
||||
|
||||
@@ -102,7 +103,7 @@ func (f File) process(ctx context.Context, db *mongo.Collection) error {
|
||||
}
|
||||
startSize := fi.Size()
|
||||
|
||||
lineIn, errChan, err := util.TailFile(sctx, f.Path, dbOffset, io.SeekStart)
|
||||
lineIn, errChan, err := tail.New(sctx, f.Path, dbOffset, io.SeekStart)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tailing file: %w", err)
|
||||
}
|
||||
|
@@ -7,7 +7,8 @@ import (
|
||||
"time"
|
||||
|
||||
m "git.k-space.ee/k-space/logmower-shipper/pkg/mongo"
|
||||
"github.com/jtagcat/util"
|
||||
"github.com/jtagcat/util/batch"
|
||||
"github.com/jtagcat/util/retry"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
@@ -56,7 +57,7 @@ func (queue Queue) Sender(db *mongo.Collection, metricsFilename string, cancelOn
|
||||
}
|
||||
}()
|
||||
|
||||
util.Batch(MaxBatchItems, MaxBatchTime, queue, batched)
|
||||
batch.Batch(MaxBatchItems, MaxBatchTime, queue, batched)
|
||||
// returns when sendQueue is closed
|
||||
}()
|
||||
|
||||
@@ -71,7 +72,7 @@ func (queue Queue) Sender(db *mongo.Collection, metricsFilename string, cancelOn
|
||||
|
||||
promShipperSynced.WithLabelValues(metricsFilename).Set(0)
|
||||
|
||||
err := util.RetryOnError(backoff(), func() (_ bool, _ error) {
|
||||
err := retry.OnError(backoff(), func() (_ bool, _ error) {
|
||||
result, err := insertManyWithSimulate(db, batch)
|
||||
|
||||
var succeedCount int
|
||||
|
Reference in New Issue
Block a user