k-space/logmower-shipper
11
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Activity

implement Simulate and globalize BufferLimit

Browse Source
This commit is contained in:
rasmus
2022-11-09 20:24:57 +02:00
parent c2e4a9eb69
commit 2232a748db
6 changed files with 62 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/file/file.go
View File

@@ -27,9 +27,9 @@ type File struct {
}
// TODO: caller could call duplicate shipFile of same name on file replace: sends might not work properly
func (f File) Process(ctx context.Context, db *mongo.Collection, recordLimitBytes int) {
func (f File) Process(ctx context.Context, db *mongo.Collection) {
_ = wait.ManagedExponentialBackoffWithContext(ctx, globals.Backoff(), func() (done bool, _ error) {
err := f.process(ctx, db, recordLimitBytes)
err := f.process(ctx, db)
if err == nil {
return true, nil
}
@@ -42,7 +42,7 @@ func (f File) Process(ctx context.Context, db *mongo.Collection, recordLimitByte
})
}
func (f File) launchChannels(cancel func(), db *mongo.Collection, recordLimitBytes int) (_ chan<- lines.Raw, deferFn func()) {
func (f File) launchChannels(cancel func(), db *mongo.Collection) (_ chan<- lines.Raw, deferFn func()) {
lineOut := make(chan lines.Raw)
sctx, scancel := context.WithCancel(context.Background())
@@ -52,7 +52,7 @@ func (f File) launchChannels(cancel func(), db *mongo.Collection, recordLimitByt
}
dbQueue := make(chan m.Record, SendQueueLimit)
go lines.RawC(lineOut).Process(sctx, recordLimitBytes, dbQueue)
go lines.RawC(lineOut).Process(sctx, dbQueue)
waitBatchSend := util.GoWg(func() {
sender.Queue(dbQueue).Sender(db, f.MetricsName, cancelAll)
@@ -65,12 +65,12 @@ func (f File) launchChannels(cancel func(), db *mongo.Collection, recordLimitByt
}
// use submitter(), don't use directly
func (f File) process(ctx context.Context, db *mongo.Collection, recordLimitBytes int) error {
func (f File) process(ctx context.Context, db *mongo.Collection) error {
lFile := lines.File(f) // file.File, but avoiding import cycle
sctx, cancel := context.WithCancel(ctx)
lineOut, dfn := f.launchChannels(cancel, db, recordLimitBytes)
lineOut, dfn := f.launchChannels(cancel, db)
defer dfn()
// get files with offset