implement Simulate and globalize BufferLimit

pkg/lines/lines.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"sync"
 
+	"git.k-space.ee/k-space/logmower-shipper/pkg/globals"
 	m "git.k-space.ee/k-space/logmower-shipper/pkg/mongo"
 )
 
@@ -24,7 +25,7 @@ type (
 )
 
 // assumes all lines are from same file
-func (unparsed RawC) Process(ctx context.Context, bufferLimitBytes int, parsed chan<- m.Record) {
+func (unparsed RawC) Process(ctx context.Context, parsed chan<- m.Record) {
 	lines := make(chan singleLine)
 	go unparsed.parse(lines)
 
@@ -33,11 +34,11 @@ func (unparsed RawC) Process(ctx context.Context, bufferLimitBytes int, parsed c
 
 	stdOut, stdErr := make(chan singleLine), make(chan singleLine)
 	go func() {
-		singleLines(stdOut).process(ctx, bufferLimitBytes, parsed)
+		singleLines(stdOut).process(ctx, parsed)
 		wg.Done()
 	}()
 	go func() {
-		singleLines(stdErr).process(ctx, bufferLimitBytes, parsed)
+		singleLines(stdErr).process(ctx, parsed)
 		wg.Done()
 	}()
 
@@ -68,7 +69,7 @@ func (unparsed RawC) Process(ctx context.Context, bufferLimitBytes int, parsed c
 	}
 }
 
-func (lines singleLines) process(ctx context.Context, bufferLimitBytes int, parsed chan<- m.Record) {
+func (lines singleLines) process(ctx context.Context, parsed chan<- m.Record) {
 	var firstMetadata *m.ParsedMetadata
 	var buffer []byte
 
@@ -86,9 +87,9 @@ func (lines singleLines) process(ctx context.Context, bufferLimitBytes int, pars
 
 		buffer = append(buffer, line.B...)
 
-		if len(buffer) > bufferLimitBytes {
+		if len(buffer) > globals.BufferLimitBytes {
 			promRecordDroppedTooLarge.WithLabelValues(line.MetricsName).Add(1)
-			log.Printf("dropped record: size in bytes exceeds limit of %d", bufferLimitBytes)
+			log.Printf("dropped record: size in bytes exceeds limit of %d", globals.BufferLimitBytes)
 
 			buffer = nil
 			continue