logmower-shipper/logmower/lines.go

104 lines
2.0 KiB
Go

package logmower
import (
"log"
"sync"
ms "git.k-space.ee/k-space/logmower-shipper/pkg/mongo_struct"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var promRecordDroppedTooLarge = promauto.NewCounterVec(prom.CounterOpts{
Namespace: PrometheusPrefix,
// Subsystem: "record",
Name: "dropped_lines", // "dropped",
Help: "Records dropped due to being too large",
}, []string{"filename"})
type (
RawLines <-chan RawLine
RawLine struct {
*file
Offset int64
B []byte
}
)
// assumes all lines are from same file
func (unparsed RawLines) Process(bufferLimitBytes int, parsed chan<- ms.Record) {
lines := make(chan singleLine)
go unparsed.parse(lines)
var wg sync.WaitGroup
wg.Add(2)
stdOut, stdErr := make(chan singleLine), make(chan singleLine)
go func() {
singleLines(stdOut).process(bufferLimitBytes, parsed)
wg.Done()
}()
go func() {
singleLines(stdErr).process(bufferLimitBytes, parsed)
wg.Done()
}()
// split stdout and stderr
for {
line, ok := <-lines
if !ok {
close(stdOut)
close(stdErr)
wg.Wait()
close(parsed)
return
}
if line.StdErr {
stdErr <- line
} else {
stdOut <- line
}
}
}
func (lines singleLines) process(bufferLimitBytes int, parsed chan<- ms.Record) {
var firstMetadata *ms.ParsedMetadata
var buffer []byte
for {
line, ok := <-lines
if !ok {
// partial line should always be finished with full line
// discard any partial lines without end (full line)
return
}
if len(buffer) == 0 {
firstMetadata = &line.ParsedMetadata
}
buffer = append(buffer, line.B...)
if len(buffer) > bufferLimitBytes {
promRecordDroppedTooLarge.WithLabelValues(line.metricsName).Add(1)
log.Printf("dropped record: size in bytes exceeds limit of %d", bufferLimitBytes)
buffer = nil
continue
}
if !line.partial {
parsed <- ms.Record{
File: line.file.File,
Offset: line.Offset,
String: string(buffer),
ParsedMetadata: *firstMetadata,
}
buffer = nil
}
}
}