104 lines
2.0 KiB
Go
104 lines
2.0 KiB
Go
package logmower
|
|
|
|
import (
|
|
"log"
|
|
"sync"
|
|
|
|
ms "git.k-space.ee/k-space/logmower-shipper/pkg/mongo_struct"
|
|
prom "github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
)
|
|
|
|
var promRecordDroppedTooLarge = promauto.NewCounterVec(prom.CounterOpts{
|
|
Namespace: PrometheusPrefix,
|
|
// Subsystem: "record",
|
|
Name: "dropped_lines", // "dropped",
|
|
Help: "Records dropped due to being too large",
|
|
}, []string{"filename"})
|
|
|
|
type (
|
|
RawLines <-chan RawLine
|
|
RawLine struct {
|
|
*file
|
|
Offset int64
|
|
B []byte
|
|
}
|
|
)
|
|
|
|
// assumes all lines are from same file
|
|
func (unparsed RawLines) Process(bufferLimitBytes int, parsed chan<- ms.Record) {
|
|
lines := make(chan singleLine)
|
|
go unparsed.parse(lines)
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
|
|
stdOut, stdErr := make(chan singleLine), make(chan singleLine)
|
|
go func() {
|
|
singleLines(stdOut).process(bufferLimitBytes, parsed)
|
|
wg.Done()
|
|
}()
|
|
go func() {
|
|
singleLines(stdErr).process(bufferLimitBytes, parsed)
|
|
wg.Done()
|
|
}()
|
|
|
|
// split stdout and stderr
|
|
for {
|
|
line, ok := <-lines
|
|
if !ok {
|
|
close(stdOut)
|
|
close(stdErr)
|
|
wg.Wait()
|
|
close(parsed)
|
|
return
|
|
}
|
|
|
|
if line.StdErr {
|
|
stdErr <- line
|
|
} else {
|
|
stdOut <- line
|
|
}
|
|
}
|
|
}
|
|
|
|
func (lines singleLines) process(bufferLimitBytes int, parsed chan<- ms.Record) {
|
|
var firstMetadata *ms.ParsedMetadata
|
|
var buffer []byte
|
|
|
|
for {
|
|
line, ok := <-lines
|
|
if !ok {
|
|
// partial line should always be finished with full line
|
|
// discard any partial lines without end (full line)
|
|
return
|
|
}
|
|
|
|
if len(buffer) == 0 {
|
|
firstMetadata = &line.ParsedMetadata
|
|
}
|
|
|
|
buffer = append(buffer, line.B...)
|
|
|
|
if len(buffer) > bufferLimitBytes {
|
|
promRecordDroppedTooLarge.WithLabelValues(line.metricsName).Add(1)
|
|
log.Printf("dropped record: size in bytes exceeds limit of %d", bufferLimitBytes)
|
|
|
|
buffer = nil
|
|
continue
|
|
}
|
|
|
|
if !line.partial {
|
|
parsed <- ms.Record{
|
|
File: line.file.File,
|
|
Offset: line.Offset,
|
|
|
|
String: string(buffer),
|
|
ParsedMetadata: *firstMetadata,
|
|
}
|
|
|
|
buffer = nil
|
|
}
|
|
}
|
|
}
|