logmower-shipper/pkg/lines/lines.go

116 lines
2.0 KiB
Go
Raw Normal View History

2022-11-09 16:07:28 +00:00
package lines
2022-11-06 20:02:29 +00:00
import (
"context"
"log"
2022-11-06 20:02:29 +00:00
"sync"
"git.k-space.ee/k-space/logmower-shipper/pkg/globals"
2022-11-09 16:07:28 +00:00
m "git.k-space.ee/k-space/logmower-shipper/pkg/mongo"
2022-11-06 20:02:29 +00:00
)
type (
2022-11-09 16:07:28 +00:00
RawC <-chan Raw
Raw struct {
*File
Offset int64
B []byte
2022-11-06 20:02:29 +00:00
}
2022-11-09 16:07:28 +00:00
// file.File, but avoiding import cycle
File struct {
*m.File
MetricsName string // filepath.Base()
}
2022-11-06 20:02:29 +00:00
)
// assumes all lines are from same file
func (unparsed RawC) Process(ctx context.Context, parsed chan<- m.Record) {
2022-11-06 20:02:29 +00:00
lines := make(chan singleLine)
go unparsed.parse(lines)
2022-11-06 20:02:29 +00:00
var wg sync.WaitGroup
wg.Add(2)
stdOut, stdErr := make(chan singleLine), make(chan singleLine)
go func() {
singleLines(stdOut).process(ctx, parsed)
2022-11-06 20:02:29 +00:00
wg.Done()
}()
go func() {
singleLines(stdErr).process(ctx, parsed)
2022-11-06 20:02:29 +00:00
wg.Done()
}()
defer func() {
close(stdOut)
close(stdErr)
wg.Wait()
close(parsed)
}()
2022-11-06 20:02:29 +00:00
// split stdout and stderr
for {
select {
case <-ctx.Done():
2022-11-06 20:02:29 +00:00
return
case line, ok := <-lines:
if !ok {
return
}
if line.StdErr {
stdErr <- line
} else {
stdOut <- line
}
2022-11-06 20:02:29 +00:00
}
}
}
func (lines singleLines) process(ctx context.Context, parsed chan<- m.Record) {
2022-11-09 16:07:28 +00:00
var firstMetadata *m.ParsedMetadata
2022-11-06 20:02:29 +00:00
var buffer []byte
for {
line, ok := <-lines
if !ok {
// partial line should always be finished with full line
// discard any partial lines without end (full line)
2022-11-06 20:02:29 +00:00
return
}
if len(buffer) == 0 {
firstMetadata = &line.ParsedMetadata
2022-11-06 20:02:29 +00:00
}
buffer = append(buffer, line.B...)
2022-11-06 20:02:29 +00:00
if len(buffer) > globals.BufferLimitBytes {
2022-11-09 16:07:28 +00:00
promRecordDroppedTooLarge.WithLabelValues(line.MetricsName).Add(1)
log.Printf("dropped record: size in bytes exceeds limit of %d", globals.BufferLimitBytes)
2022-11-06 20:02:29 +00:00
buffer = nil
continue
}
if !line.partial {
select {
case <-ctx.Done():
return
case parsed <- m.Record{
2022-11-09 16:07:28 +00:00
File: line.File.File,
Offset: line.Offset,
String: string(buffer),
ParsedMetadata: *firstMetadata,
}:
}
buffer = nil
2022-11-06 20:02:29 +00:00
}
}
}