logmower-shipper/logmower/lines.go

package logmower

import (
	"log"
	"sync"

	ms "git.k-space.ee/k-space/logmower-shipper/pkg/mongo_struct"
	prom "github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

var promRecordDroppedTooLarge = promauto.NewCounterVec(prom.CounterOpts{
	Namespace: PrometheusPrefix,
	// Subsystem: "record",
	Name: "dropped_lines", // "dropped",
	Help: "Records dropped due to being too large",
}, []string{"filename"})

type (
	RawLines <-chan RawLine
	RawLine  struct {
		*file
		Offset int64
		B      []byte
	}
)

// assumes all lines are from same file
func (unparsed RawLines) Process(bufferLimitBytes int, parsed chan<- ms.Record) {
	lines := make(chan singleLine)
	go unparsed.parse(lines)

	var wg sync.WaitGroup
	wg.Add(2)

	stdOut, stdErr := make(chan singleLine), make(chan singleLine)
	go func() {
		singleLines(stdOut).process(bufferLimitBytes, parsed)
		wg.Done()
	}()
	go func() {
		singleLines(stdErr).process(bufferLimitBytes, parsed)
		wg.Done()
	}()

	// split stdout and stderr
	for {
		line, ok := <-lines
		if !ok {
			close(stdOut)
			close(stdErr)
			wg.Wait()
			close(parsed)
			return
		}

		if line.StdErr {
			stdErr <- line
		} else {
			stdOut <- line
		}
	}
}

func (lines singleLines) process(bufferLimitBytes int, parsed chan<- ms.Record) {
	var firstMetadata *ms.ParsedMetadata
	var buffer []byte

	for {
		line, ok := <-lines
		if !ok {
			// partial line should always be finished with full line
			// discard any partial lines without end (full line)
			return
		}

		if len(buffer) == 0 {
			firstMetadata = &line.ParsedMetadata
		}

		buffer = append(buffer, line.B...)

		if len(buffer) > bufferLimitBytes {
			promRecordDroppedTooLarge.WithLabelValues(line.metricsName).Add(1)
			log.Printf("dropped record: size in bytes exceeds limit of %d", bufferLimitBytes)

			buffer = nil
			continue
		}

		if !line.partial {
			parsed <- ms.Record{
				File:   line.file.File,
				Offset: line.Offset,

				String:         string(buffer),
				ParsedMetadata: *firstMetadata,
			}

			buffer = nil
		}
	}
}