restructure project

pkg/lines/lines.go

@@ -0,0 +1,100 @@
+package lines
+
+import (
+	"log"
+	"sync"
+
+	m "git.k-space.ee/k-space/logmower-shipper/pkg/mongo"
+)
+
+type (
+	RawC <-chan Raw
+	Raw  struct {
+		*File
+		Offset int64
+		B      []byte
+	}
+
+	// file.File, but avoiding import cycle
+	File struct {
+		*m.File
+		MetricsName string // filepath.Base()
+	}
+)
+
+// assumes all lines are from same file
+func (unparsed RawC) Process(bufferLimitBytes int, parsed chan<- m.Record) {
+	lines := make(chan singleLine)
+	go unparsed.parse(lines)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	stdOut, stdErr := make(chan singleLine), make(chan singleLine)
+	go func() {
+		singleLines(stdOut).process(bufferLimitBytes, parsed)
+		wg.Done()
+	}()
+	go func() {
+		singleLines(stdErr).process(bufferLimitBytes, parsed)
+		wg.Done()
+	}()
+
+	// split stdout and stderr
+	for {
+		line, ok := <-lines
+		if !ok {
+			close(stdOut)
+			close(stdErr)
+			wg.Wait()
+			close(parsed)
+			return
+		}
+
+		if line.StdErr {
+			stdErr <- line
+		} else {
+			stdOut <- line
+		}
+	}
+}
+
+func (lines singleLines) process(bufferLimitBytes int, parsed chan<- m.Record) {
+	var firstMetadata *m.ParsedMetadata
+	var buffer []byte
+
+	for {
+		line, ok := <-lines
+		if !ok {
+			// partial line should always be finished with full line
+			// discard any partial lines without end (full line)
+			return
+		}
+
+		if len(buffer) == 0 {
+			firstMetadata = &line.ParsedMetadata
+		}
+
+		buffer = append(buffer, line.B...)
+
+		if len(buffer) > bufferLimitBytes {
+			promRecordDroppedTooLarge.WithLabelValues(line.MetricsName).Add(1)
+			log.Printf("dropped record: size in bytes exceeds limit of %d", bufferLimitBytes)
+
+			buffer = nil
+			continue
+		}
+
+		if !line.partial {
+			parsed <- m.Record{
+				File:   line.File.File,
+				Offset: line.Offset,
+
+				String:         string(buffer),
+				ParsedMetadata: *firstMetadata,
+			}
+
+			buffer = nil
+		}
+	}
+}

pkg/lines/lines_single.go

@@ -0,0 +1,66 @@
+package lines
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"time"
+
+	m "git.k-space.ee/k-space/logmower-shipper/pkg/mongo"
+)
+
+func (unparsed RawC) parse(parsed chan<- singleLine) {
+	for {
+		raw, ok := <-unparsed
+		if !ok {
+			close(parsed)
+			return
+		}
+
+		line := singleLine{Raw: raw}
+
+		if err := line.parse(); err != nil {
+			promRecordPrefixParsingErr.WithLabelValues(raw.MetricsName).Add(1)
+			log.Printf("parsing kubernetes log line in %q: %e", raw.File.Path, err)
+		}
+
+		// TODO: should this only be on success?
+		parsed <- line
+	}
+}
+
+type (
+	singleLines <-chan singleLine
+	singleLine  struct {
+		Raw
+
+		// populated by parse()
+		m.ParsedMetadata
+		partial bool // P or F
+	}
+)
+
+func (line *singleLine) parse() (err error) {
+	split := bytes.SplitN(line.B, []byte(" "), 4)
+	if len(split) != 4 {
+		return fmt.Errorf("expected at least 3 spaces in , got %d", len(split)-1)
+	}
+
+	line.TimeKubernetes, err = time.Parse(time.RFC3339Nano, string(split[0]))
+	if err != nil {
+		return fmt.Errorf("invalid time: %w", err)
+	}
+
+	line.StdErr = string(split[1]) == "stderr" // or stdout
+
+	switch string(split[2]) {
+	case "P":
+		line.partial = true
+	case "F":
+	default:
+		return fmt.Errorf("partial indicator must be 'P' or 'F', not %q", split[2])
+	}
+
+	line.B = split[3]
+	return nil
+}

pkg/lines/metrics.go

@@ -0,0 +1,23 @@
+package lines
+
+import (
+	"git.k-space.ee/k-space/logmower-shipper/pkg/globals"
+	prom "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	promRecordDroppedTooLarge = promauto.NewCounterVec(prom.CounterOpts{
+		Namespace: globals.PrometheusPrefix,
+		// Subsystem: "record",
+		Name: "dropped_lines", // "dropped",
+		Help: "Records dropped due to being too large",
+	}, []string{"filename"})
+
+	promRecordPrefixParsingErr = promauto.NewCounterVec(prom.CounterOpts{
+		Namespace: globals.PrometheusPrefix,
+		Subsystem: "record",
+		Name:      "parsing_errors",
+		Help:      "Errors while parsing log line prefixes",
+	}, []string{"filename"})
+)