restructure project

pkg/mongo/bson_lookup.go

@@ -0,0 +1,29 @@
+package mongo
+
+import (
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+// default values without ok
+
+func bsonLookupBoolean(b *bson.Raw, key ...string) bool {
+	v, _ := b.Lookup(key...).BooleanOK()
+	return v
+}
+
+func bsonLookupStringValue(b *bson.Raw, key ...string) string {
+	v, _ := b.Lookup(key...).StringValueOK()
+	return v
+}
+
+func bsonLookupInt64(b *bson.Raw, key ...string) int64 {
+	v, _ := b.Lookup(key...).Int64OK()
+	return v
+}
+
+func bsonLookupTime(b *bson.Raw, key ...string) time.Time {
+	v, _ := b.Lookup(key...).TimeOK()
+	return v
+}

pkg/mongo/metrics.go

@@ -0,0 +1,60 @@
+package mongo
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"git.k-space.ee/k-space/logmower-shipper/pkg/globals"
+	prom "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	mongoEvent "go.mongodb.org/mongo-driver/event"
+	mongoOpt "go.mongodb.org/mongo-driver/mongo/options"
+)
+
+const promSubsystem = "database"
+
+var (
+	promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{
+		Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
+		Name:    "heartbeat_time",
+		Help:    "Time in seconds for succeeded heartbeat, or 0 on failure",
+		Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
+	}, []string{"connection_id"})
+
+	promDbCmd = promauto.NewHistogramVec(prom.HistogramOpts{
+		Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
+		Name:    "operation_latency", // "command_time",
+		Help:    "Time in seconds of commands",
+		Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
+	}, []string{"connection_id", "command_name"})
+
+	promDbCmdErr = promauto.NewCounterVec(prom.CounterOpts{
+		Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
+		Name: "errors",
+		Help: "Failed commands (also reflected elsewhere)",
+	}, []string{"connection_id", "command_name"})
+)
+
+func monitoredClientOptions() *mongoOpt.ClientOptions {
+	return mongoOpt.Client().
+		SetServerMonitor(&mongoEvent.ServerMonitor{
+			ServerHeartbeatSucceeded: func(ev *mongoEvent.ServerHeartbeatSucceededEvent) {
+				promDbHeartbeat.WithLabelValues(ev.ConnectionID).Observe(time.Duration(ev.DurationNanos).Seconds())
+			},
+			ServerHeartbeatFailed: func(ev *mongoEvent.ServerHeartbeatFailedEvent) {
+				promDbHeartbeat.WithLabelValues(ev.ConnectionID).Observe(0)
+				log.Printf("database heartbeat failed on connection %q: %e", ev.ConnectionID, ev.Failure)
+			},
+		}).
+		SetMonitor(&mongoEvent.CommandMonitor{
+			Succeeded: func(_ context.Context, ev *mongoEvent.CommandSucceededEvent) {
+				promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds())
+			},
+			Failed: func(_ context.Context, ev *mongoEvent.CommandFailedEvent) {
+				promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds())
+
+				promDbCmdErr.WithLabelValues(ev.ConnectionID, ev.CommandName).Add(1)
+			},
+		})
+}

pkg/mongo/mongo.go

@@ -0,0 +1,37 @@
+package mongo
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"git.k-space.ee/k-space/logmower-shipper/pkg/globals"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+func Initialize(ctx context.Context, uri string) (*mongo.Collection, error) {
+	uriParsed, err := url.ParseRequestURI(uri)
+	if err != nil {
+		return nil, fmt.Errorf("parsing URI for database name: %w", err)
+	}
+
+	uriParsed.Path = uriParsed.Path[1:] // remove leading slash
+	if uriParsed.Path == "" {
+		return nil, fmt.Errorf("URI must include database name (as database to authenticate against)")
+	}
+
+	dbOpt := monitoredClientOptions().ApplyURI(uri)
+
+	dbClient, err := mongo.Connect(globals.MongoTimeout(ctx))
+	if err != nil {
+		return nil, fmt.Errorf("connecting to %q: %w", dbOpt.GetURI(), err)
+	}
+
+	col := dbClient.Database(uriParsed.Path).Collection("logs")
+
+	if err := InitializeIndexes(globals.MongoTimeout(ctx), col); err != nil {
+		return nil, fmt.Errorf("initializing indexes: %w", err)
+	}
+
+	return col, nil
+}

pkg/mongo/mongo_struct.go

@@ -0,0 +1,149 @@
+package mongo
+
+import (
+	"context"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+// ctx is used directly
+func InitializeIndexes(ctx context.Context, col *mongo.Collection) error {
+	ind := col.Indexes()
+
+	// (does not create duplicates)
+	_, err := ind.CreateOne(ctx, mongo.IndexModel{
+		Keys: bson.D{{Key: RecordKeyFilePath, Value: 1}, {Key: RecordKeyOffset, Value: -1}},
+	})
+
+	return err
+}
+
+// when editing, also edit everything in this file!
+type (
+	Record struct {
+		*File
+		Offset int64 // end, of last line
+
+		String string
+		ParsedMetadata
+
+		ParsedContent // TODO: not implemented
+
+		// added by ToBson()
+		timeShip time.Time
+	}
+	ParsedMetadata struct {
+		TimeKubernetes time.Time
+		StdErr         bool
+	}
+	ParsedContent struct {
+		Content      any
+		TimeUpstream time.Time
+	}
+	Host struct {
+		Id   string
+		Name string
+		Arch string
+	}
+	File struct {
+		Host *Host
+		Path string // absolute
+		KubeInfo
+	}
+	KubeInfo struct {
+		ContainerName string
+		ContainerId   string // unused
+		Namespace     string
+		Pod           string
+	}
+)
+
+const (
+	// used outside package for mongo commands
+	RecordKeyHostId   = recordKeyHost + "." + recordKeyId
+	RecordKeyFilePath = recordKeyLog + "." + recordKeyFile + "." + recordKeygenericPath
+	RecordKeyOffset   = recordKeyLog + "." + recordKeyOffset
+)
+
+// Don't use direct strings in bson types. Use the constants as keys.
+// This ensures keys (and subkeys) are consistent within the package, and by consumers of it.
+
+const (
+	recordKeygenericName = "name"
+	recordKeygenericPath = "path"
+)
+
+const (
+	recordKeyString = "message"
+
+	recordKeyLog    = "log"
+	recordKeyFile   = "file"
+	recordKeyOffset = "offset"
+	// recordKeyLevel  = "level"
+
+	recordKeyHost = "host"
+	recordKeyId   = "id"
+	recordKeyName = "name"
+	recordKeyArch = "architecture"
+
+	recordKeyKubernetes = "kubernetes"
+	recordKeyContainer  = "container"
+	recordKeyNamespace  = "namespace"
+	recordKeyPod        = "pod"
+
+	recordKeyStream = "stream"
+
+	recordKeyEvent          = "event"
+	recordKeyTimeUpstream   = "created"
+	recordKeyTimeKubernetes = "ingested"
+
+	recordKeyTimeMower = "@timestamp"
+)
+
+// not using marshal, since it is <0.1x performance
+func (l *Record) ToBson() bson.M {
+	var stream string
+	if l.StdErr {
+		stream = "stderr"
+	} else {
+		stream = "stdout"
+	}
+
+	return bson.M{
+		recordKeyString: l.String,
+		recordKeyLog: bson.M{
+			recordKeyFile: bson.M{
+				recordKeygenericPath: l.File.Path,
+			},
+			recordKeyOffset: l.Offset,
+			// recordKeyLevel:  , //TODO: ECS
+		},
+		recordKeyKubernetes: bson.M{
+			recordKeyContainer: bson.M{
+				recordKeygenericName: l.File.ContainerName,
+			},
+			recordKeyNamespace: l.File.Namespace,
+			recordKeyPod: bson.M{
+				recordKeygenericName: l.File.Pod,
+			},
+		},
+		recordKeyHost: bson.M{
+			recordKeyId:   l.File.Host.Id,
+			recordKeyName: l.File.Host.Name,
+			recordKeyArch: l.File.Host.Arch,
+		},
+		recordKeyStream: stream,
+		recordKeyEvent: bson.M{
+			// recordKeyTimeUpstream:   l.TimeUpstream, //TODO: ECS
+			recordKeyTimeKubernetes: l.TimeKubernetes,
+		},
+
+		recordKeyTimeMower: time.Now(),
+	}
+}
+
+func RecordOffsetFromBson(b *bson.Raw) int64 {
+	return bsonLookupInt64(b, recordKeyLog, recordKeyOffset)
+}