add skaffold and fix bugs

README.md

@@ -14,3 +14,13 @@ Watches log directory for logs, and ships them to mongo.
 3. `pkg/file` handles file lifecycle; watches files and tails them, streaming lines to `pkg/lines`.
 4. `pkg/lines` processes lines and streams them to `pkg/sender`.
 5. `pkg/sender` batches lines and ships them to mongo.
+
+## Skaffold
+```bash
+export NS=gitdbd-iig6x
+kubectl apply -f k8s/dev/mongodb.yaml -n "$NS"
+kubectl apply -f k8s/dev/prom.yaml -n "$NS"
+skaffold dev --namespace "$NS"
+
+hwatch kubectl get pods --namespace "$NS"
+```

k8s/dev/logmower.yaml

@@ -0,0 +1,78 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: log-shipper
+spec:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 50%
+  selector:
+    matchLabels:
+      app: log-shipper
+  template:
+    metadata:
+      labels:
+        app: log-shipper
+    spec:
+      serviceAccountName: log-shipper
+      containers:
+        - name: log-shipper
+          image: harbor.k-space.ee/rasmus/logmower-shipper
+          securityContext:
+            runAsUser: 0
+          env:
+            - name: KUBE_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: MONGODB_URI
+              valueFrom:
+                secretKeyRef:
+                  name: mongodb-application-readwrite
+                  key: connectionString.standard
+          ports:
+            - containerPort: 8000
+              name: metrics
+          resources:
+            limits:
+              memory: 200Mi
+            requests:
+              cpu: 100m
+              memory: 100Mi
+          volumeMounts:
+            - name: etcmachineid
+              mountPath: /etc/machine-id
+              readOnly: true
+            - name: varlibdockercontainers
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+            - name: varlog
+              mountPath: /var/log
+              readOnly: true
+      volumes:
+        - name: etcmachineid
+          hostPath:
+            path: /etc/machine-id
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: varlog
+          hostPath:
+            path: /var/log
+      tolerations:
+        - operator: "Exists"
+          effect: "NoExecute"
+        - operator: "Exists"
+          effect: "NoSchedule"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: log-shipper
+  labels:
+    app: log-shipper

k8s/dev/mongodb.yaml

@@ -0,0 +1,268 @@
+---
+apiVersion: codemowers.io/v1alpha1
+kind: GeneratedSecret
+metadata:
+  name: mongodb-application-readwrite-password
+spec:
+  mapping:
+    - key: password
+      value: "%(password)s"
+---
+apiVersion: codemowers.io/v1alpha1
+kind: GeneratedSecret
+metadata:
+  name: mongodb-application-readonly-password
+spec:
+  mapping:
+    - key: password
+      value: "%(password)s"
+---
+apiVersion: mongodbcommunity.mongodb.com/v1
+kind: MongoDBCommunity
+metadata:
+  name: mongodb
+spec:
+  additionalMongodConfig:
+    systemLog:
+      quiet: true
+  members: 3
+  type: ReplicaSet
+  version: "5.0.9"
+  security:
+    authentication:
+      modes: ["SCRAM"]
+  users:
+    - name: readwrite
+      db: application
+      passwordSecretRef:
+        name: mongodb-application-readwrite-password
+      roles:
+        - name: readWrite
+          db: application
+      scramCredentialsSecretName: mongodb-application-readwrite
+    - name: readonly
+      db: application
+      passwordSecretRef:
+        name: mongodb-application-readonly-password
+      roles:
+        - name: readOnly
+          db: application
+      scramCredentialsSecretName: mongodb-application-readonly
+  statefulSet:
+    spec:
+      logLevel: WARN
+      template:
+        spec:
+          containers:
+            - name: mongod
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 1Gi
+                limits:
+                  cpu: 4000m
+                  memory: 1Gi
+            - name: mongodb-agent
+              resources:
+                requests:
+                  cpu: 1m
+                  memory: 100Mi
+                limits: { }
+          affinity:
+            podAntiAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchExpressions:
+                      - key: app
+                        operator: In
+                        values:
+                          - mongodb-svc
+                  topologyKey: kubernetes.io/hostname
+      volumeClaimTemplates:
+        - metadata:
+            name: logs-volume
+          spec:
+            storageClassName: openebs-hostpath
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 512Mi
+        - metadata:
+            name: data-volume
+          spec:
+            storageClassName: openebs-hostpath
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 2Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mongoexpress
+spec:
+  revisionHistoryLimit: 0
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mongoexpress
+  template:
+    metadata:
+      labels:
+        app: mongoexpress
+    spec:
+      containers:
+        - name: mongoexpress
+          image: mongo-express
+          ports:
+            - name: mongoexpress
+              containerPort: 8081
+          env:
+            - name: ME_CONFIG_MONGODB_URL
+              valueFrom:
+                secretKeyRef:
+                  name: mongodb-application-readwrite
+                  key: connectionString.standard
+            - name: ME_CONFIG_MONGODB_ENABLE_ADMIN
+              value: "true"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mongoexpress
+  labels:
+    app: mongoexpress
+spec:
+  selector:
+    app: mongoexpress
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8081
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: mongoexpress
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+    external-dns.alpha.kubernetes.io/target: traefik-iig6x.codemowers.ee
+spec:
+  rules:
+    - host: mongoexpress-iig6x.codemowers.ee
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: mongoexpress
+                port:
+                  number: 80
+  tls:
+    - hosts:
+        - "*.codemowers.ee"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mongodb-kubernetes-operator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - services
+      - configmaps
+      - secrets
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - mongodbcommunity.mongodb.com
+    resources:
+      - mongodbcommunity
+      - mongodbcommunity/status
+      - mongodbcommunity/spec
+      - mongodbcommunity/finalizers
+    verbs:
+      - get
+      - patch
+      - list
+      - update
+      - watch
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mongodb-database
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - patch
+      - delete
+      - get
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mongodb-kubernetes-operator
+subjects:
+  - kind: ServiceAccount
+    name: mongodb-kubernetes-operator
+roleRef:
+  kind: Role
+  name: mongodb-kubernetes-operator
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mongodb-database
+subjects:
+  - kind: ServiceAccount
+    name: mongodb-database
+roleRef:
+  kind: Role
+  name: mongodb-database
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mongodb-kubernetes-operator
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mongodb-database

k8s/dev/prom.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: logmower-shipper
+spec:
+  selector:
+    matchLabels:
+      app: logmower-shipper
+  podMetricsEndpoints:
+    - port: metrics

pkg/mongo/metrics.go

@@ -1,7 +1,6 @@
 package mongo
 
 import (
-	"context"
 	"log"
 	"time"
 
@@ -14,27 +13,12 @@ import (
 
 const promSubsystem = "database"
 
-var (
-	promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{
-		Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
-		Name:    "heartbeat_time",
-		Help:    "Time in seconds for succeeded heartbeat, or 0 on failure",
-		Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
-	}, []string{"connection_id"})
-
-	promDbCmd = promauto.NewHistogramVec(prom.HistogramOpts{
-		Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
-		Name:    "operation_latency", // "command_time",
-		Help:    "Time in seconds of commands",
-		Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
-	}, []string{"connection_id", "command_name"})
-
-	promDbCmdErr = promauto.NewCounterVec(prom.CounterOpts{
-		Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
-		Name: "errors",
-		Help: "Failed commands (also reflected elsewhere)",
-	}, []string{"connection_id", "command_name"})
-)
+var promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{
+	Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
+	Name:    "heartbeat_time",
+	Help:    "Time in seconds for succeeded heartbeat, or 0 on failure",
+	Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
+}, []string{"connection_id"})
 
 func monitoredClientOptions() *mongoOpt.ClientOptions {
 	return mongoOpt.Client().
@@ -46,15 +30,5 @@ func monitoredClientOptions() *mongoOpt.ClientOptions {
 				promDbHeartbeat.WithLabelValues(ev.ConnectionID).Observe(0)
 				log.Printf("database heartbeat failed on connection %q: %e", ev.ConnectionID, ev.Failure)
 			},
-		}).
-		SetMonitor(&mongoEvent.CommandMonitor{
-			Succeeded: func(_ context.Context, ev *mongoEvent.CommandSucceededEvent) {
-				promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds())
-			},
-			Failed: func(_ context.Context, ev *mongoEvent.CommandFailedEvent) {
-				promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds())
-
-				promDbCmdErr.WithLabelValues(ev.ConnectionID, ev.CommandName).Add(1)
-			},
 		})
 }

pkg/mongo/mongo.go

@@ -22,11 +22,15 @@ func Initialize(ctx context.Context, uri string) (*mongo.Collection, error) {
 
 	dbOpt := monitoredClientOptions().ApplyURI(uri)
 
-	dbClient, err := mongo.Connect(globals.MongoTimeout(ctx))
+	dbClient, err := mongo.Connect(globals.MongoTimeout(ctx), dbOpt)
 	if err != nil {
 		return nil, fmt.Errorf("connecting to %q: %w", dbOpt.GetURI(), err)
 	}
 
+	if err := dbClient.Ping(globals.MongoTimeout(ctx), nil); err != nil {
+		return nil, fmt.Errorf("first ping to database: %w", err)
+	}
+
 	col := dbClient.Database(uriParsed.Path).Collection("logs")
 
 	if err := InitializeIndexes(globals.MongoTimeout(ctx), col); err != nil {

pkg/watcher/watcher.go

@@ -32,10 +32,10 @@ var App = &cli.App{
 		//TODO: &cli.BoolFlag{Name: "normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords"},
 		//TODO: &cli.BoolFlag{Name: "parse-json"},
 		//
-		&cli.StringSliceFlag{Category: "selectors", Name: "pod-namespace", EnvVars: []string{"KUBE_POD_NAMESPACE"}, Usage: "whitelist filter for filenames"},
+		&cli.StringSliceFlag{Category: "selectors", Name: "namespace", EnvVars: []string{"KUBE_NAMESPACE"}, Usage: "whitelist filter for filenames"},
 		&cli.StringSliceFlag{Category: "selectors", Name: "pod-prefix", EnvVars: []string{"KUBE_NODE_NAME"}, Usage: "blacklist filter for filenames"},
 		//
-		&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGO_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
+		&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGODB_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
 	},
 	Before: func(ctx *cli.Context) error {
 		globals.BufferLimitBytes = ctx.Int("max-record-size")
@@ -49,7 +49,7 @@ var App = &cli.App{
 	},
 
 	Action: func(ctx *cli.Context) error {
-		whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("pod-namespace")), ctx.StringSlice("pod-prefix")
+		whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("namespace")), ctx.StringSlice("pod-prefix")
 		var wg sync.WaitGroup
 
 		log.Printf("%s %s starting", ctx.App.Name, ctx.App.Version)

skaffold.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: skaffold/v3alpha1
+kind: Config
+metadata:
+  name: logmower-shipper
+build:
+  artifacts:
+  - image: harbor.k-space.ee/rasmus/logmower-shipper
+    docker:
+      dockerfile: Dockerfile
+deploy:
+  kubectl: {}
+# manifests:
+  # rawYaml:
+    #  - k8s/staging/deployment.yaml
+profiles:
+  - name: dev
+    activation:
+      - command: dev
+    build:
+      artifacts:
+        - image: harbor.k-space.ee/rasmus/logmower-shipper
+    manifests:
+      rawYaml:
+        - k8s/dev/logmower.yaml
+        # - k8s/dev/mongodb.yaml