logmower: Updates

logmower/application.yml

@@ -25,9 +25,10 @@ spec:
   additionalMongodConfig:
     systemLog:
       quiet: true
-  members: 3
+  members: 2
+  arbiters: 1
   type: ReplicaSet
-  version: "6.0.2"
+  version: "6.0.3"
   security:
     authentication:
       modes: ["SCRAM"]
@@ -62,6 +63,9 @@ spec:
                 limits:
                   cpu: 4000m
                   memory: 1Gi
+              volumeMounts:
+                - name: journal-volume
+                  mountPath: /data/journal
             - name: mongodb-agent
               resources:
                 requests:
@@ -88,8 +92,21 @@ spec:
       volumeClaimTemplates:
         - metadata:
             name: logs-volume
+            labels:
+              usecase: logs
           spec:
-            storageClassName: local-path
+            storageClassName: mongo
+            accessModes:
+            - ReadWriteOnce
+            resources:
+              requests:
+                storage: 100Mi
+        - metadata:
+            name: journal-volume
+            labels:
+              usecase: journal
+          spec:
+            storageClassName: mongo
             accessModes:
             - ReadWriteOnce
             resources:
@@ -97,8 +114,10 @@ spec:
                 storage: 512Mi
         - metadata:
             name: data-volume
+            labels:
+              usecase: data
           spec:
-            storageClassName: local-path
+            storageClassName: mongo
             accessModes:
             - ReadWriteOnce
             resources:
@@ -125,9 +144,7 @@ spec:
       serviceAccountName: logmower-shipper
       containers:
         - name: logmower-shipper
-          image: harbor.k-space.ee/k-space/logmower-shipper-prototype
-          securityContext:
-            runAsUser: 0
+          image: harbor.k-space.ee/k-space/logmower-shipper-prototype:latest
           env:
             - name: NODE_NAME
               valueFrom:
@@ -141,8 +158,10 @@ spec:
           ports:
             - containerPort: 8000
               name: metrics
+          securityContext:
+            readOnlyRootFilesystem: true
           command:
-            - /log_shipper.py
+            - /app/log_shipper.py
             - --parse-json
             - --normalize-log-level
             - --stream-to-log-level
@@ -159,9 +178,6 @@ spec:
             - name: etcmachineid
               mountPath: /etc/machine-id
               readOnly: true
-            - name: varlibdockercontainers
-              mountPath: /var/lib/docker/containers
-              readOnly: true
             - name: varlog
               mountPath: /var/log
               readOnly: true
@@ -169,9 +185,6 @@ spec:
         - name: etcmachineid
           hostPath:
             path: /etc/machine-id
-        - name: varlibdockercontainers
-          hostPath:
-            path: /var/lib/docker/containers
         - name: varlog
           hostPath:
             path: /var/log
@@ -282,6 +295,38 @@ spec:
   podMetricsEndpoints:
     - port: metrics
 ---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: logmower-shipper
+spec:
+  groups:
+    - name: logmower-shipper
+      rules:
+        - alert: LogmowerSingleInsertionErrors
+          annotations:
+            summary: Logmower shipper is having issues submitting log records
+              to database
+          expr: rate(logmower_insertion_error_count_total[30m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+        - alert: LogmowerBulkInsertionErrors
+          annotations:
+            summary: Logmower shipper is having issues submitting log records
+              to database
+          expr: rate(logmower_bulk_insertion_error_count_total[30m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+        - alert: LogmowerHighDatabaseLatency
+          annotations:
+            summary: Database operations are slow
+          expr: histogram_quantile(0.95, logmower_database_operation_latency_bucket) > 10
+          for: 1m
+          labels:
+            severity: warning
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -354,11 +399,35 @@ spec:
         app: logmower-frontend
     spec:
       containers:
-      - name: logmower-frontend
-        image: harbor.k-space.ee/k-space/logmower-frontend
-        ports:
-        - containerPort: 8080
-          name: http
+        - name: logmower-frontend
+          image: harbor.k-space.ee/k-space/logmower-frontend
+          ports:
+          - containerPort: 8080
+            name: http
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+          resources:
+            limits:
+              memory: 50Mi
+            requests:
+              cpu: 1m
+              memory: 20Mi
+          volumeMounts:
+          - name : nginx-cache
+            mountPath: /var/cache/nginx/
+          - name : nginx-config
+            mountPath: /var/config/nginx/
+          - name: var-run
+            mountPath: /var/run/
+      volumes:
+      - emptyDir: {}
+        name: nginx-cache
+      - emptyDir: {}
+        name: nginx-config
+      - emptyDir: {}
+        name: var-run
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -374,21 +443,31 @@ spec:
         app: logmower-eventsource
     spec:
       containers:
-      - name: logmower-eventsource
-        image: harbor.k-space.ee/k-space/logmower-eventsource
-        command:
-          - npm
-          - start
-        ports:
-          - containerPort: 3002
-            name: nodejs
-        env:
-          - name: MONGODB_HOST
-            valueFrom:
-              secretKeyRef:
-                name: logmower-mongodb-application-readonly
-                key: connectionString.standard
-
+        - name: logmower-eventsource
+          image: harbor.k-space.ee/k-space/logmower-eventsource
+          command:
+            - npm
+            - start
+          ports:
+            - containerPort: 3002
+              name: nodejs
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+          resources:
+            limits:
+              cpu: 500m
+              memory: 200Mi
+            requests:
+              cpu: 10m
+              memory: 100Mi
+          env:
+            - name: MONGODB_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: logmower-mongodb-application-readonly
+                  key: connectionString.standard
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy