logging: Replace Graylog with ZincSearch

logging/README.md

@@ -14,7 +14,7 @@ To deploy:
 
 ```
 kubectl create namespace logging
-kubectl apply -n logging -f mongodb-support.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml
+kubectl apply -n logging -f zinc.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml
 kubectl rollout restart -n logging daemonset.apps/filebeat
 ```
 

logging/application.yml

@@ -1,86 +1,4 @@
 ---
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: elasticsearch
-  labels:
-    app: elasticsearch
-spec:
-  serviceName: elasticsearch
-  revisionHistoryLimit: 0
-  replicas: 1
-  selector:
-    matchLabels:
-      app: elasticsearch
-  template:
-    metadata:
-      labels:
-        app: elasticsearch
-    spec:
-      securityContext:
-        fsGroup: 1000
-      containers:
-        - name: elasticsearch
-          image: elasticsearch:7.17.3
-          securityContext:
-            runAsNonRoot: true
-            runAsUser: 1000
-          env:
-            - name: discovery.type
-              value: single-node
-            - name: xpack.security.enabled
-              value: "false"
-          ports:
-            - containerPort: 9200
-          readinessProbe:
-            httpGet:
-              path: /_cluster/health
-              port: 9200
-            initialDelaySeconds: 5
-            periodSeconds: 10
-            failureThreshold: 3
-            successThreshold: 1
-            timeoutSeconds: 5
-          resources:
-            limits:
-              memory: "2147483648"
-          volumeMounts:
-          - name: elasticsearch-data
-            mountPath: /usr/share/elasticsearch/data
-          - name: elasticsearch-tmp
-            mountPath: /tmp/
-      volumes:
-      - emptyDir: {}
-        name: elasticsearch-keystore
-      - emptyDir: {}
-        name: elasticsearch-tmp
-      - emptyDir: {}
-        name: elasticsearch-logs
-  volumeClaimTemplates:
-    - metadata:
-        name: elasticsearch-data
-      spec:
-        accessModes:
-          - "ReadWriteOnce"
-        resources:
-          requests:
-            storage: "10Gi"
-        storageClassName: longhorn
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: elasticsearch
-  labels:
-    app: elasticsearch
-spec:
-  ports:
-  - name: api
-    port: 80
-    targetPort: 9200
-  selector:
-    app: elasticsearch
----
 apiVersion: v1
 kind: Service
 metadata:
@@ -149,304 +67,3 @@ spec:
       protocol: UDP
   selector:
     app: graylog
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: graylog
-  labels:
-    app: graylog
-spec:
-  ports:
-    - name: graylog
-      port: 9000
-      protocol: TCP
-  selector:
-    app: graylog
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: graylog
-  labels:
-    app: graylog
-  annotations:
-    keel.sh/policy: minor
-    keel.sh/trigger: poll
-    keel.sh/pollSchedule: "@midnight"
-spec:
-  serviceName: graylog
-  revisionHistoryLimit: 0
-  replicas: 1
-  selector:
-    matchLabels:
-      app: graylog
-  template:
-    metadata:
-      labels:
-        app: graylog
-      annotations:
-        prometheus.io/port: "9833"
-        prometheus.io/scrape: "true"
-    spec:
-      securityContext:
-        fsGroup: 1100
-      volumes:
-        - name: graylog-config
-          downwardAPI:
-            items:
-              - path: id
-                fieldRef:
-                  fieldPath: metadata.name
-      containers:
-        - name: graylog
-          image: graylog/graylog:4.3
-          env:
-            - name: GRAYLOG_MONGODB_URI
-              valueFrom:
-                secretKeyRef:
-                  name: mongodb-application-readwrite
-                  key: connectionString.standard
-            - name: GRAYLOG_PROMETHEUS_EXPORTER_ENABLED
-              value: "true"
-            - name: GRAYLOG_PROMETHEUS_EXPORTER_BIND_ADDRESS
-              value: "0.0.0.0:9833"
-            - name: GRAYLOG_NODE_ID_FILE
-              value: /config/id
-            - name: GRAYLOG_HTTP_EXTERNAL_URI
-              value: "https://graylog.k-space.ee/"
-            - name: GRAYLOG_TRUSTED_PROXIES
-              value: "0.0.0.0/0"
-            - name: GRAYLOG_ELASTICSEARCH_HOSTS
-              value: "http://elasticsearch"
-            - name: GRAYLOG_MESSAGE_JOURNAL_ENABLED
-              value: "false"
-            - name: GRAYLOG_ROTATION_STRATEGY
-              value: "size"
-            - name: GRAYLOG_ELASTICSEARCH_MAX_SIZE_PER_INDEX
-              value: "268435456"
-            - name: GRAYLOG_ELASTICSEARCH_MAX_NUMBER_OF_INDICES
-              value: "16"
-          envFrom:
-            - secretRef:
-                name: graylog-secrets
-          securityContext:
-            runAsNonRoot: true
-            runAsUser: 1100
-          ports:
-            - containerPort: 9000
-              name: graylog
-            - containerPort: 9833
-              name: graylog-metrics
-          livenessProbe:
-            httpGet:
-              path: /api/system/lbstatus
-              port: 9000
-            initialDelaySeconds: 5
-            periodSeconds: 30
-            failureThreshold: 3
-            successThreshold: 1
-            timeoutSeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /api/system/lbstatus
-              port: 9000
-            initialDelaySeconds: 5
-            periodSeconds: 10
-            failureThreshold: 3
-            successThreshold: 1
-            timeoutSeconds: 5
-          volumeMounts:
-            - name: graylog-config
-              mountPath: /config
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: graylog
-  annotations:
-    cert-manager.io/cluster-issuer: default
-    traefik.ingress.kubernetes.io/router.entrypoints: websecure
-    traefik.ingress.kubernetes.io/router.tls: "true"
-    external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
-    traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd
-spec:
-  rules:
-  - host: graylog.k-space.ee
-    http:
-      paths:
-      - pathType: Prefix
-        path: "/"
-        backend:
-          service:
-            name: graylog
-            port:
-              number: 9000
-  tls:
-  - hosts:
-    - graylog.k-space.ee
-    secretName: graylog-tls
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: graylog
-spec:
-  podSelector:
-    matchLabels:
-      app: graylog
-  policyTypes:
-  - Ingress
-  - Egress
-  egress:
-  - to:
-    - podSelector:
-        matchLabels:
-          app: elasticsearch
-    ports:
-    - port: 9200
-  - to:
-    - podSelector:
-        matchLabels:
-          app: mongodb-svc
-    ports:
-    - port: 27017
-  ingress:
-  - from:
-    - ipBlock:
-        cidr: 172.23.0.0/16
-    - ipBlock:
-        cidr: 172.21.0.0/16
-    - ipBlock:
-        cidr: 100.102.0.0/16
-    ports:
-    - protocol: UDP
-      port: 514
-    - protocol: TCP
-      port: 514
-  - from:
-    - podSelector:
-        matchLabels:
-          app: filebeat
-    ports:
-    - protocol: TCP
-      port: 5044
-  - from:
-    - namespaceSelector:
-        matchLabels:
-          kubernetes.io/metadata.name: monitoring
-      podSelector:
-        matchLabels:
-          app: prometheus
-    ports:
-    - port: 9833
-  - from:
-    - namespaceSelector:
-        matchLabels:
-          kubernetes.io/metadata.name: traefik
-      podSelector:
-        matchLabels:
-          app.kubernetes.io/name: traefik
-    ports:
-    - protocol: TCP
-      port: 9000
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: elasticsearch
-spec:
-  podSelector:
-    matchLabels:
-      app: elasticsearch
-  policyTypes:
-  - Ingress
-  - Egress
-  ingress:
-  - from:
-    - podSelector:
-        matchLabels:
-          app: graylog
-  - from:
-    - namespaceSelector:
-        matchLabels:
-          kubernetes.io/metadata.name: monitoring
-      podSelector:
-        matchLabels:
-          app: grafana
-  egress:
-  - to:
-    - ipBlock:
-        # geoip.elastic.co updates
-        cidr: 0.0.0.0/0
-    ports:
-    - port: 443
----
-apiVersion: mongodbcommunity.mongodb.com/v1
-kind: MongoDBCommunity
-metadata:
-  name: mongodb
-spec:
-  members: 3
-  type: ReplicaSet
-  version: "5.0.9"
-  security:
-    authentication:
-      modes: ["SCRAM"]
-  users:
-    - name: readwrite
-      db: application
-      passwordSecretRef:
-        name: mongodb-application-readwrite-password
-      roles:
-        - name: readWrite
-          db: application
-      scramCredentialsSecretName: mongodb-application-readwrite
-    - name: readonly
-      db: application
-      passwordSecretRef:
-        name: mongodb-application-readonly-password
-      roles:
-        - name: readOnly
-          db: application
-      scramCredentialsSecretName: mongodb-application-readonly
-  statefulSet:
-    spec:
-      template:
-        spec:
-          affinity:
-            podAntiAffinity:
-              requiredDuringSchedulingIgnoredDuringExecution:
-                - labelSelector:
-                    matchExpressions:
-                      - key: app
-                        operator: In
-                        values:
-                          - mongodb-svc
-                  topologyKey: kubernetes.io/hostname
-          nodeSelector:
-            dedicated: storage
-          tolerations:
-            - key: dedicated
-              operator: Equal
-              value: storage
-              effect: NoSchedule
-      volumeClaimTemplates:
-        - metadata:
-            name: logs-volume
-          spec:
-            storageClassName: local-path
-            accessModes:
-            - ReadWriteOnce
-            resources:
-              requests:
-                storage: 512Mi
-        - metadata:
-            name: data-volume
-          spec:
-            storageClassName: local-path
-            accessModes:
-            - ReadWriteOnce
-            resources:
-              requests:
-                storage: 2Gi

logging/filebeat.yml

@@ -6,18 +6,13 @@ metadata:
   namespace: logging
 data:
   filebeat.yml: |-
+    setup:
+      ilm:
+        enabled: false
+      template:
+        name: filebeat
+        pattern: filebeat-*
     http.enabled: true
-    filebeat.inputs:
-    - type: container
-      paths:
-        - /var/log/containers/*.log
-      processors:
-        - add_kubernetes_metadata:
-            in_cluster: true
-            host: ${NODE_NAME}
-            matchers:
-            - logs_path:
-                logs_path: "/var/log/containers/"
     filebeat.autodiscover:
       providers:
         - type: kubernetes
@@ -27,50 +22,24 @@ data:
             type: container
             paths:
               - /var/log/containers/*${data.kubernetes.container.id}.log
-    processors:
+    output:
-      - add_host_metadata:
+      elasticsearch:
-      - drop_fields:
+        hosts:
-          fields:
+          - http://zinc:4080
-            - stream
+        path: "/es/"
-          ignore_missing: true
+        index: "filebeat-%{+yyyy.MM.dd}"
-      - rename:
+        username: "${ZINC_FIRST_ADMIN_USER}"
-          fields:
+        password: "${ZINC_FIRST_ADMIN_PASSWORD}"
-            - from: "kubernetes.node.name"
-              to: "source"
-            - from: "kubernetes.pod.name"
-              to: "pod"
-            - from: "stream"
-              to: "stream"
-            - from: "kubernetes.labels.app"
-              to: "app"
-            - from: "kubernetes.namespace"
-              to: "namespace"
-          ignore_missing: true
-      - drop_fields:
-          fields:
-            - agent
-            - container
-            - ecs
-            - host
-            - kubernetes
-            - log
-            - "@metadata"
-          ignore_missing: true
-    output.logstash:
-      hosts: ["graylog-logstash:5044"]
-    #output.console:
-    #  pretty: true
 ---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: filebeat
-  namespace: logging
 spec:
   updateStrategy:
     type: RollingUpdate
     rollingUpdate:
-      maxUnavailable: 100%
+      maxUnavailable: 50%
   selector:
     matchLabels:
       app: filebeat
@@ -78,11 +47,13 @@ spec:
     metadata:
       labels:
         app: filebeat
+      annotations:
+        co.elastic.logs/json.keys_under_root: "true"
     spec:
       serviceAccountName: filebeat
       containers:
         - name: filebeat
-        image: docker.elastic.co/beats/filebeat:7.17.6
+          image: docker.elastic.co/beats/filebeat:8.4.1
           args:
             - -c
             - /etc/filebeat.yml
@@ -94,6 +65,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: ZINC_FIRST_ADMIN_USER
+              value: admin
+            - name: ZINC_FIRST_ADMIN_PASSWORD
+              value: salakala
           ports:
             - containerPort: 5066
           resources:
@@ -115,6 +90,14 @@ spec:
             - name: varlog
               mountPath: /var/log
               readOnly: true
+        - name: exporter
+          image: sepa/beats-exporter
+          args:
+            - -p=5066
+          ports:
+            - containerPort: 8080
+              name: exporter
+              protocol: TCP
       volumes:
         - name: filebeat-config
           configMap:
@@ -168,11 +151,33 @@ spec:
   policyTypes:
     - Ingress
     - Egress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: prometheus-operator
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus
+      ports:
+        - protocol: TCP
+          port: 8080
   egress:
     - to:
         - podSelector:
             matchLabels:
-          app: graylog
+              app: zinc
       ports:
         - protocol: TCP
-      port: 5044
+          port: 4080
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: filebeat
+spec:
+  selector:
+    matchLabels:
+      app: filebeat
+  podMetricsEndpoints:
+    - port: exporter

logging/mongodb-support.yml

@@ -1 +0,0 @@
-../mongodb-operator/mongodb-support.yml

logging/zinc.yml

@@ -0,0 +1,122 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: zinc
+spec:
+  clusterIP: None
+  selector:
+    app: zinc
+  ports:
+  - name: http
+    port: 4080
+    targetPort: 4080
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: zinc
+spec:
+  serviceName: zinc
+  replicas: 1
+  selector:
+    matchLabels:
+      app: zinc
+  template:
+    metadata:
+      labels:
+        app: zinc
+    spec:
+      securityContext:
+        fsGroup: 2000
+        runAsUser: 10000
+        runAsGroup: 3000
+        runAsNonRoot: true
+      containers:
+        - name: zinc
+          image: public.ecr.aws/zinclabs/zinc:latest
+          env:
+            - name: GIN_MODE
+              value: release
+            - name: ZINC_FIRST_ADMIN_USER
+              value: admin
+            - name: ZINC_FIRST_ADMIN_PASSWORD
+              value: salakala
+            - name: ZINC_DATA_PATH
+              value: /data
+          imagePullPolicy: Always
+          resources:
+            limits:
+              cpu: "4"
+              memory: 2048Mi
+            requests:
+              cpu: 32m
+              memory: 50Mi
+          ports:
+            - containerPort: 4080
+              name: http
+          volumeMounts:
+          - name: data
+            mountPath: /data
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes:
+        - ReadWriteOnce
+      storageClassName: longhorn
+      resources:
+        requests:
+          storage: 20Gi
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: zinc
+  annotations:
+    cert-manager.io/cluster-issuer: default
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+    external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
+    traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd
+spec:
+  rules:
+  - host: zinc.k-space.ee
+    http:
+      paths:
+      - pathType: Prefix
+        path: "/"
+        backend:
+          service:
+            name: zinc
+            port:
+              number: 4080
+  tls:
+  - hosts:
+    - zinc.k-space.ee
+    secretName: zinc-tls
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: zinc
+spec:
+  podSelector:
+    matchLabels:
+      app: zinc
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          app: filebeat
+    ports:
+    - protocol: TCP
+      port: 4080
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: traefik
+      podSelector:
+        matchLabels:
+          app.kubernetes.io/name: traefik