From 8e50f5b2fbbbfc39bc1c528ed61b19f43d4d9109 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lauri=20V=C3=B5sandi?= <lauri@k-space.ee>
Date: Mon, 29 Aug 2022 21:23:47 +0300
Subject: [PATCH] logging: Replace Fluent Bit with Filebeat

---
 logging/README.md       |   7 +-
 logging/application.yml | 221 +++-------------------------------------
 logging/filebeat.yml    | 196 +++++++++++++++++++++++++++++++++++
 3 files changed, 216 insertions(+), 208 deletions(-)
 create mode 100644 logging/filebeat.yml

diff --git a/logging/README.md b/logging/README.md
index cd09edd..2c3efb0 100644
--- a/logging/README.md
+++ b/logging/README.md
@@ -14,8 +14,8 @@ To deploy:
 
 ```
 kubectl create namespace logging
-kubectl apply -n logging -f mongodb-support.yml -f application.yml -f networkpolicy-base.yml
-kubectl rollout restart -n logging daemonset/fluent-bit
+kubectl apply -n logging -f mongodb-support.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml
+kubectl rollout restart -n logging daemonset.apps/filebeat
 ```
 
 To set secrets:
@@ -44,7 +44,8 @@ After Graylog deployment following steps were manually performed via web interfa
 
 * Add Syslog TCP input for external Linux hosts
 * Add Syslog UDP input for Mikrotik networking gear
-* Add GELF TCP input for Kubernetes workers
+* Add Beats input for Kubernetes workers,
+  enable `Do not add Beats type as prefix`
 * Trusted header authentication was enabled and set to `Remote-User`
   https://graylog.k-space.ee/system/authentication/authenticator/edit
   Note that user accounts are not provisioned automatically.
diff --git a/logging/application.yml b/logging/application.yml
index 0bd9b9c..7adca52 100644
--- a/logging/application.yml
+++ b/logging/application.yml
@@ -1,207 +1,4 @@
 ---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: fluent-bit
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: fluent-bit-read
-rules:
-- apiGroups: [""]
-  resources:
-  - namespaces
-  - pods
-  verbs: ["get", "list", "watch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: fluent-bit-read
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: fluent-bit-read
-subjects:
-- kind: ServiceAccount
-  name: fluent-bit
-  namespace: logging
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: fluent-bit-config
-  namespace: logging
-  labels:
-    app: fluent-bit
-  annotations:
-    reloader.stakater.com/match: "true"
-data:
-  fluent-bit.conf: |
-    [SERVICE]
-        Flush 1
-        Log_Level warn
-        Daemon off
-        Parsers_File parsers.conf
-        HTTP_Server On
-        HTTP_Listen 0.0.0.0
-        HTTP_Port 2020
-    @INCLUDE input-kubernetes.conf
-    @INCLUDE filter-kubernetes.conf
-    @INCLUDE output-graylog.conf
-  input-kubernetes.conf: |
-    # Following assembles the log fragments of the Kubernetes runtime
-    # https://github.com/fluent/fluent-bit/blob/d3c71f2ed4ff3625b85715aaefe6bc76b2ac3c2e/src/multiline/flb_ml_parser_docker.c#L57
-    [INPUT]
-        name tail
-        tag kube.*
-        path /var/log/containers/*.log
-        multiline.parser cri
-        db /var/log/flb_kube.db
-        mem_buf_limit 5MB
-        skip_long_lines on
-        refresh_interval 10
-  filter-kubernetes.conf: |
-    # Following reassembles stack traces
-    [FILTER]
-        name multiline
-        match *
-        multiline.key_content log
-        multiline.parser go,python,java
-    # Following annotates the Kubernetes logs using Kubernetes API-s
-    [FILTER]
-        Name kubernetes
-        Match kube.*
-        Kube_URL https://kubernetes.default.svc:443
-        Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
-        Kube_Tag_Prefix kube.var.log.containers.
-        Merge_Log On
-        K8S-Logging.Parser On
-        K8S-Logging.Exclude Off
-    # Following unnests the kubernetes map
-    [FILTER]
-        Name nest
-        Match kube.*
-        Operation lift
-        Nested_under kubernetes
-        Add_prefix kubernetes_
-  output-graylog.conf: |
-    [OUTPUT]
-        Name gelf
-        Match *
-        Host graylog-gelf-tcp
-        Port 12201
-        Mode tcp
-        Gelf_Host_Key kubernetes_host
-        Gelf_Short_Message_Key log
-        Retry_Limit no_limits
-  parsers.conf: |
-    # http://rubular.com/r/tjUt3Awgg4
-    [PARSER]
-        Name cri
-        Format regex
-        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
-        Time_Key time
-        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: fluent-bit
-  namespace: logging
-  annotations:
-    keel.sh/policy: patch
-    keel.sh/trigger: poll
-    keel.sh/pollSchedule: "@midnight"
-  annotations:
-    reloader.stakater.com/search: "true"
-spec:
-  revisionHistoryLimit: 0
-  selector:
-    matchLabels:
-      app: fluent-bit
-  template:
-    metadata:
-      labels:
-        app: fluent-bit
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "2020"
-        prometheus.io/path: /api/v1/metrics/prometheus
-    spec:
-      containers:
-      - name: fluent-bit
-        image: fluent/fluent-bit:1.9
-        imagePullPolicy: Always
-        ports:
-          - containerPort: 2020
-        volumeMounts:
-        - name: varlog
-          mountPath: /var/log
-        - name: varlibdockercontainers
-          mountPath: /var/lib/docker/containers
-          readOnly: true
-        - name: fluent-bit-config
-          mountPath: /fluent-bit/etc/
-      volumes:
-      - name: varlog
-        hostPath:
-          path: /var/log
-      - name: varlibdockercontainers
-        hostPath:
-          path: /var/lib/docker/containers
-      - name: fluent-bit-config
-        configMap:
-          name: fluent-bit-config
-      serviceAccountName: fluent-bit
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      - operator: "Exists"
-        effect: "NoExecute"
-      - operator: "Exists"
-        effect: "NoSchedule"
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: fluent-bit
-spec:
-  podSelector:
-    matchLabels:
-      app: fluent-bit
-  policyTypes:
-  - Ingress
-  - Egress
-  ingress:
-  - from:
-    - namespaceSelector:
-        matchLabels:
-          kubernetes.io/metadata.name: monitoring
-      podSelector:
-        matchLabels:
-          app: prometheus
-    ports:
-    - port: 2020
-  egress:
-  - to:
-    - podSelector:
-        matchLabels:
-          app: graylog
-    ports:
-    - protocol: TCP
-      port: 12201
-  - # Kubernetes API endpoint kubernetes.default.svc.cluster.local
-    # Determine IP-s and ports with: kubectl get ep -n default kubernetes
-    to:
-    - ipBlock:
-        cidr: 172.21.3.0/24
-    ports:
-    - port: 6443
----
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -301,6 +98,20 @@ spec:
 ---
 apiVersion: v1
 kind: Service
+metadata:
+  name: graylog-logstash
+  labels:
+    app: graylog
+spec:
+  ports:
+    - name: graylog-logstash
+      port: 5044
+      protocol: TCP
+  selector:
+    app: graylog
+---
+apiVersion: v1
+kind: Service
 metadata:
   name: graylog-syslog-tcp
   labels:
@@ -516,10 +327,10 @@ spec:
   - from:
     - podSelector:
         matchLabels:
-          app: fluent-bit
+          app: filebeat
     ports:
     - protocol: TCP
-      port: 12201
+      port: 5044
   - from:
     - namespaceSelector:
         matchLabels:
diff --git a/logging/filebeat.yml b/logging/filebeat.yml
new file mode 100644
index 0000000..eece903
--- /dev/null
+++ b/logging/filebeat.yml
@@ -0,0 +1,196 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: logging
+data:
+  filebeat.yml: |-
+    http.enabled: true
+    filebeat.inputs:
+    - type: container
+      paths:
+        - /var/log/containers/*.log
+      processors:
+        - add_kubernetes_metadata:
+            in_cluster: true
+            host: ${NODE_NAME}
+            matchers:
+            - logs_path:
+                logs_path: "/var/log/containers/"
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          host: ${NODE_NAME}
+          hints.enabled: true
+          hints.default_config:
+            type: container
+            paths:
+              - /var/log/containers/*${data.kubernetes.container.id}.log
+    processors:
+      - add_host_metadata:
+      - drop_fields:
+          fields:
+            - stream
+          ignore_missing: true
+      - rename:
+          fields:
+            - from: "kubernetes.node.name"
+              to: "source"
+            - from: "kubernetes.pod.name"
+              to: "pod"
+            - from: "stream"
+              to: "stream"
+            - from: "kubernetes.labels.app"
+              to: "app"
+            - from: "kubernetes.namespace"
+              to: "namespace"
+          ignore_missing: true
+      - drop_fields:
+          fields:
+            - agent
+            - container
+            - ecs
+            - host
+            - kubernetes
+            - log
+            - "@metadata"
+          ignore_missing: true
+    output.logstash:
+      hosts: ["graylog-logstash:5044"]
+    #output.console:
+    #  pretty: true
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: filebeat
+  namespace: logging
+spec:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 100%
+  selector:
+    matchLabels:
+      app: filebeat
+  template:
+    metadata:
+      labels:
+        app: filebeat
+    spec:
+      serviceAccountName: filebeat
+      containers:
+      - name: filebeat
+        image: docker.elastic.co/beats/filebeat:7.17.6
+        args:
+         - -c
+         - /etc/filebeat.yml
+         - -e
+        securityContext:
+          runAsUser: 0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+          - containerPort: 5066
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: filebeat-config
+          mountPath: /etc/filebeat.yml
+          readOnly: true
+          subPath: filebeat.yml
+        - name: data
+          mountPath: /usr/share/filebeat/data
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: varlog
+          mountPath: /var/log
+          readOnly: true
+      volumes:
+      - name: filebeat-config
+        configMap:
+          defaultMode: 0600
+          name: filebeat-config
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: data
+        hostPath:
+          path: /var/lib/filebeat-data
+          type: DirectoryOrCreate
+      tolerations:
+      - operator: "Exists"
+        effect: "NoExecute"
+      - operator: "Exists"
+        effect: "NoSchedule"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: filebeat
+subjects:
+- kind: ServiceAccount
+  name: filebeat
+  namespace: logging
+roleRef:
+  kind: ClusterRole
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - nodes
+  verbs:
+  - get
+  - watch
+  - list
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: logging
+  labels:
+    app: filebeat
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: filebeat
+spec:
+  podSelector:
+    matchLabels:
+      app: filebeat
+  policyTypes:
+  - Ingress
+  - Egress
+  egress:
+  - to:
+    - podSelector:
+        matchLabels:
+          app: graylog
+    ports:
+    - protocol: TCP
+      port: 5044