logging: Replace Fluent Bit with Filebeat

This commit is contained in:
Lauri Võsandi 2022-08-29 21:23:47 +03:00
parent 1cfc82c049
commit 8e50f5b2fb
3 changed files with 216 additions and 208 deletions

View File

@ -14,8 +14,8 @@ To deploy:
``` ```
kubectl create namespace logging kubectl create namespace logging
kubectl apply -n logging -f mongodb-support.yml -f application.yml -f networkpolicy-base.yml kubectl apply -n logging -f mongodb-support.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml
kubectl rollout restart -n logging daemonset/fluent-bit kubectl rollout restart -n logging daemonset.apps/filebeat
``` ```
To set secrets: To set secrets:
@ -44,7 +44,8 @@ After Graylog deployment following steps were manually performed via web interfa
* Add Syslog TCP input for external Linux hosts * Add Syslog TCP input for external Linux hosts
* Add Syslog UDP input for Mikrotik networking gear * Add Syslog UDP input for Mikrotik networking gear
* Add GELF TCP input for Kubernetes workers * Add Beats input for Kubernetes workers,
enable `Do not add Beats type as prefix`
* Trusted header authentication was enabled and set to `Remote-User` * Trusted header authentication was enabled and set to `Remote-User`
https://graylog.k-space.ee/system/authentication/authenticator/edit https://graylog.k-space.ee/system/authentication/authenticator/edit
Note that user accounts are not provisioned automatically. Note that user accounts are not provisioned automatically.

View File

@ -1,207 +1,4 @@
--- ---
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluent-bit
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fluent-bit-read
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: fluent-bit-read
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: fluent-bit-read
subjects:
- kind: ServiceAccount
name: fluent-bit
namespace: logging
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-config
namespace: logging
labels:
app: fluent-bit
annotations:
reloader.stakater.com/match: "true"
data:
fluent-bit.conf: |
[SERVICE]
Flush 1
Log_Level warn
Daemon off
Parsers_File parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
@INCLUDE input-kubernetes.conf
@INCLUDE filter-kubernetes.conf
@INCLUDE output-graylog.conf
input-kubernetes.conf: |
# Following assembles the log fragments of the Kubernetes runtime
# https://github.com/fluent/fluent-bit/blob/d3c71f2ed4ff3625b85715aaefe6bc76b2ac3c2e/src/multiline/flb_ml_parser_docker.c#L57
[INPUT]
name tail
tag kube.*
path /var/log/containers/*.log
multiline.parser cri
db /var/log/flb_kube.db
mem_buf_limit 5MB
skip_long_lines on
refresh_interval 10
filter-kubernetes.conf: |
# Following reassembles stack traces
[FILTER]
name multiline
match *
multiline.key_content log
multiline.parser go,python,java
# Following annotates the Kubernetes logs using Kubernetes API-s
[FILTER]
Name kubernetes
Match kube.*
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Kube_Tag_Prefix kube.var.log.containers.
Merge_Log On
K8S-Logging.Parser On
K8S-Logging.Exclude Off
# Following unnests the kubernetes map
[FILTER]
Name nest
Match kube.*
Operation lift
Nested_under kubernetes
Add_prefix kubernetes_
output-graylog.conf: |
[OUTPUT]
Name gelf
Match *
Host graylog-gelf-tcp
Port 12201
Mode tcp
Gelf_Host_Key kubernetes_host
Gelf_Short_Message_Key log
Retry_Limit no_limits
parsers.conf: |
# http://rubular.com/r/tjUt3Awgg4
[PARSER]
Name cri
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fluent-bit
namespace: logging
annotations:
keel.sh/policy: patch
keel.sh/trigger: poll
keel.sh/pollSchedule: "@midnight"
annotations:
reloader.stakater.com/search: "true"
spec:
revisionHistoryLimit: 0
selector:
matchLabels:
app: fluent-bit
template:
metadata:
labels:
app: fluent-bit
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "2020"
prometheus.io/path: /api/v1/metrics/prometheus
spec:
containers:
- name: fluent-bit
image: fluent/fluent-bit:1.9
imagePullPolicy: Always
ports:
- containerPort: 2020
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: fluent-bit-config
mountPath: /fluent-bit/etc/
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: fluent-bit-config
configMap:
name: fluent-bit-config
serviceAccountName: fluent-bit
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- operator: "Exists"
effect: "NoExecute"
- operator: "Exists"
effect: "NoSchedule"
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: fluent-bit
spec:
podSelector:
matchLabels:
app: fluent-bit
policyTypes:
- Ingress
- Egress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: monitoring
podSelector:
matchLabels:
app: prometheus
ports:
- port: 2020
egress:
- to:
- podSelector:
matchLabels:
app: graylog
ports:
- protocol: TCP
port: 12201
- # Kubernetes API endpoint kubernetes.default.svc.cluster.local
# Determine IP-s and ports with: kubectl get ep -n default kubernetes
to:
- ipBlock:
cidr: 172.21.3.0/24
ports:
- port: 6443
---
apiVersion: apps/v1 apiVersion: apps/v1
kind: StatefulSet kind: StatefulSet
metadata: metadata:
@ -301,6 +98,20 @@ spec:
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service
metadata:
name: graylog-logstash
labels:
app: graylog
spec:
ports:
- name: graylog-logstash
port: 5044
protocol: TCP
selector:
app: graylog
---
apiVersion: v1
kind: Service
metadata: metadata:
name: graylog-syslog-tcp name: graylog-syslog-tcp
labels: labels:
@ -516,10 +327,10 @@ spec:
- from: - from:
- podSelector: - podSelector:
matchLabels: matchLabels:
app: fluent-bit app: filebeat
ports: ports:
- protocol: TCP - protocol: TCP
port: 12201 port: 5044
- from: - from:
- namespaceSelector: - namespaceSelector:
matchLabels: matchLabels:

196
logging/filebeat.yml Normal file
View File

@ -0,0 +1,196 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: logging
data:
filebeat.yml: |-
http.enabled: true
filebeat.inputs:
- type: container
paths:
- /var/log/containers/*.log
processors:
- add_kubernetes_metadata:
in_cluster: true
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
filebeat.autodiscover:
providers:
- type: kubernetes
host: ${NODE_NAME}
hints.enabled: true
hints.default_config:
type: container
paths:
- /var/log/containers/*${data.kubernetes.container.id}.log
processors:
- add_host_metadata:
- drop_fields:
fields:
- stream
ignore_missing: true
- rename:
fields:
- from: "kubernetes.node.name"
to: "source"
- from: "kubernetes.pod.name"
to: "pod"
- from: "stream"
to: "stream"
- from: "kubernetes.labels.app"
to: "app"
- from: "kubernetes.namespace"
to: "namespace"
ignore_missing: true
- drop_fields:
fields:
- agent
- container
- ecs
- host
- kubernetes
- log
- "@metadata"
ignore_missing: true
output.logstash:
hosts: ["graylog-logstash:5044"]
#output.console:
# pretty: true
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: logging
spec:
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 100%
selector:
matchLabels:
app: filebeat
template:
metadata:
labels:
app: filebeat
spec:
serviceAccountName: filebeat
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.17.6
args:
- -c
- /etc/filebeat.yml
- -e
securityContext:
runAsUser: 0
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
ports:
- containerPort: 5066
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: filebeat-config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: filebeat-config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
tolerations:
- operator: "Exists"
effect: "NoExecute"
- operator: "Exists"
effect: "NoSchedule"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: logging
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
app: filebeat
rules:
- apiGroups:
- ""
resources:
- namespaces
- pods
- nodes
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: logging
labels:
app: filebeat
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: filebeat
spec:
podSelector:
matchLabels:
app: filebeat
policyTypes:
- Ingress
- Egress
egress:
- to:
- podSelector:
matchLabels:
app: graylog
ports:
- protocol: TCP
port: 5044