logging: Replace Fluent Bit with Filebeat
This commit is contained in:
parent
1cfc82c049
commit
8e50f5b2fb
@ -14,8 +14,8 @@ To deploy:
|
|||||||
|
|
||||||
```
|
```
|
||||||
kubectl create namespace logging
|
kubectl create namespace logging
|
||||||
kubectl apply -n logging -f mongodb-support.yml -f application.yml -f networkpolicy-base.yml
|
kubectl apply -n logging -f mongodb-support.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml
|
||||||
kubectl rollout restart -n logging daemonset/fluent-bit
|
kubectl rollout restart -n logging daemonset.apps/filebeat
|
||||||
```
|
```
|
||||||
|
|
||||||
To set secrets:
|
To set secrets:
|
||||||
@ -44,7 +44,8 @@ After Graylog deployment following steps were manually performed via web interfa
|
|||||||
|
|
||||||
* Add Syslog TCP input for external Linux hosts
|
* Add Syslog TCP input for external Linux hosts
|
||||||
* Add Syslog UDP input for Mikrotik networking gear
|
* Add Syslog UDP input for Mikrotik networking gear
|
||||||
* Add GELF TCP input for Kubernetes workers
|
* Add Beats input for Kubernetes workers,
|
||||||
|
enable `Do not add Beats type as prefix`
|
||||||
* Trusted header authentication was enabled and set to `Remote-User`
|
* Trusted header authentication was enabled and set to `Remote-User`
|
||||||
https://graylog.k-space.ee/system/authentication/authenticator/edit
|
https://graylog.k-space.ee/system/authentication/authenticator/edit
|
||||||
Note that user accounts are not provisioned automatically.
|
Note that user accounts are not provisioned automatically.
|
||||||
|
@ -1,207 +1,4 @@
|
|||||||
---
|
---
|
||||||
apiVersion: v1
|
|
||||||
kind: ServiceAccount
|
|
||||||
metadata:
|
|
||||||
name: fluent-bit
|
|
||||||
---
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
kind: ClusterRole
|
|
||||||
metadata:
|
|
||||||
name: fluent-bit-read
|
|
||||||
rules:
|
|
||||||
- apiGroups: [""]
|
|
||||||
resources:
|
|
||||||
- namespaces
|
|
||||||
- pods
|
|
||||||
verbs: ["get", "list", "watch"]
|
|
||||||
---
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
kind: ClusterRoleBinding
|
|
||||||
metadata:
|
|
||||||
name: fluent-bit-read
|
|
||||||
roleRef:
|
|
||||||
apiGroup: rbac.authorization.k8s.io
|
|
||||||
kind: ClusterRole
|
|
||||||
name: fluent-bit-read
|
|
||||||
subjects:
|
|
||||||
- kind: ServiceAccount
|
|
||||||
name: fluent-bit
|
|
||||||
namespace: logging
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: fluent-bit-config
|
|
||||||
namespace: logging
|
|
||||||
labels:
|
|
||||||
app: fluent-bit
|
|
||||||
annotations:
|
|
||||||
reloader.stakater.com/match: "true"
|
|
||||||
data:
|
|
||||||
fluent-bit.conf: |
|
|
||||||
[SERVICE]
|
|
||||||
Flush 1
|
|
||||||
Log_Level warn
|
|
||||||
Daemon off
|
|
||||||
Parsers_File parsers.conf
|
|
||||||
HTTP_Server On
|
|
||||||
HTTP_Listen 0.0.0.0
|
|
||||||
HTTP_Port 2020
|
|
||||||
@INCLUDE input-kubernetes.conf
|
|
||||||
@INCLUDE filter-kubernetes.conf
|
|
||||||
@INCLUDE output-graylog.conf
|
|
||||||
input-kubernetes.conf: |
|
|
||||||
# Following assembles the log fragments of the Kubernetes runtime
|
|
||||||
# https://github.com/fluent/fluent-bit/blob/d3c71f2ed4ff3625b85715aaefe6bc76b2ac3c2e/src/multiline/flb_ml_parser_docker.c#L57
|
|
||||||
[INPUT]
|
|
||||||
name tail
|
|
||||||
tag kube.*
|
|
||||||
path /var/log/containers/*.log
|
|
||||||
multiline.parser cri
|
|
||||||
db /var/log/flb_kube.db
|
|
||||||
mem_buf_limit 5MB
|
|
||||||
skip_long_lines on
|
|
||||||
refresh_interval 10
|
|
||||||
filter-kubernetes.conf: |
|
|
||||||
# Following reassembles stack traces
|
|
||||||
[FILTER]
|
|
||||||
name multiline
|
|
||||||
match *
|
|
||||||
multiline.key_content log
|
|
||||||
multiline.parser go,python,java
|
|
||||||
# Following annotates the Kubernetes logs using Kubernetes API-s
|
|
||||||
[FILTER]
|
|
||||||
Name kubernetes
|
|
||||||
Match kube.*
|
|
||||||
Kube_URL https://kubernetes.default.svc:443
|
|
||||||
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
||||||
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
||||||
Kube_Tag_Prefix kube.var.log.containers.
|
|
||||||
Merge_Log On
|
|
||||||
K8S-Logging.Parser On
|
|
||||||
K8S-Logging.Exclude Off
|
|
||||||
# Following unnests the kubernetes map
|
|
||||||
[FILTER]
|
|
||||||
Name nest
|
|
||||||
Match kube.*
|
|
||||||
Operation lift
|
|
||||||
Nested_under kubernetes
|
|
||||||
Add_prefix kubernetes_
|
|
||||||
output-graylog.conf: |
|
|
||||||
[OUTPUT]
|
|
||||||
Name gelf
|
|
||||||
Match *
|
|
||||||
Host graylog-gelf-tcp
|
|
||||||
Port 12201
|
|
||||||
Mode tcp
|
|
||||||
Gelf_Host_Key kubernetes_host
|
|
||||||
Gelf_Short_Message_Key log
|
|
||||||
Retry_Limit no_limits
|
|
||||||
parsers.conf: |
|
|
||||||
# http://rubular.com/r/tjUt3Awgg4
|
|
||||||
[PARSER]
|
|
||||||
Name cri
|
|
||||||
Format regex
|
|
||||||
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
|
|
||||||
Time_Key time
|
|
||||||
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: DaemonSet
|
|
||||||
metadata:
|
|
||||||
name: fluent-bit
|
|
||||||
namespace: logging
|
|
||||||
annotations:
|
|
||||||
keel.sh/policy: patch
|
|
||||||
keel.sh/trigger: poll
|
|
||||||
keel.sh/pollSchedule: "@midnight"
|
|
||||||
annotations:
|
|
||||||
reloader.stakater.com/search: "true"
|
|
||||||
spec:
|
|
||||||
revisionHistoryLimit: 0
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app: fluent-bit
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: fluent-bit
|
|
||||||
annotations:
|
|
||||||
prometheus.io/scrape: "true"
|
|
||||||
prometheus.io/port: "2020"
|
|
||||||
prometheus.io/path: /api/v1/metrics/prometheus
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: fluent-bit
|
|
||||||
image: fluent/fluent-bit:1.9
|
|
||||||
imagePullPolicy: Always
|
|
||||||
ports:
|
|
||||||
- containerPort: 2020
|
|
||||||
volumeMounts:
|
|
||||||
- name: varlog
|
|
||||||
mountPath: /var/log
|
|
||||||
- name: varlibdockercontainers
|
|
||||||
mountPath: /var/lib/docker/containers
|
|
||||||
readOnly: true
|
|
||||||
- name: fluent-bit-config
|
|
||||||
mountPath: /fluent-bit/etc/
|
|
||||||
volumes:
|
|
||||||
- name: varlog
|
|
||||||
hostPath:
|
|
||||||
path: /var/log
|
|
||||||
- name: varlibdockercontainers
|
|
||||||
hostPath:
|
|
||||||
path: /var/lib/docker/containers
|
|
||||||
- name: fluent-bit-config
|
|
||||||
configMap:
|
|
||||||
name: fluent-bit-config
|
|
||||||
serviceAccountName: fluent-bit
|
|
||||||
tolerations:
|
|
||||||
- key: node-role.kubernetes.io/master
|
|
||||||
operator: Exists
|
|
||||||
effect: NoSchedule
|
|
||||||
- operator: "Exists"
|
|
||||||
effect: "NoExecute"
|
|
||||||
- operator: "Exists"
|
|
||||||
effect: "NoSchedule"
|
|
||||||
---
|
|
||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: NetworkPolicy
|
|
||||||
metadata:
|
|
||||||
name: fluent-bit
|
|
||||||
spec:
|
|
||||||
podSelector:
|
|
||||||
matchLabels:
|
|
||||||
app: fluent-bit
|
|
||||||
policyTypes:
|
|
||||||
- Ingress
|
|
||||||
- Egress
|
|
||||||
ingress:
|
|
||||||
- from:
|
|
||||||
- namespaceSelector:
|
|
||||||
matchLabels:
|
|
||||||
kubernetes.io/metadata.name: monitoring
|
|
||||||
podSelector:
|
|
||||||
matchLabels:
|
|
||||||
app: prometheus
|
|
||||||
ports:
|
|
||||||
- port: 2020
|
|
||||||
egress:
|
|
||||||
- to:
|
|
||||||
- podSelector:
|
|
||||||
matchLabels:
|
|
||||||
app: graylog
|
|
||||||
ports:
|
|
||||||
- protocol: TCP
|
|
||||||
port: 12201
|
|
||||||
- # Kubernetes API endpoint kubernetes.default.svc.cluster.local
|
|
||||||
# Determine IP-s and ports with: kubectl get ep -n default kubernetes
|
|
||||||
to:
|
|
||||||
- ipBlock:
|
|
||||||
cidr: 172.21.3.0/24
|
|
||||||
ports:
|
|
||||||
- port: 6443
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: StatefulSet
|
kind: StatefulSet
|
||||||
metadata:
|
metadata:
|
||||||
@ -301,6 +98,20 @@ spec:
|
|||||||
---
|
---
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Service
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: graylog-logstash
|
||||||
|
labels:
|
||||||
|
app: graylog
|
||||||
|
spec:
|
||||||
|
ports:
|
||||||
|
- name: graylog-logstash
|
||||||
|
port: 5044
|
||||||
|
protocol: TCP
|
||||||
|
selector:
|
||||||
|
app: graylog
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
metadata:
|
metadata:
|
||||||
name: graylog-syslog-tcp
|
name: graylog-syslog-tcp
|
||||||
labels:
|
labels:
|
||||||
@ -516,10 +327,10 @@ spec:
|
|||||||
- from:
|
- from:
|
||||||
- podSelector:
|
- podSelector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: fluent-bit
|
app: filebeat
|
||||||
ports:
|
ports:
|
||||||
- protocol: TCP
|
- protocol: TCP
|
||||||
port: 12201
|
port: 5044
|
||||||
- from:
|
- from:
|
||||||
- namespaceSelector:
|
- namespaceSelector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
|
196
logging/filebeat.yml
Normal file
196
logging/filebeat.yml
Normal file
@ -0,0 +1,196 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: filebeat-config
|
||||||
|
namespace: logging
|
||||||
|
data:
|
||||||
|
filebeat.yml: |-
|
||||||
|
http.enabled: true
|
||||||
|
filebeat.inputs:
|
||||||
|
- type: container
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*.log
|
||||||
|
processors:
|
||||||
|
- add_kubernetes_metadata:
|
||||||
|
in_cluster: true
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
matchers:
|
||||||
|
- logs_path:
|
||||||
|
logs_path: "/var/log/containers/"
|
||||||
|
filebeat.autodiscover:
|
||||||
|
providers:
|
||||||
|
- type: kubernetes
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
hints.enabled: true
|
||||||
|
hints.default_config:
|
||||||
|
type: container
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- add_host_metadata:
|
||||||
|
- drop_fields:
|
||||||
|
fields:
|
||||||
|
- stream
|
||||||
|
ignore_missing: true
|
||||||
|
- rename:
|
||||||
|
fields:
|
||||||
|
- from: "kubernetes.node.name"
|
||||||
|
to: "source"
|
||||||
|
- from: "kubernetes.pod.name"
|
||||||
|
to: "pod"
|
||||||
|
- from: "stream"
|
||||||
|
to: "stream"
|
||||||
|
- from: "kubernetes.labels.app"
|
||||||
|
to: "app"
|
||||||
|
- from: "kubernetes.namespace"
|
||||||
|
to: "namespace"
|
||||||
|
ignore_missing: true
|
||||||
|
- drop_fields:
|
||||||
|
fields:
|
||||||
|
- agent
|
||||||
|
- container
|
||||||
|
- ecs
|
||||||
|
- host
|
||||||
|
- kubernetes
|
||||||
|
- log
|
||||||
|
- "@metadata"
|
||||||
|
ignore_missing: true
|
||||||
|
output.logstash:
|
||||||
|
hosts: ["graylog-logstash:5044"]
|
||||||
|
#output.console:
|
||||||
|
# pretty: true
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: filebeat
|
||||||
|
namespace: logging
|
||||||
|
spec:
|
||||||
|
updateStrategy:
|
||||||
|
type: RollingUpdate
|
||||||
|
rollingUpdate:
|
||||||
|
maxUnavailable: 100%
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: filebeat
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: filebeat
|
||||||
|
spec:
|
||||||
|
serviceAccountName: filebeat
|
||||||
|
containers:
|
||||||
|
- name: filebeat
|
||||||
|
image: docker.elastic.co/beats/filebeat:7.17.6
|
||||||
|
args:
|
||||||
|
- -c
|
||||||
|
- /etc/filebeat.yml
|
||||||
|
- -e
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 0
|
||||||
|
env:
|
||||||
|
- name: NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
ports:
|
||||||
|
- containerPort: 5066
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: 200Mi
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 100Mi
|
||||||
|
volumeMounts:
|
||||||
|
- name: filebeat-config
|
||||||
|
mountPath: /etc/filebeat.yml
|
||||||
|
readOnly: true
|
||||||
|
subPath: filebeat.yml
|
||||||
|
- name: data
|
||||||
|
mountPath: /usr/share/filebeat/data
|
||||||
|
- name: varlibdockercontainers
|
||||||
|
mountPath: /var/lib/docker/containers
|
||||||
|
readOnly: true
|
||||||
|
- name: varlog
|
||||||
|
mountPath: /var/log
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: filebeat-config
|
||||||
|
configMap:
|
||||||
|
defaultMode: 0600
|
||||||
|
name: filebeat-config
|
||||||
|
- name: varlibdockercontainers
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/docker/containers
|
||||||
|
- name: varlog
|
||||||
|
hostPath:
|
||||||
|
path: /var/log
|
||||||
|
- name: data
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/filebeat-data
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
tolerations:
|
||||||
|
- operator: "Exists"
|
||||||
|
effect: "NoExecute"
|
||||||
|
- operator: "Exists"
|
||||||
|
effect: "NoSchedule"
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: filebeat
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: filebeat
|
||||||
|
namespace: logging
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: filebeat
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: filebeat
|
||||||
|
labels:
|
||||||
|
app: filebeat
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- namespaces
|
||||||
|
- pods
|
||||||
|
- nodes
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- watch
|
||||||
|
- list
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: filebeat
|
||||||
|
namespace: logging
|
||||||
|
labels:
|
||||||
|
app: filebeat
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: filebeat
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: filebeat
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
- Egress
|
||||||
|
egress:
|
||||||
|
- to:
|
||||||
|
- podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: graylog
|
||||||
|
ports:
|
||||||
|
- protocol: TCP
|
||||||
|
port: 5044
|
Loading…
Reference in New Issue
Block a user