k-space
/
kube
9
1
Fork 1
Code Issues 17 Pull Requests Projects Activity
kube/logging/application.yml

635 lines
15 KiB
YAML
Raw Blame History

---
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluent-bit
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fluent-bit-read
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: fluent-bit-read
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: fluent-bit-read
subjects:
- kind: ServiceAccount
name: fluent-bit
namespace: logging
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-config
namespace: logging
labels:
app: fluent-bit
annotations:
reloader.stakater.com/match: "true"
data:
fluent-bit.conf: |
[SERVICE]
Flush 1
Log_Level warn
Daemon off
Parsers_File parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
@INCLUDE input-kubernetes.conf
@INCLUDE filter-kubernetes.conf
@INCLUDE output-graylog.conf
input-kubernetes.conf: |
# Following assembles the log fragments of the Kubernetes runtime
# https://github.com/fluent/fluent-bit/blob/d3c71f2ed4ff3625b85715aaefe6bc76b2ac3c2e/src/multiline/flb_ml_parser_docker.c#L57
[INPUT]
name tail
tag kube.*
path /var/log/containers/*.log
multiline.parser cri
db /var/log/flb_kube.db
mem_buf_limit 5MB
skip_long_lines on
refresh_interval 10
filter-kubernetes.conf: |
# Following reassembles stack traces
[FILTER]
name multiline
match *
multiline.key_content log
multiline.parser go,python,java
# Following annotates the Kubernetes logs using Kubernetes API-s
[FILTER]
Name kubernetes
Match kube.*
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Kube_Tag_Prefix kube.var.log.containers.
Merge_Log On
K8S-Logging.Parser On
K8S-Logging.Exclude Off
# Following unnests the kubernetes map
[FILTER]
Name nest
Match kube.*
Operation lift
Nested_under kubernetes
Add_prefix kubernetes_
output-graylog.conf: |
[OUTPUT]
Name gelf
Match *
Host graylog-gelf-tcp
Port 12201
Mode tcp
Gelf_Host_Key kubernetes_host
Gelf_Short_Message_Key log
Retry_Limit no_limits
parsers.conf: |
# http://rubular.com/r/tjUt3Awgg4
[PARSER]
Name cri
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fluent-bit
namespace: logging
annotations:
keel.sh/policy: patch
keel.sh/trigger: poll
keel.sh/pollSchedule: "@midnight"
annotations:
reloader.stakater.com/search: "true"
spec:
revisionHistoryLimit: 0
selector:
matchLabels:
app: fluent-bit
template:
metadata:
labels:
app: fluent-bit
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "2020"
prometheus.io/path: /api/v1/metrics/prometheus
spec:
containers:
- name: fluent-bit
image: fluent/fluent-bit:1.9
imagePullPolicy: Always
ports:
- containerPort: 2020
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: fluent-bit-config
mountPath: /fluent-bit/etc/
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: fluent-bit-config
configMap:
name: fluent-bit-config
serviceAccountName: fluent-bit
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- operator: "Exists"
effect: "NoExecute"
- operator: "Exists"
effect: "NoSchedule"
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: fluent-bit
spec:
podSelector:
matchLabels:
app: fluent-bit
policyTypes:
- Ingress
- Egress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: monitoring
podSelector:
matchLabels:
app: prometheus
ports:
- port: 2020
egress:
- to:
- podSelector:
matchLabels:
app: graylog
ports:
- protocol: TCP
port: 12201
- # Kubernetes API endpoint kubernetes.default.svc.cluster.local
# Determine IP-s and ports with: kubectl get ep -n default kubernetes
to:
- ipBlock:
cidr: 172.21.3.0/24
ports:
- port: 6443
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: elasticsearch
labels:
app: elasticsearch
spec:
serviceName: elasticsearch
revisionHistoryLimit: 0
replicas: 1
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
securityContext:
fsGroup: 1000
containers:
- name: elasticsearch
image: elasticsearch:7.17.3
securityContext:
runAsNonRoot: true
runAsUser: 1000
env:
- name: discovery.type
value: single-node
- name: xpack.security.enabled
value: "false"
ports:
- containerPort: 9200
readinessProbe:
httpGet:
path: /_cluster/health
port: 9200
initialDelaySeconds: 5
periodSeconds: 10
failureThreshold: 3
successThreshold: 1
timeoutSeconds: 5
resources:
limits:
memory: "2147483648"
volumeMounts:
- name: elasticsearch-data
mountPath: /usr/share/elasticsearch/data
- name: elasticsearch-tmp
mountPath: /tmp/
volumes:
- emptyDir: {}
name: elasticsearch-keystore
- emptyDir: {}
name: elasticsearch-tmp
- emptyDir: {}
name: elasticsearch-logs
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "10Gi"
storageClassName: longhorn
---
apiVersion: v1
kind: Service
metadata:
name: elasticsearch
labels:
app: elasticsearch
spec:
ports:
- name: api
port: 80
targetPort: 9200
selector:
app: elasticsearch
---
apiVersion: v1
kind: Service
metadata:
name: graylog-gelf-tcp
labels:
app: graylog
spec:
ports:
- name: graylog-gelf-tcp
port: 12201
protocol: TCP
targetPort: 12201
selector:
app: graylog
---
apiVersion: v1
kind: Service
metadata:
name: graylog-syslog-tcp
labels:
app: graylog
annotations:
external-dns.alpha.kubernetes.io/hostname: syslog.k-space.ee
metallb.universe.tf/allow-shared-ip: syslog.k-space.ee
spec:
type: LoadBalancer
externalTrafficPolicy: Local
loadBalancerIP: 172.20.51.4
ports:
- name: graylog-syslog
port: 514
protocol: TCP
selector:
app: graylog
---
apiVersion: v1
kind: Service
metadata:
name: graylog-syslog-udp
labels:
app: graylog
annotations:
external-dns.alpha.kubernetes.io/hostname: syslog.k-space.ee
metallb.universe.tf/allow-shared-ip: syslog.k-space.ee
spec:
type: LoadBalancer
externalTrafficPolicy: Local
loadBalancerIP: 172.20.51.4
ports:
- name: graylog-syslog
port: 514
protocol: UDP
selector:
app: graylog
---
apiVersion: v1
kind: Service
metadata:
name: graylog
labels:
app: graylog
spec:
ports:
- name: graylog
port: 9000
protocol: TCP
selector:
app: graylog
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: graylog
labels:
app: graylog
annotations:
keel.sh/policy: minor
keel.sh/trigger: poll
keel.sh/pollSchedule: "@midnight"
spec:
serviceName: graylog
revisionHistoryLimit: 0
replicas: 1
selector:
matchLabels:
app: graylog
template:
metadata:
labels:
app: graylog
annotations:
prometheus.io/port: "9833"
prometheus.io/scrape: "true"
spec:
securityContext:
fsGroup: 1100
volumes:
- name: graylog-config
downwardAPI:
items:
- path: id
fieldRef:
fieldPath: metadata.name
containers:
- name: graylog
image: graylog/graylog:4.3
env:
- name: GRAYLOG_MONGODB_URI
valueFrom:
secretKeyRef:
name: mongodb-application-readwrite
key: connectionString.standard
- name: GRAYLOG_PROMETHEUS_EXPORTER_ENABLED
value: "true"
- name: GRAYLOG_PROMETHEUS_EXPORTER_BIND_ADDRESS
value: "0.0.0.0:9833"
- name: GRAYLOG_NODE_ID_FILE
value: /config/id
- name: GRAYLOG_HTTP_EXTERNAL_URI
value: "https://graylog.k-space.ee/"
- name: GRAYLOG_TRUSTED_PROXIES
value: "0.0.0.0/0"
- name: GRAYLOG_ELASTICSEARCH_HOSTS
value: "http://elasticsearch"
- name: GRAYLOG_MESSAGE_JOURNAL_ENABLED
value: "false"
- name: GRAYLOG_ROTATION_STRATEGY
value: "size"
- name: GRAYLOG_ELASTICSEARCH_MAX_SIZE_PER_INDEX
value: "268435456"
- name: GRAYLOG_ELASTICSEARCH_MAX_NUMBER_OF_INDICES
value: "16"
envFrom:
- secretRef:
name: graylog-secrets
securityContext:
runAsNonRoot: true
runAsUser: 1100
ports:
- containerPort: 9000
name: graylog
- containerPort: 9833
name: graylog-metrics
livenessProbe:
httpGet:
path: /api/system/lbstatus
port: 9000
initialDelaySeconds: 5
periodSeconds: 30
failureThreshold: 3
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /api/system/lbstatus
port: 9000
initialDelaySeconds: 5
periodSeconds: 10
failureThreshold: 3
successThreshold: 1
timeoutSeconds: 5
volumeMounts:
- name: graylog-config
mountPath: /config
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: graylog
annotations:
cert-manager.io/cluster-issuer: default
traefik.ingress.kubernetes.io/router.entrypoints: websecure
traefik.ingress.kubernetes.io/router.tls: "true"
external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd
spec:
rules:
- host: graylog.k-space.ee
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: graylog
port:
number: 9000
tls:
- hosts:
- graylog.k-space.ee
secretName: graylog-tls
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: graylog
spec:
podSelector:
matchLabels:
app: graylog
policyTypes:
- Ingress
- Egress
egress:
- to:
- podSelector:
matchLabels:
app: elasticsearch
ports:
- port: 9200
- to:
- podSelector:
matchLabels:
app: mongodb-svc
ports:
- port: 27017
ingress:
- from:
- ipBlock:
cidr: 172.23.0.0/16
- ipBlock:
cidr: 172.21.0.0/16
- ipBlock:
cidr: 100.102.0.0/16
ports:
- protocol: UDP
port: 514
- protocol: TCP
port: 514
- from:
- podSelector:
matchLabels:
app: fluent-bit
ports:
- protocol: TCP
port: 12201
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: monitoring
podSelector:
matchLabels:
app: prometheus
ports:
- port: 9833
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: traefik
podSelector:
matchLabels:
app.kubernetes.io/name: traefik
ports:
- protocol: TCP
port: 9000
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: elasticsearch
spec:
podSelector:
matchLabels:
app: elasticsearch
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app: graylog
egress:
- to:
- ipBlock:
# geoip.elastic.co updates
cidr: 0.0.0.0/0
ports:
- port: 443
---
apiVersion: mongodbcommunity.mongodb.com/v1
kind: MongoDBCommunity
metadata:
name: mongodb
spec:
members: 3
type: ReplicaSet
version: "5.0.9"
security:
authentication:
modes: ["SCRAM"]
users:
- name: readwrite
db: application
passwordSecretRef:
name: mongodb-application-readwrite-password
roles:
- name: readWrite
db: application
scramCredentialsSecretName: mongodb-application-readwrite
- name: readonly
db: application
passwordSecretRef:
name: mongodb-application-readonly-password
roles:
- name: readOnly
db: application
scramCredentialsSecretName: mongodb-application-readonly
statefulSet:
spec:
template:
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- mongodb-svc
topologyKey: kubernetes.io/hostname
nodeSelector:
dedicated: storage
tolerations:
- key: dedicated
operator: Equal
value: storage
effect: NoSchedule
volumeClaimTemplates:
- metadata:
name: logs-volume
spec:
storageClassName: local-path
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 512Mi
- metadata:
name: data-volume
spec:
storageClassName: local-path
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
Reference in New Issue View Git Blame Copy Permalink