kube/logging/application.yml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluent-bit
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluent-bit-read
rules:
- apiGroups: [""]
  resources:
  - namespaces
  - pods
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fluent-bit-read
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluent-bit-read
subjects:
- kind: ServiceAccount
  name: fluent-bit
  namespace: logging
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: logging
  labels:
    app: fluent-bit
  annotations:
    reloader.stakater.com/match: "true"
data:
  fluent-bit.conf: |
    [SERVICE]
        Flush 1
        Log_Level warn
        Daemon off
        Parsers_File parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port 2020
    @INCLUDE input-kubernetes.conf
    @INCLUDE filter-kubernetes.conf
    @INCLUDE output-graylog.conf    
  input-kubernetes.conf: |
    # Following assembles the log fragments of the Kubernetes runtime
    # https://github.com/fluent/fluent-bit/blob/d3c71f2ed4ff3625b85715aaefe6bc76b2ac3c2e/src/multiline/flb_ml_parser_docker.c#L57
    [INPUT]
        name tail
        tag kube.*
        path /var/log/containers/*.log
        multiline.parser cri
        db /var/log/flb_kube.db
        mem_buf_limit 5MB
        skip_long_lines on
        refresh_interval 10    
  filter-kubernetes.conf: |
    # Following reassembles stack traces
    [FILTER]
        name multiline
        match *
        multiline.key_content log
        multiline.parser go,python,java
    # Following annotates the Kubernetes logs using Kubernetes API-s
    [FILTER]
        Name kubernetes
        Match kube.*
        Kube_URL https://kubernetes.default.svc:443
        Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix kube.var.log.containers.
        Merge_Log On
        K8S-Logging.Parser On
        K8S-Logging.Exclude Off
    # Following unnests the kubernetes map
    [FILTER]
        Name nest
        Match kube.*
        Operation lift
        Nested_under kubernetes
        Add_prefix kubernetes_    
  output-graylog.conf: |
    [OUTPUT]
        Name gelf
        Match *
        Host graylog-gelf-tcp
        Port 12201
        Mode tcp
        Gelf_Host_Key kubernetes_host
        Gelf_Short_Message_Key log
        Retry_Limit no_limits    
  parsers.conf: |
    # http://rubular.com/r/tjUt3Awgg4
    [PARSER]
        Name cri
        Format regex
        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z    
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: logging
  annotations:
    keel.sh/policy: patch
    keel.sh/trigger: poll
    keel.sh/pollSchedule: "@midnight"
  annotations:
    reloader.stakater.com/search: "true"
spec:
  revisionHistoryLimit: 0
  selector:
    matchLabels:
      app: fluent-bit
  template:
    metadata:
      labels:
        app: fluent-bit
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "2020"
        prometheus.io/path: /api/v1/metrics/prometheus
    spec:
      containers:
      - name: fluent-bit
        image: fluent/fluent-bit:1.9
        imagePullPolicy: Always
        ports:
          - containerPort: 2020
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: fluent-bit-config
          mountPath: /fluent-bit/etc/
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: fluent-bit-config
        configMap:
          name: fluent-bit-config
      serviceAccountName: fluent-bit
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Exists
        effect: NoSchedule
      - operator: "Exists"
        effect: "NoExecute"
      - operator: "Exists"
        effect: "NoSchedule"
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: fluent-bit
spec:
  podSelector:
    matchLabels:
      app: fluent-bit
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: monitoring
      podSelector:
        matchLabels:
          app: prometheus
    ports:
    - port: 2020
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: graylog
    ports:
    - protocol: TCP
      port: 12201
  - # Kubernetes API endpoint kubernetes.default.svc.cluster.local
    # Determine IP-s and ports with: kubectl get ep -n default kubernetes
    to:
    - ipBlock:
        cidr: 172.21.3.0/24
    ports:
    - port: 6443
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
spec:
  serviceName: elasticsearch
  revisionHistoryLimit: 0
  replicas: 1
  selector:
    matchLabels:
      app: elasticsearch
  template:
    metadata:
      labels:
        app: elasticsearch
    spec:
      securityContext:
        fsGroup: 1000
      containers:
        - name: elasticsearch
          image: elasticsearch:7.17.3
          securityContext:
            runAsNonRoot: true
            runAsUser: 1000
          env:
            - name: discovery.type
              value: single-node
            - name: xpack.security.enabled
              value: "false"
          ports:
            - containerPort: 9200
          readinessProbe:
            httpGet:
              path: /_cluster/health
              port: 9200
            initialDelaySeconds: 5
            periodSeconds: 10
            failureThreshold: 3
            successThreshold: 1
            timeoutSeconds: 5
          resources:
            limits:
              memory: "2147483648"
          volumeMounts:
          - name: elasticsearch-data
            mountPath: /usr/share/elasticsearch/data
          - name: elasticsearch-tmp
            mountPath: /tmp/
      volumes:
      - emptyDir: {}
        name: elasticsearch-keystore
      - emptyDir: {}
        name: elasticsearch-tmp
      - emptyDir: {}
        name: elasticsearch-logs
  volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data
      spec:
        accessModes:
          - "ReadWriteOnce"
        resources:
          requests:
            storage: "10Gi"
        storageClassName: longhorn
---
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
spec:
  ports:
  - name: api
    port: 80
    targetPort: 9200
  selector:
    app: elasticsearch
---
apiVersion: v1
kind: Service
metadata:
  name: graylog-gelf-tcp
  labels:
    app: graylog
spec:
  ports:
    - name: graylog-gelf-tcp
      port: 12201
      protocol: TCP
      targetPort: 12201
  selector:
    app: graylog
---
apiVersion: v1
kind: Service
metadata:
  name: graylog-syslog-tcp
  labels:
    app: graylog
  annotations:
    external-dns.alpha.kubernetes.io/hostname: syslog.k-space.ee
    metallb.universe.tf/allow-shared-ip: syslog.k-space.ee
spec:
  type: LoadBalancer
  externalTrafficPolicy: Local
  loadBalancerIP: 172.20.51.4
  ports:
    - name: graylog-syslog
      port: 514
      protocol: TCP
  selector:
    app: graylog
---
apiVersion: v1
kind: Service
metadata:
  name: graylog-syslog-udp
  labels:
    app: graylog
  annotations:
    external-dns.alpha.kubernetes.io/hostname: syslog.k-space.ee
    metallb.universe.tf/allow-shared-ip: syslog.k-space.ee
spec:
  type: LoadBalancer
  externalTrafficPolicy: Local
  loadBalancerIP: 172.20.51.4
  ports:
    - name: graylog-syslog
      port: 514
      protocol: UDP
  selector:
    app: graylog
---
apiVersion: v1
kind: Service
metadata:
  name: graylog
  labels:
    app: graylog
spec:
  ports:
    - name: graylog
      port: 9000
      protocol: TCP
  selector:
    app: graylog
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: graylog
  labels:
    app: graylog
  annotations:
    keel.sh/policy: minor
    keel.sh/trigger: poll
    keel.sh/pollSchedule: "@midnight"
spec:
  serviceName: graylog
  revisionHistoryLimit: 0
  replicas: 1
  selector:
    matchLabels:
      app: graylog
  template:
    metadata:
      labels:
        app: graylog
      annotations:
        prometheus.io/port: "9833"
        prometheus.io/scrape: "true"
    spec:
      securityContext:
        fsGroup: 1100
      volumes:
        - name: graylog-config
          downwardAPI:
            items:
              - path: id
                fieldRef:
                  fieldPath: metadata.name
      containers:
        - name: graylog
          image: graylog/graylog:4.3
          env:
            - name: GRAYLOG_MONGODB_URI
              valueFrom:
                secretKeyRef:
                  name: mongodb-application-readwrite
                  key: connectionString.standard
            - name: GRAYLOG_PROMETHEUS_EXPORTER_ENABLED
              value: "true"
            - name: GRAYLOG_PROMETHEUS_EXPORTER_BIND_ADDRESS
              value: "0.0.0.0:9833"
            - name: GRAYLOG_NODE_ID_FILE
              value: /config/id
            - name: GRAYLOG_HTTP_EXTERNAL_URI
              value: "https://graylog.k-space.ee/"
            - name: GRAYLOG_TRUSTED_PROXIES
              value: "0.0.0.0/0"
            - name: GRAYLOG_ELASTICSEARCH_HOSTS
              value: "http://elasticsearch"
            - name: GRAYLOG_MESSAGE_JOURNAL_ENABLED
              value: "false"
            - name: GRAYLOG_ROTATION_STRATEGY
              value: "size"
            - name: GRAYLOG_ELASTICSEARCH_MAX_SIZE_PER_INDEX
              value: "268435456"
            - name: GRAYLOG_ELASTICSEARCH_MAX_NUMBER_OF_INDICES
              value: "16"
          envFrom:
            - secretRef:
                name: graylog-secrets
          securityContext:
            runAsNonRoot: true
            runAsUser: 1100
          ports:
            - containerPort: 9000
              name: graylog
            - containerPort: 9833
              name: graylog-metrics
          livenessProbe:
            httpGet:
              path: /api/system/lbstatus
              port: 9000
            initialDelaySeconds: 5
            periodSeconds: 30
            failureThreshold: 3
            successThreshold: 1
            timeoutSeconds: 5
          readinessProbe:
            httpGet:
              path: /api/system/lbstatus
              port: 9000
            initialDelaySeconds: 5
            periodSeconds: 10
            failureThreshold: 3
            successThreshold: 1
            timeoutSeconds: 5
          volumeMounts:
            - name: graylog-config
              mountPath: /config
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: graylog
  annotations:
    cert-manager.io/cluster-issuer: default
    traefik.ingress.kubernetes.io/router.entrypoints: websecure
    traefik.ingress.kubernetes.io/router.tls: "true"
    external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
    traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd
spec:
  rules:
  - host: graylog.k-space.ee
    http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: graylog
            port:
              number: 9000
  tls:
  - hosts:
    - graylog.k-space.ee
    secretName: graylog-tls
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: graylog
spec:
  podSelector:
    matchLabels:
      app: graylog
  policyTypes:
  - Ingress
  - Egress
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: elasticsearch
    ports:
    - port: 9200
  - to:
    - podSelector:
        matchLabels:
          app: mongodb-svc
    ports:
    - port: 27017
  ingress:
  - from:
    - ipBlock:
        cidr: 172.23.0.0/16
    - ipBlock:
        cidr: 172.21.0.0/16
    - ipBlock:
        cidr: 100.102.0.0/16
    ports:
    - protocol: UDP
      port: 514
    - protocol: TCP
      port: 514
  - from:
    - podSelector:
        matchLabels:
          app: fluent-bit
    ports:
    - protocol: TCP
      port: 12201
  - from:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: monitoring
      podSelector:
        matchLabels:
          app: prometheus
    ports:
    - port: 9833
  - from:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: traefik
      podSelector:
        matchLabels:
          app.kubernetes.io/name: traefik
    ports:
    - protocol: TCP
      port: 9000
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: elasticsearch
spec:
  podSelector:
    matchLabels:
      app: elasticsearch
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: graylog
  egress:
  - to:
    - ipBlock:
        # geoip.elastic.co updates
        cidr: 0.0.0.0/0
    ports:
    - port: 443
---
apiVersion: mongodbcommunity.mongodb.com/v1
kind: MongoDBCommunity
metadata:
  name: mongodb
spec:
  members: 3
  type: ReplicaSet
  version: "5.0.9"
  security:
    authentication:
      modes: ["SCRAM"]
  users:
    - name: readwrite
      db: application
      passwordSecretRef:
        name: mongodb-application-readwrite-password
      roles:
        - name: readWrite
          db: application
      scramCredentialsSecretName: mongodb-application-readwrite
    - name: readonly
      db: application
      passwordSecretRef:
        name: mongodb-application-readonly-password
      roles:
        - name: readOnly
          db: application
      scramCredentialsSecretName: mongodb-application-readonly
  statefulSet:
    spec:
      template:
        spec:
          affinity:
            podAntiAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchExpressions:
                      - key: app
                        operator: In
                        values:
                          - mongodb-svc
                  topologyKey: kubernetes.io/hostname
          nodeSelector:
            dedicated: storage
          tolerations:
            - key: dedicated
              operator: Equal
              value: storage
              effect: NoSchedule
      volumeClaimTemplates:
        - metadata:
            name: logs-volume
          spec:
            storageClassName: local-path
            accessModes:
            - ReadWriteOnce
            resources:
              requests:
                storage: 512Mi
        - metadata:
            name: data-volume
          spec:
            storageClassName: local-path
            accessModes:
            - ReadWriteOnce
            resources:
              requests:
                storage: 2Gi