From c4208037e2515b45b003d5141b2e0f58700942ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauri=20V=C3=B5sandi?= Date: Fri, 16 Sep 2022 08:33:29 +0300 Subject: [PATCH] logging: Replace Graylog with ZincSearch --- logging/README.md | 2 +- logging/application.yml | 383 ------------------------------------ logging/filebeat.yml | 225 ++++++++++----------- logging/mongodb-support.yml | 1 - logging/zinc.yml | 122 ++++++++++++ 5 files changed, 238 insertions(+), 495 deletions(-) delete mode 120000 logging/mongodb-support.yml create mode 100644 logging/zinc.yml diff --git a/logging/README.md b/logging/README.md index 2c3efb0..a14aa4f 100644 --- a/logging/README.md +++ b/logging/README.md @@ -14,7 +14,7 @@ To deploy: ``` kubectl create namespace logging -kubectl apply -n logging -f mongodb-support.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml +kubectl apply -n logging -f zinc.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml kubectl rollout restart -n logging daemonset.apps/filebeat ``` diff --git a/logging/application.yml b/logging/application.yml index 62c8e65..97b5222 100644 --- a/logging/application.yml +++ b/logging/application.yml @@ -1,86 +1,4 @@ --- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: elasticsearch - labels: - app: elasticsearch -spec: - serviceName: elasticsearch - revisionHistoryLimit: 0 - replicas: 1 - selector: - matchLabels: - app: elasticsearch - template: - metadata: - labels: - app: elasticsearch - spec: - securityContext: - fsGroup: 1000 - containers: - - name: elasticsearch - image: elasticsearch:7.17.3 - securityContext: - runAsNonRoot: true - runAsUser: 1000 - env: - - name: discovery.type - value: single-node - - name: xpack.security.enabled - value: "false" - ports: - - containerPort: 9200 - readinessProbe: - httpGet: - path: /_cluster/health - port: 9200 - initialDelaySeconds: 5 - periodSeconds: 10 - failureThreshold: 3 - successThreshold: 1 - timeoutSeconds: 5 - resources: - limits: - memory: "2147483648" - volumeMounts: - - name: elasticsearch-data - mountPath: /usr/share/elasticsearch/data - - name: elasticsearch-tmp - mountPath: /tmp/ - volumes: - - emptyDir: {} - name: elasticsearch-keystore - - emptyDir: {} - name: elasticsearch-tmp - - emptyDir: {} - name: elasticsearch-logs - volumeClaimTemplates: - - metadata: - name: elasticsearch-data - spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: "10Gi" - storageClassName: longhorn ---- -apiVersion: v1 -kind: Service -metadata: - name: elasticsearch - labels: - app: elasticsearch -spec: - ports: - - name: api - port: 80 - targetPort: 9200 - selector: - app: elasticsearch ---- apiVersion: v1 kind: Service metadata: @@ -149,304 +67,3 @@ spec: protocol: UDP selector: app: graylog ---- -apiVersion: v1 -kind: Service -metadata: - name: graylog - labels: - app: graylog -spec: - ports: - - name: graylog - port: 9000 - protocol: TCP - selector: - app: graylog ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: graylog - labels: - app: graylog - annotations: - keel.sh/policy: minor - keel.sh/trigger: poll - keel.sh/pollSchedule: "@midnight" -spec: - serviceName: graylog - revisionHistoryLimit: 0 - replicas: 1 - selector: - matchLabels: - app: graylog - template: - metadata: - labels: - app: graylog - annotations: - prometheus.io/port: "9833" - prometheus.io/scrape: "true" - spec: - securityContext: - fsGroup: 1100 - volumes: - - name: graylog-config - downwardAPI: - items: - - path: id - fieldRef: - fieldPath: metadata.name - containers: - - name: graylog - image: graylog/graylog:4.3 - env: - - name: GRAYLOG_MONGODB_URI - valueFrom: - secretKeyRef: - name: mongodb-application-readwrite - key: connectionString.standard - - name: GRAYLOG_PROMETHEUS_EXPORTER_ENABLED - value: "true" - - name: GRAYLOG_PROMETHEUS_EXPORTER_BIND_ADDRESS - value: "0.0.0.0:9833" - - name: GRAYLOG_NODE_ID_FILE - value: /config/id - - name: GRAYLOG_HTTP_EXTERNAL_URI - value: "https://graylog.k-space.ee/" - - name: GRAYLOG_TRUSTED_PROXIES - value: "0.0.0.0/0" - - name: GRAYLOG_ELASTICSEARCH_HOSTS - value: "http://elasticsearch" - - name: GRAYLOG_MESSAGE_JOURNAL_ENABLED - value: "false" - - name: GRAYLOG_ROTATION_STRATEGY - value: "size" - - name: GRAYLOG_ELASTICSEARCH_MAX_SIZE_PER_INDEX - value: "268435456" - - name: GRAYLOG_ELASTICSEARCH_MAX_NUMBER_OF_INDICES - value: "16" - envFrom: - - secretRef: - name: graylog-secrets - securityContext: - runAsNonRoot: true - runAsUser: 1100 - ports: - - containerPort: 9000 - name: graylog - - containerPort: 9833 - name: graylog-metrics - livenessProbe: - httpGet: - path: /api/system/lbstatus - port: 9000 - initialDelaySeconds: 5 - periodSeconds: 30 - failureThreshold: 3 - successThreshold: 1 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /api/system/lbstatus - port: 9000 - initialDelaySeconds: 5 - periodSeconds: 10 - failureThreshold: 3 - successThreshold: 1 - timeoutSeconds: 5 - volumeMounts: - - name: graylog-config - mountPath: /config ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: graylog - annotations: - cert-manager.io/cluster-issuer: default - traefik.ingress.kubernetes.io/router.entrypoints: websecure - traefik.ingress.kubernetes.io/router.tls: "true" - external-dns.alpha.kubernetes.io/target: traefik.k-space.ee - traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd -spec: - rules: - - host: graylog.k-space.ee - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: graylog - port: - number: 9000 - tls: - - hosts: - - graylog.k-space.ee - secretName: graylog-tls ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: graylog -spec: - podSelector: - matchLabels: - app: graylog - policyTypes: - - Ingress - - Egress - egress: - - to: - - podSelector: - matchLabels: - app: elasticsearch - ports: - - port: 9200 - - to: - - podSelector: - matchLabels: - app: mongodb-svc - ports: - - port: 27017 - ingress: - - from: - - ipBlock: - cidr: 172.23.0.0/16 - - ipBlock: - cidr: 172.21.0.0/16 - - ipBlock: - cidr: 100.102.0.0/16 - ports: - - protocol: UDP - port: 514 - - protocol: TCP - port: 514 - - from: - - podSelector: - matchLabels: - app: filebeat - ports: - - protocol: TCP - port: 5044 - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: monitoring - podSelector: - matchLabels: - app: prometheus - ports: - - port: 9833 - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: traefik - podSelector: - matchLabels: - app.kubernetes.io/name: traefik - ports: - - protocol: TCP - port: 9000 ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: elasticsearch -spec: - podSelector: - matchLabels: - app: elasticsearch - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app: graylog - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: monitoring - podSelector: - matchLabels: - app: grafana - egress: - - to: - - ipBlock: - # geoip.elastic.co updates - cidr: 0.0.0.0/0 - ports: - - port: 443 ---- -apiVersion: mongodbcommunity.mongodb.com/v1 -kind: MongoDBCommunity -metadata: - name: mongodb -spec: - members: 3 - type: ReplicaSet - version: "5.0.9" - security: - authentication: - modes: ["SCRAM"] - users: - - name: readwrite - db: application - passwordSecretRef: - name: mongodb-application-readwrite-password - roles: - - name: readWrite - db: application - scramCredentialsSecretName: mongodb-application-readwrite - - name: readonly - db: application - passwordSecretRef: - name: mongodb-application-readonly-password - roles: - - name: readOnly - db: application - scramCredentialsSecretName: mongodb-application-readonly - statefulSet: - spec: - template: - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - mongodb-svc - topologyKey: kubernetes.io/hostname - nodeSelector: - dedicated: storage - tolerations: - - key: dedicated - operator: Equal - value: storage - effect: NoSchedule - volumeClaimTemplates: - - metadata: - name: logs-volume - spec: - storageClassName: local-path - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 512Mi - - metadata: - name: data-volume - spec: - storageClassName: local-path - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi diff --git a/logging/filebeat.yml b/logging/filebeat.yml index 3b54066..8c488b5 100644 --- a/logging/filebeat.yml +++ b/logging/filebeat.yml @@ -6,18 +6,13 @@ metadata: namespace: logging data: filebeat.yml: |- + setup: + ilm: + enabled: false + template: + name: filebeat + pattern: filebeat-* http.enabled: true - filebeat.inputs: - - type: container - paths: - - /var/log/containers/*.log - processors: - - add_kubernetes_metadata: - in_cluster: true - host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" filebeat.autodiscover: providers: - type: kubernetes @@ -27,50 +22,24 @@ data: type: container paths: - /var/log/containers/*${data.kubernetes.container.id}.log - processors: - - add_host_metadata: - - drop_fields: - fields: - - stream - ignore_missing: true - - rename: - fields: - - from: "kubernetes.node.name" - to: "source" - - from: "kubernetes.pod.name" - to: "pod" - - from: "stream" - to: "stream" - - from: "kubernetes.labels.app" - to: "app" - - from: "kubernetes.namespace" - to: "namespace" - ignore_missing: true - - drop_fields: - fields: - - agent - - container - - ecs - - host - - kubernetes - - log - - "@metadata" - ignore_missing: true - output.logstash: - hosts: ["graylog-logstash:5044"] - #output.console: - # pretty: true + output: + elasticsearch: + hosts: + - http://zinc:4080 + path: "/es/" + index: "filebeat-%{+yyyy.MM.dd}" + username: "${ZINC_FIRST_ADMIN_USER}" + password: "${ZINC_FIRST_ADMIN_PASSWORD}" --- apiVersion: apps/v1 kind: DaemonSet metadata: name: filebeat - namespace: logging spec: updateStrategy: type: RollingUpdate rollingUpdate: - maxUnavailable: 100% + maxUnavailable: 50% selector: matchLabels: app: filebeat @@ -78,72 +47,86 @@ spec: metadata: labels: app: filebeat + annotations: + co.elastic.logs/json.keys_under_root: "true" spec: serviceAccountName: filebeat containers: - - name: filebeat - image: docker.elastic.co/beats/filebeat:7.17.6 - args: - - -c - - /etc/filebeat.yml - - -e - securityContext: - runAsUser: 0 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - ports: - - containerPort: 5066 - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: filebeat-config - mountPath: /etc/filebeat.yml - readOnly: true - subPath: filebeat.yml - - name: data - mountPath: /usr/share/filebeat/data - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - - name: varlog - mountPath: /var/log - readOnly: true + - name: filebeat + image: docker.elastic.co/beats/filebeat:8.4.1 + args: + - -c + - /etc/filebeat.yml + - -e + securityContext: + runAsUser: 0 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ZINC_FIRST_ADMIN_USER + value: admin + - name: ZINC_FIRST_ADMIN_PASSWORD + value: salakala + ports: + - containerPort: 5066 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: filebeat-config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + - name: exporter + image: sepa/beats-exporter + args: + - -p=5066 + ports: + - containerPort: 8080 + name: exporter + protocol: TCP volumes: - - name: filebeat-config - configMap: - defaultMode: 0600 - name: filebeat-config - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers - - name: varlog - hostPath: - path: /var/log - - name: data - hostPath: - path: /var/lib/filebeat-data - type: DirectoryOrCreate + - name: filebeat-config + configMap: + defaultMode: 0600 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log + - name: data + hostPath: + path: /var/lib/filebeat-data + type: DirectoryOrCreate tolerations: - - operator: "Exists" - effect: "NoExecute" - - operator: "Exists" - effect: "NoSchedule" + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "NoSchedule" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: logging-filebeat subjects: -- kind: ServiceAccount - name: filebeat - namespace: logging + - kind: ServiceAccount + name: filebeat + namespace: logging roleRef: kind: ClusterRole name: filebeat @@ -166,13 +149,35 @@ spec: matchLabels: app: filebeat policyTypes: - - Ingress - - Egress + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: prometheus-operator + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 8080 egress: - - to: - - podSelector: - matchLabels: - app: graylog - ports: - - protocol: TCP - port: 5044 + - to: + - podSelector: + matchLabels: + app: zinc + ports: + - protocol: TCP + port: 4080 +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: filebeat +spec: + selector: + matchLabels: + app: filebeat + podMetricsEndpoints: + - port: exporter diff --git a/logging/mongodb-support.yml b/logging/mongodb-support.yml deleted file mode 120000 index e834dd2..0000000 --- a/logging/mongodb-support.yml +++ /dev/null @@ -1 +0,0 @@ -../mongodb-operator/mongodb-support.yml \ No newline at end of file diff --git a/logging/zinc.yml b/logging/zinc.yml new file mode 100644 index 0000000..81080dc --- /dev/null +++ b/logging/zinc.yml @@ -0,0 +1,122 @@ +apiVersion: v1 +kind: Service +metadata: + name: zinc +spec: + clusterIP: None + selector: + app: zinc + ports: + - name: http + port: 4080 + targetPort: 4080 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zinc +spec: + serviceName: zinc + replicas: 1 + selector: + matchLabels: + app: zinc + template: + metadata: + labels: + app: zinc + spec: + securityContext: + fsGroup: 2000 + runAsUser: 10000 + runAsGroup: 3000 + runAsNonRoot: true + containers: + - name: zinc + image: public.ecr.aws/zinclabs/zinc:latest + env: + - name: GIN_MODE + value: release + - name: ZINC_FIRST_ADMIN_USER + value: admin + - name: ZINC_FIRST_ADMIN_PASSWORD + value: salakala + - name: ZINC_DATA_PATH + value: /data + imagePullPolicy: Always + resources: + limits: + cpu: "4" + memory: 2048Mi + requests: + cpu: 32m + memory: 50Mi + ports: + - containerPort: 4080 + name: http + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 20Gi +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: zinc + annotations: + cert-manager.io/cluster-issuer: default + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + external-dns.alpha.kubernetes.io/target: traefik.k-space.ee + traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd +spec: + rules: + - host: zinc.k-space.ee + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: zinc + port: + number: 4080 + tls: + - hosts: + - zinc.k-space.ee + secretName: zinc-tls +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: zinc +spec: + podSelector: + matchLabels: + app: zinc + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: filebeat + ports: + - protocol: TCP + port: 4080 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app.kubernetes.io/name: traefik