forked from k-space/kube
logging: Replace Graylog with ZincSearch
This commit is contained in:
parent
edcb6399df
commit
c4208037e2
@ -14,7 +14,7 @@ To deploy:
|
||||
|
||||
```
|
||||
kubectl create namespace logging
|
||||
kubectl apply -n logging -f mongodb-support.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml
|
||||
kubectl apply -n logging -f zinc.yml -f application.yml -f filebeat.yml -f networkpolicy-base.yml
|
||||
kubectl rollout restart -n logging daemonset.apps/filebeat
|
||||
```
|
||||
|
||||
|
@ -1,86 +1,4 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: elasticsearch
|
||||
labels:
|
||||
app: elasticsearch
|
||||
spec:
|
||||
serviceName: elasticsearch
|
||||
revisionHistoryLimit: 0
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: elasticsearch
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: elasticsearch
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: elasticsearch
|
||||
image: elasticsearch:7.17.3
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
env:
|
||||
- name: discovery.type
|
||||
value: single-node
|
||||
- name: xpack.security.enabled
|
||||
value: "false"
|
||||
ports:
|
||||
- containerPort: 9200
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /_cluster/health
|
||||
port: 9200
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
memory: "2147483648"
|
||||
volumeMounts:
|
||||
- name: elasticsearch-data
|
||||
mountPath: /usr/share/elasticsearch/data
|
||||
- name: elasticsearch-tmp
|
||||
mountPath: /tmp/
|
||||
volumes:
|
||||
- emptyDir: {}
|
||||
name: elasticsearch-keystore
|
||||
- emptyDir: {}
|
||||
name: elasticsearch-tmp
|
||||
- emptyDir: {}
|
||||
name: elasticsearch-logs
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: elasticsearch-data
|
||||
spec:
|
||||
accessModes:
|
||||
- "ReadWriteOnce"
|
||||
resources:
|
||||
requests:
|
||||
storage: "10Gi"
|
||||
storageClassName: longhorn
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: elasticsearch
|
||||
labels:
|
||||
app: elasticsearch
|
||||
spec:
|
||||
ports:
|
||||
- name: api
|
||||
port: 80
|
||||
targetPort: 9200
|
||||
selector:
|
||||
app: elasticsearch
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
@ -149,304 +67,3 @@ spec:
|
||||
protocol: UDP
|
||||
selector:
|
||||
app: graylog
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: graylog
|
||||
labels:
|
||||
app: graylog
|
||||
spec:
|
||||
ports:
|
||||
- name: graylog
|
||||
port: 9000
|
||||
protocol: TCP
|
||||
selector:
|
||||
app: graylog
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: graylog
|
||||
labels:
|
||||
app: graylog
|
||||
annotations:
|
||||
keel.sh/policy: minor
|
||||
keel.sh/trigger: poll
|
||||
keel.sh/pollSchedule: "@midnight"
|
||||
spec:
|
||||
serviceName: graylog
|
||||
revisionHistoryLimit: 0
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: graylog
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: graylog
|
||||
annotations:
|
||||
prometheus.io/port: "9833"
|
||||
prometheus.io/scrape: "true"
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 1100
|
||||
volumes:
|
||||
- name: graylog-config
|
||||
downwardAPI:
|
||||
items:
|
||||
- path: id
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
containers:
|
||||
- name: graylog
|
||||
image: graylog/graylog:4.3
|
||||
env:
|
||||
- name: GRAYLOG_MONGODB_URI
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mongodb-application-readwrite
|
||||
key: connectionString.standard
|
||||
- name: GRAYLOG_PROMETHEUS_EXPORTER_ENABLED
|
||||
value: "true"
|
||||
- name: GRAYLOG_PROMETHEUS_EXPORTER_BIND_ADDRESS
|
||||
value: "0.0.0.0:9833"
|
||||
- name: GRAYLOG_NODE_ID_FILE
|
||||
value: /config/id
|
||||
- name: GRAYLOG_HTTP_EXTERNAL_URI
|
||||
value: "https://graylog.k-space.ee/"
|
||||
- name: GRAYLOG_TRUSTED_PROXIES
|
||||
value: "0.0.0.0/0"
|
||||
- name: GRAYLOG_ELASTICSEARCH_HOSTS
|
||||
value: "http://elasticsearch"
|
||||
- name: GRAYLOG_MESSAGE_JOURNAL_ENABLED
|
||||
value: "false"
|
||||
- name: GRAYLOG_ROTATION_STRATEGY
|
||||
value: "size"
|
||||
- name: GRAYLOG_ELASTICSEARCH_MAX_SIZE_PER_INDEX
|
||||
value: "268435456"
|
||||
- name: GRAYLOG_ELASTICSEARCH_MAX_NUMBER_OF_INDICES
|
||||
value: "16"
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: graylog-secrets
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1100
|
||||
ports:
|
||||
- containerPort: 9000
|
||||
name: graylog
|
||||
- containerPort: 9833
|
||||
name: graylog-metrics
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/system/lbstatus
|
||||
port: 9000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/system/lbstatus
|
||||
port: 9000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 5
|
||||
volumeMounts:
|
||||
- name: graylog-config
|
||||
mountPath: /config
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: graylog
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: default
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
traefik.ingress.kubernetes.io/router.tls: "true"
|
||||
external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
|
||||
traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd
|
||||
spec:
|
||||
rules:
|
||||
- host: graylog.k-space.ee
|
||||
http:
|
||||
paths:
|
||||
- pathType: Prefix
|
||||
path: "/"
|
||||
backend:
|
||||
service:
|
||||
name: graylog
|
||||
port:
|
||||
number: 9000
|
||||
tls:
|
||||
- hosts:
|
||||
- graylog.k-space.ee
|
||||
secretName: graylog-tls
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: graylog
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: graylog
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: elasticsearch
|
||||
ports:
|
||||
- port: 9200
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: mongodb-svc
|
||||
ports:
|
||||
- port: 27017
|
||||
ingress:
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 172.23.0.0/16
|
||||
- ipBlock:
|
||||
cidr: 172.21.0.0/16
|
||||
- ipBlock:
|
||||
cidr: 100.102.0.0/16
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 514
|
||||
- protocol: TCP
|
||||
port: 514
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: filebeat
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5044
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: monitoring
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: prometheus
|
||||
ports:
|
||||
- port: 9833
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: traefik
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: traefik
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 9000
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: elasticsearch
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: elasticsearch
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: graylog
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: monitoring
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: grafana
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
# geoip.elastic.co updates
|
||||
cidr: 0.0.0.0/0
|
||||
ports:
|
||||
- port: 443
|
||||
---
|
||||
apiVersion: mongodbcommunity.mongodb.com/v1
|
||||
kind: MongoDBCommunity
|
||||
metadata:
|
||||
name: mongodb
|
||||
spec:
|
||||
members: 3
|
||||
type: ReplicaSet
|
||||
version: "5.0.9"
|
||||
security:
|
||||
authentication:
|
||||
modes: ["SCRAM"]
|
||||
users:
|
||||
- name: readwrite
|
||||
db: application
|
||||
passwordSecretRef:
|
||||
name: mongodb-application-readwrite-password
|
||||
roles:
|
||||
- name: readWrite
|
||||
db: application
|
||||
scramCredentialsSecretName: mongodb-application-readwrite
|
||||
- name: readonly
|
||||
db: application
|
||||
passwordSecretRef:
|
||||
name: mongodb-application-readonly-password
|
||||
roles:
|
||||
- name: readOnly
|
||||
db: application
|
||||
scramCredentialsSecretName: mongodb-application-readonly
|
||||
statefulSet:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- mongodb-svc
|
||||
topologyKey: kubernetes.io/hostname
|
||||
nodeSelector:
|
||||
dedicated: storage
|
||||
tolerations:
|
||||
- key: dedicated
|
||||
operator: Equal
|
||||
value: storage
|
||||
effect: NoSchedule
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: logs-volume
|
||||
spec:
|
||||
storageClassName: local-path
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 512Mi
|
||||
- metadata:
|
||||
name: data-volume
|
||||
spec:
|
||||
storageClassName: local-path
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
|
@ -6,18 +6,13 @@ metadata:
|
||||
namespace: logging
|
||||
data:
|
||||
filebeat.yml: |-
|
||||
setup:
|
||||
ilm:
|
||||
enabled: false
|
||||
template:
|
||||
name: filebeat
|
||||
pattern: filebeat-*
|
||||
http.enabled: true
|
||||
filebeat.inputs:
|
||||
- type: container
|
||||
paths:
|
||||
- /var/log/containers/*.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
in_cluster: true
|
||||
host: ${NODE_NAME}
|
||||
matchers:
|
||||
- logs_path:
|
||||
logs_path: "/var/log/containers/"
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: kubernetes
|
||||
@ -27,50 +22,24 @@ data:
|
||||
type: container
|
||||
paths:
|
||||
- /var/log/containers/*${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_host_metadata:
|
||||
- drop_fields:
|
||||
fields:
|
||||
- stream
|
||||
ignore_missing: true
|
||||
- rename:
|
||||
fields:
|
||||
- from: "kubernetes.node.name"
|
||||
to: "source"
|
||||
- from: "kubernetes.pod.name"
|
||||
to: "pod"
|
||||
- from: "stream"
|
||||
to: "stream"
|
||||
- from: "kubernetes.labels.app"
|
||||
to: "app"
|
||||
- from: "kubernetes.namespace"
|
||||
to: "namespace"
|
||||
ignore_missing: true
|
||||
- drop_fields:
|
||||
fields:
|
||||
- agent
|
||||
- container
|
||||
- ecs
|
||||
- host
|
||||
- kubernetes
|
||||
- log
|
||||
- "@metadata"
|
||||
ignore_missing: true
|
||||
output.logstash:
|
||||
hosts: ["graylog-logstash:5044"]
|
||||
#output.console:
|
||||
# pretty: true
|
||||
output:
|
||||
elasticsearch:
|
||||
hosts:
|
||||
- http://zinc:4080
|
||||
path: "/es/"
|
||||
index: "filebeat-%{+yyyy.MM.dd}"
|
||||
username: "${ZINC_FIRST_ADMIN_USER}"
|
||||
password: "${ZINC_FIRST_ADMIN_PASSWORD}"
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: filebeat
|
||||
namespace: logging
|
||||
spec:
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 100%
|
||||
maxUnavailable: 50%
|
||||
selector:
|
||||
matchLabels:
|
||||
app: filebeat
|
||||
@ -78,11 +47,13 @@ spec:
|
||||
metadata:
|
||||
labels:
|
||||
app: filebeat
|
||||
annotations:
|
||||
co.elastic.logs/json.keys_under_root: "true"
|
||||
spec:
|
||||
serviceAccountName: filebeat
|
||||
containers:
|
||||
- name: filebeat
|
||||
image: docker.elastic.co/beats/filebeat:7.17.6
|
||||
image: docker.elastic.co/beats/filebeat:8.4.1
|
||||
args:
|
||||
- -c
|
||||
- /etc/filebeat.yml
|
||||
@ -94,6 +65,10 @@ spec:
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: ZINC_FIRST_ADMIN_USER
|
||||
value: admin
|
||||
- name: ZINC_FIRST_ADMIN_PASSWORD
|
||||
value: salakala
|
||||
ports:
|
||||
- containerPort: 5066
|
||||
resources:
|
||||
@ -115,6 +90,14 @@ spec:
|
||||
- name: varlog
|
||||
mountPath: /var/log
|
||||
readOnly: true
|
||||
- name: exporter
|
||||
image: sepa/beats-exporter
|
||||
args:
|
||||
- -p=5066
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: exporter
|
||||
protocol: TCP
|
||||
volumes:
|
||||
- name: filebeat-config
|
||||
configMap:
|
||||
@ -168,11 +151,33 @@ spec:
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: prometheus-operator
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8080
|
||||
egress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: graylog
|
||||
app: zinc
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5044
|
||||
port: 4080
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: filebeat
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: filebeat
|
||||
podMetricsEndpoints:
|
||||
- port: exporter
|
||||
|
@ -1 +0,0 @@
|
||||
../mongodb-operator/mongodb-support.yml
|
122
logging/zinc.yml
Normal file
122
logging/zinc.yml
Normal file
@ -0,0 +1,122 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: zinc
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app: zinc
|
||||
ports:
|
||||
- name: http
|
||||
port: 4080
|
||||
targetPort: 4080
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: zinc
|
||||
spec:
|
||||
serviceName: zinc
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: zinc
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: zinc
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 2000
|
||||
runAsUser: 10000
|
||||
runAsGroup: 3000
|
||||
runAsNonRoot: true
|
||||
containers:
|
||||
- name: zinc
|
||||
image: public.ecr.aws/zinclabs/zinc:latest
|
||||
env:
|
||||
- name: GIN_MODE
|
||||
value: release
|
||||
- name: ZINC_FIRST_ADMIN_USER
|
||||
value: admin
|
||||
- name: ZINC_FIRST_ADMIN_PASSWORD
|
||||
value: salakala
|
||||
- name: ZINC_DATA_PATH
|
||||
value: /data
|
||||
imagePullPolicy: Always
|
||||
resources:
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 2048Mi
|
||||
requests:
|
||||
cpu: 32m
|
||||
memory: 50Mi
|
||||
ports:
|
||||
- containerPort: 4080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: zinc
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: default
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
traefik.ingress.kubernetes.io/router.tls: "true"
|
||||
external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
|
||||
traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd
|
||||
spec:
|
||||
rules:
|
||||
- host: zinc.k-space.ee
|
||||
http:
|
||||
paths:
|
||||
- pathType: Prefix
|
||||
path: "/"
|
||||
backend:
|
||||
service:
|
||||
name: zinc
|
||||
port:
|
||||
number: 4080
|
||||
tls:
|
||||
- hosts:
|
||||
- zinc.k-space.ee
|
||||
secretName: zinc-tls
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: zinc
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: zinc
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: filebeat
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 4080
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: traefik
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: traefik
|
Loading…
Reference in New Issue
Block a user