1
0
forked from k-space/kube
kube/logmower/application.yml

513 lines
12 KiB
YAML
Raw Normal View History

2022-11-05 18:55:52 +00:00
---
2024-07-27 00:15:41 +00:00
apiVersion: codemowers.cloud/v1beta1
kind: OIDCMiddlewareClient
2023-07-28 09:21:50 +00:00
metadata:
name: frontend
spec:
displayName: Kubernetes pod log aggregator
uri: 'https://log.k-space.ee'
allowedGroups:
2023-08-16 07:45:55 +00:00
- k-space:kubernetes:developers
2023-07-28 09:21:50 +00:00
- k-space:kubernetes:admins
headerMapping:
email: Remote-Email
groups: Remote-Groups
name: Remote-Name
user: Remote-Username
---
apiVersion: codemowers.cloud/v1beta1
kind: SecretClaim
2022-11-05 18:55:52 +00:00
metadata:
name: logmower-readwrite-password
spec:
2023-07-28 09:21:50 +00:00
size: 32
2022-11-05 18:55:52 +00:00
mapping:
- key: password
2023-08-16 07:45:55 +00:00
value: "%(plaintext)s"
2022-11-05 18:55:52 +00:00
---
2023-07-28 09:21:50 +00:00
apiVersion: codemowers.cloud/v1beta1
kind: SecretClaim
2022-11-05 18:55:52 +00:00
metadata:
name: logmower-readonly-password
spec:
2023-07-28 09:21:50 +00:00
size: 32
2022-11-05 18:55:52 +00:00
mapping:
- key: password
2023-08-15 10:54:49 +00:00
value: "%(plaintext)s"
2022-11-05 18:55:52 +00:00
---
apiVersion: mongodbcommunity.mongodb.com/v1
kind: MongoDBCommunity
metadata:
name: logmower-mongodb
spec:
2023-08-15 10:54:49 +00:00
agent:
logLevel: ERROR
maxLogFileDurationHours: 1
2022-11-05 18:55:52 +00:00
additionalMongodConfig:
systemLog:
quiet: true
2022-12-14 16:56:08 +00:00
members: 2
arbiters: 1
2022-11-05 18:55:52 +00:00
type: ReplicaSet
2022-12-14 16:56:08 +00:00
version: "6.0.3"
2022-11-05 18:55:52 +00:00
security:
authentication:
modes: ["SCRAM"]
users:
- name: readwrite
db: application
passwordSecretRef:
name: logmower-readwrite-password
roles:
- name: readWrite
db: application
scramCredentialsSecretName: logmower-readwrite
- name: readonly
db: application
passwordSecretRef:
name: logmower-readonly-password
roles:
- name: read
db: application
scramCredentialsSecretName: logmower-readonly
statefulSet:
spec:
logLevel: WARN
template:
spec:
containers:
- name: mongod
resources:
requests:
cpu: 100m
2022-11-09 09:49:13 +00:00
memory: 1Gi
2022-11-05 18:55:52 +00:00
limits:
2022-11-09 09:49:13 +00:00
cpu: 4000m
memory: 1Gi
2022-12-14 16:56:08 +00:00
volumeMounts:
- name: journal-volume
mountPath: /data/journal
2022-11-05 18:55:52 +00:00
- name: mongodb-agent
resources:
requests:
cpu: 1m
memory: 100Mi
limits: {}
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
2022-12-23 21:31:10 +00:00
- logmower-mongodb-svc
2022-11-05 18:55:52 +00:00
topologyKey: kubernetes.io/hostname
nodeSelector:
dedicated: monitoring
tolerations:
- key: dedicated
operator: Equal
value: monitoring
effect: NoSchedule
volumeClaimTemplates:
- metadata:
name: logs-volume
2022-12-14 16:56:08 +00:00
labels:
usecase: logs
2022-11-05 18:55:52 +00:00
spec:
2022-12-14 16:56:08 +00:00
storageClassName: mongo
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 100Mi
- metadata:
name: journal-volume
labels:
usecase: journal
spec:
storageClassName: mongo
2022-11-05 18:55:52 +00:00
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 512Mi
- metadata:
name: data-volume
2022-12-14 16:56:08 +00:00
labels:
usecase: data
2022-11-05 18:55:52 +00:00
spec:
2022-12-14 16:56:08 +00:00
storageClassName: mongo
2022-11-05 18:55:52 +00:00
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: logmower-shipper
spec:
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 50%
selector:
matchLabels:
app: logmower-shipper
template:
metadata:
labels:
app: logmower-shipper
spec:
serviceAccountName: logmower-shipper
containers:
- name: logmower-shipper
image: logmower/shipper:latest
2022-11-05 18:55:52 +00:00
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
2022-11-09 09:49:13 +00:00
- name: MONGO_URI
2022-11-05 18:55:52 +00:00
valueFrom:
secretKeyRef:
name: logmower-mongodb-application-readwrite
key: connectionString.standard
ports:
- containerPort: 8000
name: metrics
2022-12-14 16:56:08 +00:00
securityContext:
readOnlyRootFilesystem: true
2022-11-05 18:55:52 +00:00
command:
2022-12-14 16:56:08 +00:00
- /app/log_shipper.py
2022-11-10 19:07:54 +00:00
- --parse-json
- --normalize-log-level
- --stream-to-log-level
- --merge-top-level
- --max-collection-size
2022-11-05 18:55:52 +00:00
- "10000000000"
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: etcmachineid
mountPath: /etc/machine-id
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: etcmachineid
hostPath:
path: /etc/machine-id
- name: varlog
hostPath:
path: /var/log
tolerations:
- operator: "Exists"
effect: "NoSchedule"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: logging-logmower-shipper
subjects:
- kind: ServiceAccount
name: logmower-shipper
namespace: logmower
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: logmower-shipper
labels:
app: logmower-shipper
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: logmower-shipper
spec:
podSelector:
matchLabels:
app: logmower-shipper
policyTypes:
- Ingress
- Egress
ingress:
- from:
- namespaceSelector:
matchLabels:
2023-08-19 06:50:53 +00:00
kubernetes.io/metadata.name: monitoring
2022-11-05 18:55:52 +00:00
podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
egress:
- to:
- podSelector:
matchLabels:
app: logmower-mongodb-svc
ports:
- port: 27017
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: logmower-eventsource
spec:
podSelector:
matchLabels:
app: logmower-eventsource
policyTypes:
- Ingress
- Egress
egress:
- to:
- podSelector:
matchLabels:
app: logmower-mongodb-svc
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: traefik
podSelector:
matchLabels:
app.kubernetes.io/name: traefik
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: logmower-frontend
spec:
podSelector:
matchLabels:
app: logmower-frontend
policyTypes:
- Ingress
- Egress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: traefik
podSelector:
matchLabels:
app.kubernetes.io/name: traefik
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: logmower-shipper
spec:
selector:
matchLabels:
app: logmower-shipper
podMetricsEndpoints:
- port: metrics
---
2022-12-14 16:56:08 +00:00
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: logmower-shipper
spec:
groups:
- name: logmower-shipper
rules:
- alert: LogmowerSingleInsertionErrors
annotations:
summary: Logmower shipper is having issues submitting log records
to database
expr: rate(logmower_insertion_error_count_total[30m]) > 0
for: 0m
labels:
severity: warning
- alert: LogmowerBulkInsertionErrors
annotations:
summary: Logmower shipper is having issues submitting log records
to database
expr: rate(logmower_bulk_insertion_error_count_total[30m]) > 0
for: 0m
labels:
severity: warning
- alert: LogmowerHighDatabaseLatency
annotations:
summary: Database operations are slow
expr: histogram_quantile(0.95, logmower_database_operation_latency_bucket) > 10
for: 1m
labels:
severity: warning
---
2022-11-05 18:55:52 +00:00
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: logmower
annotations:
kubernetes.io/ingress.class: traefik
cert-manager.io/cluster-issuer: default
traefik.ingress.kubernetes.io/router.entrypoints: websecure
2023-07-28 09:21:50 +00:00
traefik.ingress.kubernetes.io/router.middlewares: logmower-frontend@kubernetescrd
2022-11-05 18:55:52 +00:00
traefik.ingress.kubernetes.io/router.tls: "true"
external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
spec:
rules:
- host: log.k-space.ee
http:
paths:
- pathType: Prefix
path: "/events"
backend:
service:
name: logmower-eventsource
port:
number: 3002
- pathType: Prefix
path: "/"
backend:
service:
name: logmower-frontend
port:
number: 8080
tls:
- hosts:
- "*.k-space.ee"
---
apiVersion: v1
kind: Service
metadata:
name: logmower-eventsource
spec:
type: ClusterIP
selector:
app: logmower-eventsource
ports:
- protocol: TCP
port: 3002
---
apiVersion: v1
kind: Service
metadata:
name: logmower-frontend
spec:
type: ClusterIP
selector:
app: logmower-frontend
ports:
- protocol: TCP
port: 8080
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logmower-frontend
spec:
selector:
matchLabels:
app: logmower-frontend
template:
metadata:
labels:
app: logmower-frontend
spec:
containers:
2022-12-14 16:56:08 +00:00
- name: logmower-frontend
image: logmower/frontend:latest
2022-12-14 16:56:08 +00:00
ports:
- containerPort: 8080
name: http
securityContext:
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
resources:
limits:
memory: 50Mi
requests:
cpu: 1m
memory: 20Mi
volumeMounts:
- name : nginx-cache
mountPath: /var/cache/nginx/
- name : nginx-config
mountPath: /var/config/nginx/
- name: var-run
mountPath: /var/run/
volumes:
- emptyDir: {}
name: nginx-cache
- emptyDir: {}
name: nginx-config
- emptyDir: {}
name: var-run
2022-11-05 18:55:52 +00:00
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logmower-eventsource
spec:
selector:
matchLabels:
app: logmower-eventsource
template:
metadata:
labels:
app: logmower-eventsource
spec:
containers:
2022-12-14 16:56:08 +00:00
- name: logmower-eventsource
image: logmower/eventsource:latest
2022-12-14 16:56:08 +00:00
ports:
- containerPort: 3002
name: nodejs
securityContext:
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
resources:
limits:
cpu: 500m
memory: 200Mi
requests:
cpu: 10m
memory: 100Mi
env:
- name: MONGODB_HOST
valueFrom:
secretKeyRef:
name: logmower-mongodb-application-readonly
key: connectionString.standard
2022-11-05 18:55:52 +00:00
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: logmower-mongodb
spec:
podSelector:
matchLabels:
app: logmower-mongodb-svc
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector: {}
ports:
- port: 27017
egress:
- to:
- podSelector:
matchLabels:
app: logmower-mongodb-svc
ports:
- port: 27017