--- apiVersion: codemowers.io/v1alpha1 kind: GeneratedSecret metadata: name: logmower-readwrite-password spec: mapping: - key: password value: "%(password)s" --- apiVersion: codemowers.io/v1alpha1 kind: GeneratedSecret metadata: name: logmower-readonly-password spec: mapping: - key: password value: "%(password)s" --- apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: logmower-mongodb spec: additionalMongodConfig: systemLog: quiet: true members: 2 arbiters: 1 type: ReplicaSet version: "6.0.3" security: authentication: modes: ["SCRAM"] users: - name: readwrite db: application passwordSecretRef: name: logmower-readwrite-password roles: - name: readWrite db: application scramCredentialsSecretName: logmower-readwrite - name: readonly db: application passwordSecretRef: name: logmower-readonly-password roles: - name: read db: application scramCredentialsSecretName: logmower-readonly statefulSet: spec: logLevel: WARN template: spec: containers: - name: mongod resources: requests: cpu: 100m memory: 1Gi limits: cpu: 4000m memory: 1Gi volumeMounts: - name: journal-volume mountPath: /data/journal - name: mongodb-agent resources: requests: cpu: 1m memory: 100Mi limits: {} affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: app operator: In values: - logmower-mongodb-svc topologyKey: kubernetes.io/hostname nodeSelector: dedicated: monitoring tolerations: - key: dedicated operator: Equal value: monitoring effect: NoSchedule volumeClaimTemplates: - metadata: name: logs-volume labels: usecase: logs spec: storageClassName: mongo accessModes: - ReadWriteOnce resources: requests: storage: 100Mi - metadata: name: journal-volume labels: usecase: journal spec: storageClassName: mongo accessModes: - ReadWriteOnce resources: requests: storage: 512Mi - metadata: name: data-volume labels: usecase: data spec: storageClassName: mongo accessModes: - ReadWriteOnce resources: requests: storage: 10Gi --- apiVersion: apps/v1 kind: DaemonSet metadata: name: logmower-shipper spec: updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: 50% selector: matchLabels: app: logmower-shipper template: metadata: labels: app: logmower-shipper spec: serviceAccountName: logmower-shipper containers: - name: logmower-shipper image: harbor.k-space.ee/k-space/logmower-shipper-prototype:latest env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: MONGO_URI valueFrom: secretKeyRef: name: logmower-mongodb-application-readwrite key: connectionString.standard ports: - containerPort: 8000 name: metrics securityContext: readOnlyRootFilesystem: true command: - /app/log_shipper.py - --parse-json - --normalize-log-level - --stream-to-log-level - --merge-top-level - --max-collection-size - "10000000000" resources: limits: memory: 200Mi requests: cpu: 100m memory: 100Mi volumeMounts: - name: etcmachineid mountPath: /etc/machine-id readOnly: true - name: varlog mountPath: /var/log readOnly: true volumes: - name: etcmachineid hostPath: path: /etc/machine-id - name: varlog hostPath: path: /var/log tolerations: - operator: "Exists" effect: "NoSchedule" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: logging-logmower-shipper subjects: - kind: ServiceAccount name: logmower-shipper namespace: logmower roleRef: kind: ClusterRole name: filebeat apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ServiceAccount metadata: name: logmower-shipper labels: app: logmower-shipper --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: logmower-shipper spec: podSelector: matchLabels: app: logmower-shipper policyTypes: - Ingress - Egress ingress: - from: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: prometheus-operator podSelector: matchLabels: app.kubernetes.io/name: prometheus egress: - to: - podSelector: matchLabels: app: logmower-mongodb-svc ports: - port: 27017 --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: logmower-eventsource spec: podSelector: matchLabels: app: logmower-eventsource policyTypes: - Ingress - Egress egress: - to: - podSelector: matchLabels: app: logmower-mongodb-svc ingress: - from: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: traefik podSelector: matchLabels: app.kubernetes.io/name: traefik --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: logmower-frontend spec: podSelector: matchLabels: app: logmower-frontend policyTypes: - Ingress - Egress ingress: - from: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: traefik podSelector: matchLabels: app.kubernetes.io/name: traefik --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: logmower-shipper spec: selector: matchLabels: app: logmower-shipper podMetricsEndpoints: - port: metrics --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: logmower-shipper spec: groups: - name: logmower-shipper rules: - alert: LogmowerSingleInsertionErrors annotations: summary: Logmower shipper is having issues submitting log records to database expr: rate(logmower_insertion_error_count_total[30m]) > 0 for: 0m labels: severity: warning - alert: LogmowerBulkInsertionErrors annotations: summary: Logmower shipper is having issues submitting log records to database expr: rate(logmower_bulk_insertion_error_count_total[30m]) > 0 for: 0m labels: severity: warning - alert: LogmowerHighDatabaseLatency annotations: summary: Database operations are slow expr: histogram_quantile(0.95, logmower_database_operation_latency_bucket) > 10 for: 1m labels: severity: warning --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: logmower annotations: kubernetes.io/ingress.class: traefik cert-manager.io/cluster-issuer: default traefik.ingress.kubernetes.io/router.entrypoints: websecure traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd traefik.ingress.kubernetes.io/router.tls: "true" external-dns.alpha.kubernetes.io/target: traefik.k-space.ee spec: rules: - host: log.k-space.ee http: paths: - pathType: Prefix path: "/events" backend: service: name: logmower-eventsource port: number: 3002 - pathType: Prefix path: "/" backend: service: name: logmower-frontend port: number: 8080 tls: - hosts: - "*.k-space.ee" --- apiVersion: v1 kind: Service metadata: name: logmower-eventsource spec: type: ClusterIP selector: app: logmower-eventsource ports: - protocol: TCP port: 3002 --- apiVersion: v1 kind: Service metadata: name: logmower-frontend spec: type: ClusterIP selector: app: logmower-frontend ports: - protocol: TCP port: 8080 --- apiVersion: apps/v1 kind: Deployment metadata: name: logmower-frontend spec: selector: matchLabels: app: logmower-frontend template: metadata: labels: app: logmower-frontend spec: containers: - name: logmower-frontend image: harbor.k-space.ee/k-space/logmower-frontend ports: - containerPort: 8080 name: http securityContext: readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 1000 resources: limits: memory: 50Mi requests: cpu: 1m memory: 20Mi volumeMounts: - name : nginx-cache mountPath: /var/cache/nginx/ - name : nginx-config mountPath: /var/config/nginx/ - name: var-run mountPath: /var/run/ volumes: - emptyDir: {} name: nginx-cache - emptyDir: {} name: nginx-config - emptyDir: {} name: var-run --- apiVersion: apps/v1 kind: Deployment metadata: name: logmower-eventsource spec: selector: matchLabels: app: logmower-eventsource template: metadata: labels: app: logmower-eventsource spec: containers: - name: logmower-eventsource image: harbor.k-space.ee/k-space/logmower-eventsource ports: - containerPort: 3002 name: nodejs securityContext: readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 1000 resources: limits: cpu: 500m memory: 200Mi requests: cpu: 10m memory: 100Mi env: - name: MONGODB_HOST valueFrom: secretKeyRef: name: logmower-mongodb-application-readonly key: connectionString.standard --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: logmower-mongodb spec: podSelector: matchLabels: app: logmower-mongodb-svc policyTypes: - Ingress - Egress ingress: - from: - podSelector: {} ports: - port: 27017 egress: - to: - podSelector: matchLabels: app: logmower-mongodb-svc ports: - port: 27017