kube/prometheus-operator/application.yml

---
apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: alertmanager
  labels:
    app.kubernetes.io/name: alertmanager
spec:
  route:
    receiver: 'slack-notifications'
  receivers:
  - name: 'slack-notifications'
    slackConfigs:
    - channel: '#kube-prod'
      sendResolved: true
      apiURL:
        name: slack-secrets
        key: webhook-url
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: metrics
spec:
  namespaceSelector: {}
  selector: {}
  podMetricsEndpoints:
    - port: exporter
    - port: metrics
---
apiVersion: monitoring.coreos.com/v1
kind: Alertmanager
metadata:
  name: alertmanager
spec:
  alertmanagerConfigSelector:
    matchLabels:
      app.kubernetes.io/name: alertmanager
  secrets:
    - slack-secrets
  nodeSelector:
    dedicated: monitoring
  tolerations:
    - key: dedicated
      operator: Equal
      value: monitoring
      effect: NoSchedule
  replicas: 3
  serviceAccountName: alertmanager
  externalUrl: http://am.k-space.ee/
  routePrefix: "/"
  securityContext:
    fsGroup: 2000
    runAsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: alertmanager
---
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: prometheus
spec:
  nodeSelector:
    dedicated: monitoring
  tolerations:
    - key: dedicated
      operator: Equal
      value: monitoring
      effect: NoSchedule
  alerting:
    alertmanagers:
      - namespace: prometheus-operator
        name: alertmanager-operated
        port: web
  externalUrl: "http://prom.k-space.ee/"
  replicas: 2
  shards: 1
  serviceAccountName: prometheus
  securityContext:
    fsGroup: 2000
    runAsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  podMonitorNamespaceSelector: {}
  podMonitorSelector: {}
  probeNamespaceSelector: {}
  probeSelector: {}
  ruleNamespaceSelector: {}
  ruleSelector: {}
  retentionSize: 80GB
  storage:
    volumeClaimTemplate:
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 100Gi
        storageClassName: local-path
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups: [""]
  resources:
  - nodes
  - nodes/metrics
  - services
  - endpoints
  - pods
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources:
  - configmaps
  verbs: ["get"]
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: prometheus-operator
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: prometheus
spec:
  groups:
  - name: prometheus
    rules:
    - alert: PrometheusJobMissing
      annotations:
        description: "A Prometheus job has disappeared\n  VALUE = {{ $value }}\n \
          \ LABELS = {{ $labels }}"
        summary: Prometheus job missing (instance {{ $labels.instance }})
      expr: absent(up{job="prometheus-operator/prometheus"})
      for: 0m
      labels:
        severity: warning
    - alert: PrometheusTargetMissing
      annotations:
        description: "A Prometheus target has disappeared. An exporter might be crashed.\n\
          \  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus target missing (instance {{ $labels.instance }})
      expr: up == 0
      for: 5m
      labels:
        severity: critical
    - alert: PrometheusAllTargetsMissing
      annotations:
        description: "A Prometheus job does not have living target anymore.\n  VALUE\
          \ = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus all targets missing (instance {{ $labels.instance }})
      expr: count by (job) (up) == 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusConfigurationReloadFailure
      annotations:
        description: "Prometheus configuration reload error\n  VALUE = {{ $value }}\n\
          \  LABELS = {{ $labels }}"
        summary: Prometheus configuration reload failure (instance {{ $labels.instance
          }})
      expr: prometheus_config_last_reload_successful != 1
      for: 0m
      labels:
        severity: warning
    - alert: PrometheusTooManyRestarts
      annotations:
        description: "Prometheus has restarted more than twice in the last 15 minutes.\
          \ It might be crashlooping.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels\
          \ }}"
        summary: Prometheus too many restarts (instance {{ $labels.instance }})
      expr: changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m])
        > 2
      for: 0m
      labels:
        severity: warning
    - alert: PrometheusAlertmanagerJobMissing
      annotations:
        description: "A Prometheus AlertManager job has disappeared\n  VALUE = {{\
          \ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus AlertManager job missing (instance {{ $labels.instance
          }})
      expr: absent(up{job="prometheus-operator/alertmanager"})
      for: 0m
      labels:
        severity: warning
    - alert: PrometheusAlertmanagerConfigurationReloadFailure
      annotations:
        description: "AlertManager configuration reload error\n  VALUE = {{ $value\
          \ }}\n  LABELS = {{ $labels }}"
        summary: Prometheus AlertManager configuration reload failure (instance {{
          $labels.instance }})
      expr: alertmanager_config_last_reload_successful != 1
      for: 0m
      labels:
        severity: warning
    - alert: PrometheusAlertmanagerConfigNotSynced
      annotations:
        description: "Configurations of AlertManager cluster instances are out of\
          \ sync\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus AlertManager config not synced (instance {{ $labels.instance
          }})
      expr: count(count_values("config_hash", alertmanager_config_hash)) > 1
      for: 0m
      labels:
        severity: warning
    - alert: PrometheusNotConnectedToAlertmanager
      annotations:
        description: "Prometheus cannot connect the alertmanager\n  VALUE = {{ $value\
          \ }}\n  LABELS = {{ $labels }}"
        summary: Prometheus not connected to alertmanager (instance {{ $labels.instance
          }})
      expr: prometheus_notifications_alertmanagers_discovered < 1
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusRuleEvaluationFailures
      annotations:
        description: "Prometheus encountered {{ $value }} rule evaluation failures,\
          \ leading to potentially ignored alerts.\n  VALUE = {{ $value }}\n  LABELS\
          \ = {{ $labels }}"
        summary: Prometheus rule evaluation failures (instance {{ $labels.instance
          }})
      expr: increase(prometheus_rule_evaluation_failures_total[3m]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusTemplateTextExpansionFailures
      annotations:
        description: "Prometheus encountered {{ $value }} template text expansion\
          \ failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus template text expansion failures (instance {{ $labels.instance
          }})
      expr: increase(prometheus_template_text_expansion_failures_total[3m]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusRuleEvaluationSlow
      annotations:
        description: "Prometheus rule evaluation took more time than the scheduled\
          \ interval. It indicates a slower storage backend access or too complex\
          \ query.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus rule evaluation slow (instance {{ $labels.instance }})
      expr: prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds
      for: 5m
      labels:
        severity: warning
    - alert: PrometheusNotificationsBacklog
      annotations:
        description: "The Prometheus notification queue has not been empty for 10\
          \ minutes\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus notifications backlog (instance {{ $labels.instance }})
      expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0
      for: 0m
      labels:
        severity: warning
    - alert: PrometheusAlertmanagerNotificationFailing
      annotations:
        description: "Alertmanager is failing sending notifications\n  VALUE = {{\
          \ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus AlertManager notification failing (instance {{ $labels.instance
          }})
      expr: rate(alertmanager_notifications_failed_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusTargetEmpty
      annotations:
        description: "Prometheus has no target in service discovery\n  VALUE = {{\
          \ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus target empty (instance {{ $labels.instance }})
      expr: prometheus_sd_discovered_targets == 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusLargeScrape
      annotations:
        description: "Prometheus has many scrapes that exceed the sample limit\n \
          \ VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus large scrape (instance {{ $labels.instance }})
      expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) >
        10
      for: 5m
      labels:
        severity: warning
    - alert: PrometheusTargetScrapeDuplicate
      annotations:
        description: "Prometheus has many samples rejected due to duplicate timestamps\
          \ but different values\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus target scrape duplicate (instance {{ $labels.instance
          }})
      expr: increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m])
        > 0
      for: 0m
      labels:
        severity: warning
    - alert: PrometheusTsdbCheckpointCreationFailures
      annotations:
        description: "Prometheus encountered {{ $value }} checkpoint creation failures\n\
          \  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus TSDB checkpoint creation failures (instance {{ $labels.instance
          }})
      expr: increase(prometheus_tsdb_checkpoint_creations_failed_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusTsdbCheckpointDeletionFailures
      annotations:
        description: "Prometheus encountered {{ $value }} checkpoint deletion failures\n\
          \  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus TSDB checkpoint deletion failures (instance {{ $labels.instance
          }})
      expr: increase(prometheus_tsdb_checkpoint_deletions_failed_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusTsdbCompactionsFailed
      annotations:
        description: "Prometheus encountered {{ $value }} TSDB compactions failures\n\
          \  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus TSDB compactions failed (instance {{ $labels.instance
          }})
      expr: increase(prometheus_tsdb_compactions_failed_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusTsdbHeadTruncationsFailed
      annotations:
        description: "Prometheus encountered {{ $value }} TSDB head truncation failures\n\
          \  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus TSDB head truncations failed (instance {{ $labels.instance
          }})
      expr: increase(prometheus_tsdb_head_truncations_failed_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusTsdbReloadFailures
      annotations:
        description: "Prometheus encountered {{ $value }} TSDB reload failures\n \
          \ VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus TSDB reload failures (instance {{ $labels.instance }})
      expr: increase(prometheus_tsdb_reloads_failures_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusTsdbWalCorruptions
      annotations:
        description: "Prometheus encountered {{ $value }} TSDB WAL corruptions\n \
          \ VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus TSDB WAL is corrupt, make sure there is enough disk space
          and wipe /data/wal
      expr: increase(prometheus_tsdb_wal_corruptions_total[2h]) > 0
      for: 0m
      labels:
        severity: critical
    - alert: PrometheusTsdbWalTruncationsFailed
      annotations:
        description: "Prometheus encountered {{ $value }} TSDB WAL truncation failures\n\
          \  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
        summary: Prometheus TSDB WAL truncations failed (instance {{ $labels.instance
          }})
      expr: increase(prometheus_tsdb_wal_truncations_failed_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: prometheus
  annotations:
    cert-manager.io/cluster-issuer: default
    traefik.ingress.kubernetes.io/router.entrypoints: websecure
    traefik.ingress.kubernetes.io/router.tls: "true"
    external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
    traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd
spec:
  rules:
  - host: prom.k-space.ee
    http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: prometheus-operated
            port:
              number: 9090
  tls:
  - hosts:
    - prom.k-space.ee
    secretName: prom-tls
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: alertmanager
  annotations:
    cert-manager.io/cluster-issuer: default
    traefik.ingress.kubernetes.io/router.entrypoints: websecure
    traefik.ingress.kubernetes.io/router.tls: "true"
    external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
    traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd
spec:
  rules:
  - host: am.k-space.ee
    http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: alertmanager-operated
            port:
              number: 9093
  tls:
  - hosts:
    - am.k-space.ee
    secretName: alertmanager-tls
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: prometheus
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus
  podMetricsEndpoints:
    - port: web
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: alertmanager
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: alertmanager
  podMetricsEndpoints:
    - port: web
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: operator
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus-operator
  podMetricsEndpoints:
    - port: http
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: kubelet
spec:
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    honorLabels: true
    interval: 30s
    port: https-metrics
    scheme: https
    tlsConfig:
      insecureSkipVerify: true
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    honorLabels: true
    interval: 30s
    path: /metrics/cadvisor
    port: https-metrics
    scheme: https
    tlsConfig:
      insecureSkipVerify: true
  namespaceSelector:
    matchNames:
    - kube-system
  selector:
    matchLabels:
      app.kubernetes.io/name: kubelet