diff --git a/kube-system/kube-state-metrics.yaml b/kube-system/kube-state-metrics.yaml index 678dee0..be5232a 100644 --- a/kube-system/kube-state-metrics.yaml +++ b/kube-system/kube-state-metrics.yaml @@ -279,13 +279,33 @@ spec: summary: Kubernetes container oom killer (instance {{ $labels.instance }}) description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - alert: KubernetesJobFailed - expr: kube_job_status_failed > 0 + expr: | + ( + kube_job_status_failed > 0 + unless on (namespace, job_name) + kube_job_owner{owner_kind="CronJob", owner_is_controller="true"} + ) + or + ( + kube_job_status_failed > 0 + and on (namespace, job_name) + ( + kube_job_status_start_time + * on (namespace, job_name) group_left (owner_name) + kube_job_owner{owner_kind="CronJob", owner_is_controller="true"} + > on (namespace, owner_name) group_left () + label_replace( + kube_cronjob_status_last_successful_time, + "owner_name", "$1", "cronjob", "(.*)" + ) + ) + ) for: 0m labels: severity: warning annotations: summary: Kubernetes Job failed (instance {{ $labels.instance }}) - description: "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + description: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - alert: KubernetesCronjobSuspended expr: kube_cronjob_spec_suspend != 0 for: 0m