Improve KubernetesJobFailed alert

kube-system/kube-state-metrics.yaml

@@ -279,13 +279,33 @@ spec:
             summary: Kubernetes container oom killer (instance {{ $labels.instance }})
             description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
         - alert: KubernetesJobFailed
-          expr: kube_job_status_failed > 0
+          expr: |
+            (
+              kube_job_status_failed > 0
+              unless on (namespace, job_name)
+              kube_job_owner{owner_kind="CronJob", owner_is_controller="true"}
+            )
+            or
+            (
+              kube_job_status_failed > 0
+              and on (namespace, job_name)
+              (
+                kube_job_status_start_time
+                * on (namespace, job_name) group_left (owner_name)
+                kube_job_owner{owner_kind="CronJob", owner_is_controller="true"}
+                > on (namespace, owner_name) group_left ()
+                label_replace(
+                  kube_cronjob_status_last_successful_time,
+                  "owner_name", "$1", "cronjob", "(.*)"
+                )
+              )
+            )
           for: 0m
           labels:
             severity: warning
           annotations:
             summary: Kubernetes Job failed (instance {{ $labels.instance }})
-            description: "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+            description: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
         - alert: KubernetesCronjobSuspended
           expr: kube_cronjob_spec_suspend != 0
           for: 0m