forked from k-space/kube
Move Prometheus instance to monitoring namespace
This commit is contained in:
parent
62661efc42
commit
6e2f353916
28
monitoring/README.md
Normal file
28
monitoring/README.md
Normal file
@ -0,0 +1,28 @@
|
||||
## Monitoring
|
||||
|
||||
This namespace is managed by
|
||||
[ArgoCD](https://argocd.k-space.ee/applications/argocd/monitoring)
|
||||
|
||||
To reconfigure SNMP targets etc:
|
||||
|
||||
```
|
||||
kubectl delete -n monitoring configmap snmp-exporter
|
||||
kubectl create -n monitoring configmap snmp-exporter --from-file=snmp.yml=snmp-configs.yaml
|
||||
```
|
||||
|
||||
To set Slack secrets:
|
||||
|
||||
```
|
||||
kubectl create -n monitoring secret generic slack-secrets \
|
||||
--from-literal=webhook-url=https://hooks.slack.com/services/...
|
||||
```
|
||||
|
||||
To set Mikrotik secrets:
|
||||
|
||||
```
|
||||
kubectl create -n monitoring secret generic mikrotik-exporter \
|
||||
--from-literal=MIKROTIK_PASSWORD='f7W!H*Pu' \
|
||||
--from-literal=PROMETHEUS_BEARER_TOKEN=$(cat /dev/urandom | base64 | head -c 30)
|
||||
```
|
||||
|
||||
|
62
monitoring/alertmanager.yaml
Normal file
62
monitoring/alertmanager.yaml
Normal file
@ -0,0 +1,62 @@
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1alpha1
|
||||
kind: AlertmanagerConfig
|
||||
metadata:
|
||||
name: alertmanager
|
||||
labels:
|
||||
app.kubernetes.io/name: alertmanager
|
||||
spec:
|
||||
route:
|
||||
routes:
|
||||
- continue: false
|
||||
receiver: slack-notifications
|
||||
matchers:
|
||||
- matchType: "="
|
||||
name: severity
|
||||
value: critical
|
||||
receiver: 'null'
|
||||
receivers:
|
||||
- name: 'null'
|
||||
- name: 'slack-notifications'
|
||||
slackConfigs:
|
||||
- channel: '#kube-prod'
|
||||
sendResolved: true
|
||||
apiURL:
|
||||
name: slack-secrets
|
||||
key: webhook-url
|
||||
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Alertmanager
|
||||
metadata:
|
||||
name: alertmanager
|
||||
spec:
|
||||
alertmanagerConfigMatcherStrategy:
|
||||
type: None
|
||||
alertmanagerConfigNamespaceSelector: {}
|
||||
alertmanagerConfigSelector: {}
|
||||
alertmanagerConfiguration:
|
||||
name: alertmanager
|
||||
secrets:
|
||||
- slack-secrets
|
||||
nodeSelector:
|
||||
dedicated: monitoring
|
||||
tolerations:
|
||||
- key: dedicated
|
||||
operator: Equal
|
||||
value: monitoring
|
||||
effect: NoSchedule
|
||||
replicas: 3
|
||||
serviceAccountName: alertmanager
|
||||
externalUrl: http://am.k-space.ee/
|
||||
routePrefix: "/"
|
||||
securityContext:
|
||||
fsGroup: 2000
|
||||
runAsGroup: 2000
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: alertmanager
|
@ -156,7 +156,7 @@ metadata:
|
||||
name: blackbox-exporter
|
||||
spec:
|
||||
revisionHistoryLimit: 0
|
||||
replicas: 3
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: blackbox-exporter
|
@ -1,30 +1,4 @@
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1alpha1
|
||||
kind: AlertmanagerConfig
|
||||
metadata:
|
||||
name: alertmanager
|
||||
labels:
|
||||
app.kubernetes.io/name: alertmanager
|
||||
spec:
|
||||
route:
|
||||
routes:
|
||||
- continue: false
|
||||
receiver: slack-notifications
|
||||
matchers:
|
||||
- matchType: "="
|
||||
name: severity
|
||||
value: critical
|
||||
receiver: 'null'
|
||||
receivers:
|
||||
- name: 'null'
|
||||
- name: 'slack-notifications'
|
||||
slackConfigs:
|
||||
- channel: '#kube-prod'
|
||||
sendResolved: true
|
||||
apiURL:
|
||||
name: slack-secrets
|
||||
key: webhook-url
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
@ -37,41 +11,6 @@ spec:
|
||||
- port: metrics
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Alertmanager
|
||||
metadata:
|
||||
name: alertmanager
|
||||
spec:
|
||||
alertmanagerConfigMatcherStrategy:
|
||||
type: None
|
||||
alertmanagerConfigNamespaceSelector: {}
|
||||
alertmanagerConfigSelector: {}
|
||||
alertmanagerConfiguration:
|
||||
name: alertmanager
|
||||
secrets:
|
||||
- slack-secrets
|
||||
nodeSelector:
|
||||
dedicated: monitoring
|
||||
tolerations:
|
||||
- key: dedicated
|
||||
operator: Equal
|
||||
value: monitoring
|
||||
effect: NoSchedule
|
||||
replicas: 3
|
||||
serviceAccountName: alertmanager
|
||||
externalUrl: http://am.k-space.ee/
|
||||
routePrefix: "/"
|
||||
securityContext:
|
||||
fsGroup: 2000
|
||||
runAsGroup: 2000
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: alertmanager
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Prometheus
|
||||
metadata:
|
||||
name: prometheus
|
||||
@ -172,7 +111,7 @@ spec:
|
||||
description: "A Prometheus job has disappeared\n VALUE = {{ $value }}\n \
|
||||
\ LABELS = {{ $labels }}"
|
||||
summary: Prometheus job missing (instance {{ $labels.instance }})
|
||||
expr: absent(up{job="prometheus-operator/prometheus"})
|
||||
expr: absent(up{job="monitoring/prometheus"})
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
@ -221,7 +160,7 @@ spec:
|
||||
\ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Prometheus AlertManager job missing (instance {{ $labels.instance
|
||||
}})
|
||||
expr: absent(up{job="prometheus-operator/alertmanager"})
|
||||
expr: absent(up{job="monitoring/alertmanager"})
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
@ -413,7 +352,7 @@ metadata:
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
traefik.ingress.kubernetes.io/router.tls: "true"
|
||||
external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
|
||||
traefik.ingress.kubernetes.io/router.middlewares: prometheus-operator-prometheus@kubernetescrd
|
||||
traefik.ingress.kubernetes.io/router.middlewares: monitoring-prometheus@kubernetescrd
|
||||
spec:
|
||||
rules:
|
||||
- host: prom.k-space.ee
|
||||
@ -438,7 +377,7 @@ metadata:
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
traefik.ingress.kubernetes.io/router.tls: "true"
|
||||
external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
|
||||
traefik.ingress.kubernetes.io/router.middlewares: prometheus-operator-alertmanager@kubernetescrd
|
||||
traefik.ingress.kubernetes.io/router.middlewares: monitoring-alertmanager@kubernetescrd
|
||||
spec:
|
||||
rules:
|
||||
- host: am.k-space.ee
|
@ -1,28 +1,11 @@
|
||||
# Prometheus operator
|
||||
|
||||
To deploy Prometheus operator:
|
||||
|
||||
```
|
||||
curl -L https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.61.1/bundle.yaml | sed -e 's/namespace: default/namespace: prometheus-operator/g' > bundle.yml
|
||||
kubectl create namespace prometheus-operator
|
||||
kubectl apply --server-side -n prometheus-operator -f bundle.yml
|
||||
kubectl delete -n prometheus-operator configmap snmp-exporter
|
||||
kubectl create -n prometheus-operator configmap snmp-exporter --from-file=snmp.yml
|
||||
kubectl apply -n prometheus-operator -f application.yml -f node-exporter.yml -f blackbox-exporter.yml -f snmp-exporter.yml -f mikrotik-exporter.yml
|
||||
```
|
||||
|
||||
|
||||
# Slack
|
||||
|
||||
```
|
||||
kubectl create -n prometheus-operator secret generic slack-secrets \
|
||||
--from-literal=webhook-url=https://hooks.slack.com/services/...
|
||||
```
|
||||
|
||||
|
||||
# Mikrotik exporter
|
||||
|
||||
```
|
||||
kubectl create -n prometheus-operator secret generic mikrotik-exporter \
|
||||
--from-literal=MIKROTIK_PASSWORD='f7W!H*Pu' \
|
||||
--from-literal=PROMETHEUS_BEARER_TOKEN=$(cat /dev/urandom | base64 | head -c 30)
|
||||
```
|
||||
|
||||
Note: Do not put any Prometheus instances or exporters in this namespace, instead have them in `monitoring` namespace
|
||||
|
Loading…
Reference in New Issue
Block a user