k-space
/
kube
10
1
Fork 1
Code Issues 17 Pull Requests Projects Activity

camtiler: Dozen updates

This commit is contained in:
Lauri Võsandi 2022-12-12 20:37:03 +02:00
parent b5ee891c97
commit ea9b63b7cc
4 changed files with 317 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
camtiler/README.md
View File

@ -1,7 +1,7 @@
To apply changes:
```
kubectl apply -n camtiler -f application.yml -f persistence.yml -f mongoexpress.yml -f mongodb-support.yml -f networkpolicy-base.yml -f minio-support.yml
kubectl apply -n camtiler -f application.yml -f persistence.yml -f mongoexpress.yml -f mongodb-support.yml -f networkpolicy-base.yml
```
To deploy changes:
@ -15,14 +15,9 @@ To initialize secrets:
```
kubectl create secret generic -n camtiler mongodb-application-readwrite-password --from-literal="password=$(cat /dev/urandom | base64 | head -c 30)"
kubectl create secret generic -n camtiler mongodb-application-readonly-password --from-literal="password=$(cat /dev/urandom | base64 | head -c 30)"
kubectl create secret generic -n camtiler minio-secret \
--from-literal=accesskey=application \
--from-literal=secretkey=$(cat /dev/urandom | base64 | head -c 30)
kubectl create secret generic -n camtiler minio-env-configuration \
--from-literal="MINIO_BROWSER=off" \
kubectl create secret generic -n camtiler minio-secrets \
--from-literal="MINIO_ROOT_USER=root" \
--from-literal="MINIO_ROOT_PASSWORD=$(cat /dev/urandom | base64 | head -c 30)" \
--from-literal="MINIO_STORAGE_CLASS_STANDARD=EC:4"
--from-literal="MINIO_ROOT_PASSWORD=$(cat /dev/urandom | base64 | head -c 30)"
kubectl -n camtiler create secret generic camera-secrets \
--from-literal=username=... \
--from-literal=password=...

191
camtiler/application.yml
View File

@ -2,7 +2,7 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: camtiler
name: camera-tiler
annotations:
keel.sh/policy: force
keel.sh/trigger: poll
@ -10,17 +10,16 @@ spec:
revisionHistoryLimit: 0
replicas: 2
selector:
matchLabels:
matchLabels: &selectorLabels
app.kubernetes.io/name: camtiler
component: camera-tiler
template:
metadata:
labels:
app.kubernetes.io/name: camtiler
component: camtiler
labels: *selectorLabels
spec:
serviceAccountName: camtiler
serviceAccountName: camera-tiler
containers:
- name: camtiler
- name: camera-tiler
image: harbor.k-space.ee/k-space/camera-tiler:latest
securityContext:
readOnlyRootFilesystem: true
@ -29,6 +28,13 @@ spec:
ports:
- containerPort: 5001
name: "http"
resources:
requests:
memory: "200Mi"
cpu: "100m"
limits:
memory: "500Mi"
cpu: "1"
---
apiVersion: apps/v1
kind: Deployment
@ -41,20 +47,40 @@ spec:
revisionHistoryLimit: 0
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: log-viewer-frontend
matchLabels: &selectorLabels
app.kubernetes.io/name: camtiler
component: log-viewer-frontend
template:
metadata:
labels:
app.kubernetes.io/name: log-viewer-frontend
labels: *selectorLabels
spec:
containers:
- name: log-viewer-frontend
image: harbor.k-space.ee/k-space/log-viewer-frontend:latest
# securityContext:
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
securityContext:
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
resources:
limits:
memory: 50Mi
requests:
cpu: 1m
memory: 20Mi
volumeMounts:
- name : nginx-cache
mountPath: /var/cache/nginx/
- name : nginx-config
mountPath: /var/config/nginx/
- name: var-run
mountPath: /var/run/
volumes:
- emptyDir: {}
name: nginx-cache
- emptyDir: {}
name: nginx-config
- emptyDir: {}
name: var-run
---
apiVersion: apps/v1
kind: Deployment
@ -65,14 +91,14 @@ metadata:
keel.sh/trigger: poll
spec:
revisionHistoryLimit: 0
replicas: 3
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: log-viewer-backend
matchLabels: &selectorLabels
app.kubernetes.io/name: camtiler
component: log-viewer-backend
template:
metadata:
labels:
app.kubernetes.io/name: log-viewer-backend
labels: *selectorLabels
spec:
containers:
- name: log-backend-backend
@ -98,13 +124,13 @@ spec:
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: secretkey
name: minio-secrets
key: MINIO_ROOT_PASSWORD
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: accesskey
name: minio-secrets
key: MINIO_ROOT_USER
---
apiVersion: v1
kind: Service
@ -113,7 +139,8 @@ metadata:
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: log-viewer-frontend
app.kubernetes.io/name: camtiler
component: log-viewer-frontend
ports:
- protocol: TCP
port: 3003
@ -125,7 +152,8 @@ metadata:
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: log-viewer-backend
app.kubernetes.io/name: camtiler
component: log-viewer-backend
ports:
- protocol: TCP
port: 3002
@ -133,14 +161,15 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: camtiler
name: camera-tiler
labels:
component: camtiler
app.kubernetes.io/name: camtiler
component: camera-tiler
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: camtiler
component: camtiler
component: camera-tiler
ports:
- protocol: TCP
port: 5001
@ -148,12 +177,12 @@ spec:
apiVersion: v1
kind: ServiceAccount
metadata:
name: camtiler
name: camera-tiler
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: camtiler
name: camera-tiler
rules:
- apiGroups:
- ""
@ -165,14 +194,14 @@ rules:
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: camtiler
name: camera-tiler
subjects:
- kind: ServiceAccount
name: camtiler
name: camera-tiler
apiGroup: ""
roleRef:
kind: Role
name: camtiler
name: camera-tiler
apiGroup: ""
---
apiVersion: networking.k8s.io/v1
@ -209,7 +238,7 @@ spec:
path: "/tiled"
backend:
service:
name: camtiler
name: camera-tiler
port:
number: 5001
- pathType: Prefix
@ -237,7 +266,7 @@ metadata:
spec:
podSelector:
matchLabels:
component: camdetect
component: camera-motion-detect
policyTypes:
- Ingress
- Egress
@ -245,7 +274,8 @@ spec:
- from:
- podSelector:
matchLabels:
component: camtiler
app.kubernetes.io/name: camtiler
component: camera-tiler
- from:
- namespaceSelector:
matchLabels:
@ -267,7 +297,7 @@ spec:
- to:
- podSelector:
matchLabels:
v1.min.io/tenant: minio
app.kubernetes.io/name: minio
ports:
- port: 9000
---
@ -278,7 +308,8 @@ metadata:
spec:
podSelector:
matchLabels:
component: camtiler
app.kubernetes.io/name: camtiler
component: camera-tiler
policyTypes:
- Ingress
- Egress
@ -286,7 +317,7 @@ spec:
- to:
- podSelector:
matchLabels:
component: camdetect
component: camera-motion-detect
ports:
- port: 5000
ingress:
@ -312,7 +343,8 @@ metadata:
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: log-viewer-backend
app.kubernetes.io/name: camtiler
component: log-viewer-backend
policyTypes:
- Ingress
- Egress
@ -345,7 +377,8 @@ metadata:
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: log-viewer-frontend
app.kubernetes.io/name: camtiler
component: log-viewer-frontend
policyTypes:
- Ingress
- Egress
@ -358,31 +391,6 @@ spec:
matchLabels:
app.kubernetes.io/name: traefik
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: minio
annotations:
kubernetes.io/ingress.class: traefik
traefik.ingress.kubernetes.io/router.entrypoints: websecure
traefik.ingress.kubernetes.io/router.tls: "true"
external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
spec:
rules:
- host: cams-s3.k-space.ee
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: minio
port:
number: 80
tls:
- hosts:
- "*.k-space.ee"
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
@ -473,12 +481,13 @@ spec:
metadata:
name: foobar
labels:
component: camdetect
app.kubernetes.io/name: foobar
component: camera-motion-detect
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: foobar
component: camdetect
component: camera-motion-detect
ports:
- protocol: TCP
port: 80
@ -508,18 +517,25 @@ spec:
metadata:
labels:
app.kubernetes.io/name: foobar
component: camdetect
component: camera-motion-detect
spec:
containers:
- name: camdetect
- name: camera-motion-detect
image: harbor.k-space.ee/k-space/camera-motion-detect:latest
starupProbe:
httpGet:
path: /healthz
port: 5000
initialDelaySeconds: 2
periodSeconds: 180
timeoutSeconds: 60
readinessProbe:
httpGet:
path: /readyz
port: 5000
initialDelaySeconds: 10
periodSeconds: 180
timeoutSeconds: 60
initialDelaySeconds: 60
periodSeconds: 60
timeoutSeconds: 5
ports:
- containerPort: 5000
name: "http"
@ -557,13 +573,13 @@ spec:
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: secretkey
name: minio-secrets
key: MINIO_ROOT_PASSWORD
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: minio-secret
key: accesskey
name: minio-secrets
key: MINIO_ROOT_USER
# Make sure 2+ pods of same camera are scheduled on different hosts
affinity:
@ -585,7 +601,7 @@ spec:
labelSelector:
matchLabels:
app.kubernetes.io/name: foobar
component: camdetect
component: camera-motion-detect
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
@ -597,6 +613,7 @@ spec:
- port: http
podTargetLabels:
- app.kubernetes.io/name
- component
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
@ -607,21 +624,21 @@ spec:
- name: cameras
rules:
- alert: CameraLost
expr: rate(camdetect_rx_frames_total[2m]) < 1
expr: rate(camtiler_frames_total{stage="downloaded"}[1m]) < 1
for: 2m
labels:
severity: warning
annotations:
summary: Camera feed stopped
- alert: CameraServerRoomMotion
expr: camdetect_event_active {app="camdetect-server-room"} > 0
expr: rate(camtiler_events_total{app_kubernetes_io_name="server-room"}[30m]) > 0
for: 1m
labels:
severity: warning
annotations:
summary: Motion was detected in server room
- alert: CameraSlowUploads
expr: rate(camdetect_upload_dropped_frames_total[2m]) > 1
expr: camtiler_queue_frames{stage="upload"} > 10
for: 5m
labels:
severity: warning
@ -629,7 +646,7 @@ spec:
summary: Motion detect snapshots are piling up and
not getting uploaded to S3
- alert: CameraSlowProcessing
expr: rate(camdetect_download_dropped_frames_total[2m]) > 1
expr: camtiler_queue_frames{stage="download"} > 10
for: 5m
labels:
severity: warning
@ -644,6 +661,7 @@ metadata:
spec:
target: http://user@workshop.cam.k-space.ee:8080/?action=stream
secretRef: camera-secrets
replicas: 2
---
apiVersion: k-space.ee/v1alpha1
kind: Camera
@ -652,6 +670,7 @@ metadata:
spec:
target: http://user@server-room.cam.k-space.ee:8080/?action=stream
secretRef: camera-secrets
replicas: 2
---
apiVersion: k-space.ee/v1alpha1
kind: Camera
@ -660,6 +679,7 @@ metadata:
spec:
target: http://user@printer.cam.k-space.ee:8080/?action=stream
secretRef: camera-secrets
replicas: 2
---
apiVersion: k-space.ee/v1alpha1
kind: Camera
@ -668,6 +688,7 @@ metadata:
spec:
target: http://user@chaos.cam.k-space.ee:8080/?action=stream
secretRef: camera-secrets
replicas: 2
---
apiVersion: k-space.ee/v1alpha1
kind: Camera
@ -676,6 +697,7 @@ metadata:
spec:
target: http://user@cyber.cam.k-space.ee:8080/?action=stream
secretRef: camera-secrets
replicas: 2
---
apiVersion: k-space.ee/v1alpha1
kind: Camera
@ -684,6 +706,7 @@ metadata:
spec:
target: http://user@kitchen.cam.k-space.ee:8080/?action=stream
secretRef: camera-secrets
replicas: 2
---
apiVersion: k-space.ee/v1alpha1
kind: Camera
@ -692,6 +715,7 @@ metadata:
spec:
target: http://user@back-door.cam.k-space.ee:8080/?action=stream
secretRef: camera-secrets
replicas: 2
---
apiVersion: k-space.ee/v1alpha1
kind: Camera
@ -700,3 +724,4 @@ metadata:
spec:
target: http://user@ground-door.cam.k-space.ee:8080/?action=stream
secretRef: camera-secrets
replicas: 2

1
camtiler/minio-support.yml
View File

@ -1 +0,0 @@
../shared/minio-support.yml

248
camtiler/persistence.yml
View File

@ -7,9 +7,10 @@ spec:
additionalMongodConfig:
systemLog:
quiet: true
members: 3
members: 2
arbiters: 1
type: ReplicaSet
version: "5.0.9"
version: "6.0.3"
security:
authentication:
modes: ["SCRAM"]
@ -27,7 +28,7 @@ spec:
passwordSecretRef:
name: mongodb-application-readonly-password
roles:
- name: readOnly
- name: read
db: application
scramCredentialsSecretName: mongodb-application-readonly
statefulSet:
@ -35,6 +36,24 @@ spec:
logLevel: WARN
template:
spec:
containers:
- name: mongod
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
volumeMounts:
- name: journal-volume
mountPath: /data/journal
- name: mongodb-agent
resources:
requests:
cpu: 1m
memory: 100Mi
limits: {}
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
@ -55,8 +74,21 @@ spec:
volumeClaimTemplates:
- metadata:
name: logs-volume
labels:
usecase: logs
spec:
storageClassName: local-path
storageClassName: mongo
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 100Mi
- metadata:
name: journal-volume
labels:
usecase: journal
spec:
storageClassName: mongo
accessModes:
- ReadWriteOnce
resources:
@ -64,63 +96,44 @@ spec:
storage: 512Mi
- metadata:
name: data-volume
labels:
usecase: data
spec:
storageClassName: local-path
storageClassName: mongo
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
---
apiVersion: minio.min.io/v2
kind: Tenant
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: minio
annotations:
prometheus.io/path: /minio/prometheus/metrics
prometheus.io/port: "9000"
prometheus.io/scrape: "true"
labels:
app.kubernetes.io/name: minio
spec:
credsSecret:
name: minio-secret
buckets:
- name: application
requestAutoCert: false
users:
- name: minio-user-0
pools:
- name: pool-0
selector:
matchLabels:
app.kubernetes.io/name: minio
serviceName: minio-svc
replicas: 4
podManagementPolicy: Parallel
template:
metadata:
labels:
app.kubernetes.io/name: minio
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: v1.min.io/tenant
- key: app
operator: In
values:
- minio
- key: v1.min.io/pool
operator: In
values:
- pool-0
topologyKey: kubernetes.io/hostname
resources:
requests:
cpu: '1'
memory: 512Mi
servers: 4
volumesPerServer: 1
volumeClaimTemplate:
metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: '30Gi'
storageClassName: local-path
status: {}
nodeSelector:
dedicated: storage
tolerations:
@ -128,3 +141,154 @@ spec:
operator: Equal
value: storage
effect: NoSchedule
containers:
- name: minio
env:
- name: MINIO_PROMETHEUS_AUTH_TYPE
value: public
envFrom:
- secretRef:
name: minio-secrets
image: minio/minio:latest
args:
- server
- http://minio-{0...3}.minio-svc.camtiler.svc.cluster.local/data
- --address
- 0.0.0.0:9000
- --console-address
- 0.0.0.0:9001
ports:
- containerPort: 9000
name: http
- containerPort: 9001
name: console
livenessProbe:
httpGet:
path: /minio/health/live
port: 9000
initialDelaySeconds: 10
periodSeconds: 20
resources:
requests:
cpu: 1m
memory: 512Mi
limits:
cpu: 1000m
memory: 1Gi
volumeMounts:
- name: minio-data
mountPath: /data
volumeClaimTemplates:
- metadata:
name: minio-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: '30Gi'
storageClassName: minio
---
apiVersion: v1
kind: Service
metadata:
name: minio
spec:
type: ClusterIP
ports:
- port: 80
targetPort: 9000
protocol: TCP
name: http
selector:
app.kubernetes.io/name: minio
---
kind: Service
apiVersion: v1
metadata:
name: minio-svc
spec:
selector:
app.kubernetes.io/name: minio
clusterIP: None
publishNotReadyAddresses: true
ports:
- name: http
port: 9000
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: minio
spec:
selector:
matchLabels:
app.kubernetes.io/name: minio
podMetricsEndpoints:
- port: http
path: /minio/v2/metrics/node
podTargetLabels:
- app.kubernetes.io/name
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: minio
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: minio
policyTypes:
- Ingress
- Egress
egress:
- ports:
- port: http
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ingress:
- ports:
- port: http
from:
- podSelector: {}
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: traefik
podSelector:
matchLabels:
app.kubernetes.io/name: traefik
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: prometheus-operator
podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: minio
annotations:
kubernetes.io/ingress.class: traefik
traefik.ingress.kubernetes.io/router.entrypoints: websecure
traefik.ingress.kubernetes.io/router.tls: "true"
external-dns.alpha.kubernetes.io/target: traefik.k-space.ee
spec:
rules:
- host: cams-s3.k-space.ee
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: minio-svc
port:
number: 9000
tls:
- hosts:
- "*.k-space.ee"