diff --git a/camtiler/README.md b/camtiler/README.md index bac2239..d99c6ed 100644 --- a/camtiler/README.md +++ b/camtiler/README.md @@ -1,7 +1,7 @@ To apply changes: ``` -kubectl apply -n camtiler -f application.yml -f persistence.yml -f mongoexpress.yml -f mongodb-support.yml -f networkpolicy-base.yml -f minio-support.yml +kubectl apply -n camtiler -f application.yml -f persistence.yml -f mongoexpress.yml -f mongodb-support.yml -f networkpolicy-base.yml ``` To deploy changes: @@ -15,14 +15,9 @@ To initialize secrets: ``` kubectl create secret generic -n camtiler mongodb-application-readwrite-password --from-literal="password=$(cat /dev/urandom | base64 | head -c 30)" kubectl create secret generic -n camtiler mongodb-application-readonly-password --from-literal="password=$(cat /dev/urandom | base64 | head -c 30)" -kubectl create secret generic -n camtiler minio-secret \ - --from-literal=accesskey=application \ - --from-literal=secretkey=$(cat /dev/urandom | base64 | head -c 30) -kubectl create secret generic -n camtiler minio-env-configuration \ - --from-literal="MINIO_BROWSER=off" \ +kubectl create secret generic -n camtiler minio-secrets \ --from-literal="MINIO_ROOT_USER=root" \ - --from-literal="MINIO_ROOT_PASSWORD=$(cat /dev/urandom | base64 | head -c 30)" \ - --from-literal="MINIO_STORAGE_CLASS_STANDARD=EC:4" + --from-literal="MINIO_ROOT_PASSWORD=$(cat /dev/urandom | base64 | head -c 30)" kubectl -n camtiler create secret generic camera-secrets \ --from-literal=username=... \ --from-literal=password=... diff --git a/camtiler/application.yml b/camtiler/application.yml index c506cd0..aab6e34 100644 --- a/camtiler/application.yml +++ b/camtiler/application.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: camtiler + name: camera-tiler annotations: keel.sh/policy: force keel.sh/trigger: poll @@ -10,17 +10,16 @@ spec: revisionHistoryLimit: 0 replicas: 2 selector: - matchLabels: + matchLabels: &selectorLabels app.kubernetes.io/name: camtiler + component: camera-tiler template: metadata: - labels: - app.kubernetes.io/name: camtiler - component: camtiler + labels: *selectorLabels spec: - serviceAccountName: camtiler + serviceAccountName: camera-tiler containers: - - name: camtiler + - name: camera-tiler image: harbor.k-space.ee/k-space/camera-tiler:latest securityContext: readOnlyRootFilesystem: true @@ -29,6 +28,13 @@ spec: ports: - containerPort: 5001 name: "http" + resources: + requests: + memory: "200Mi" + cpu: "100m" + limits: + memory: "500Mi" + cpu: "1" --- apiVersion: apps/v1 kind: Deployment @@ -41,20 +47,40 @@ spec: revisionHistoryLimit: 0 replicas: 2 selector: - matchLabels: - app.kubernetes.io/name: log-viewer-frontend + matchLabels: &selectorLabels + app.kubernetes.io/name: camtiler + component: log-viewer-frontend template: metadata: - labels: - app.kubernetes.io/name: log-viewer-frontend + labels: *selectorLabels spec: containers: - name: log-viewer-frontend image: harbor.k-space.ee/k-space/log-viewer-frontend:latest -# securityContext: -# readOnlyRootFilesystem: true -# runAsNonRoot: true -# runAsUser: 1000 + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + resources: + limits: + memory: 50Mi + requests: + cpu: 1m + memory: 20Mi + volumeMounts: + - name : nginx-cache + mountPath: /var/cache/nginx/ + - name : nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run --- apiVersion: apps/v1 kind: Deployment @@ -65,14 +91,14 @@ metadata: keel.sh/trigger: poll spec: revisionHistoryLimit: 0 - replicas: 3 + replicas: 2 selector: - matchLabels: - app.kubernetes.io/name: log-viewer-backend + matchLabels: &selectorLabels + app.kubernetes.io/name: camtiler + component: log-viewer-backend template: metadata: - labels: - app.kubernetes.io/name: log-viewer-backend + labels: *selectorLabels spec: containers: - name: log-backend-backend @@ -98,13 +124,13 @@ spec: - name: MINIO_SECRET_KEY valueFrom: secretKeyRef: - name: minio-secret - key: secretkey + name: minio-secrets + key: MINIO_ROOT_PASSWORD - name: MINIO_ACCESS_KEY valueFrom: secretKeyRef: - name: minio-secret - key: accesskey + name: minio-secrets + key: MINIO_ROOT_USER --- apiVersion: v1 kind: Service @@ -113,7 +139,8 @@ metadata: spec: type: ClusterIP selector: - app.kubernetes.io/name: log-viewer-frontend + app.kubernetes.io/name: camtiler + component: log-viewer-frontend ports: - protocol: TCP port: 3003 @@ -125,7 +152,8 @@ metadata: spec: type: ClusterIP selector: - app.kubernetes.io/name: log-viewer-backend + app.kubernetes.io/name: camtiler + component: log-viewer-backend ports: - protocol: TCP port: 3002 @@ -133,14 +161,15 @@ spec: apiVersion: v1 kind: Service metadata: - name: camtiler + name: camera-tiler labels: - component: camtiler + app.kubernetes.io/name: camtiler + component: camera-tiler spec: type: ClusterIP selector: app.kubernetes.io/name: camtiler - component: camtiler + component: camera-tiler ports: - protocol: TCP port: 5001 @@ -148,12 +177,12 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: camtiler + name: camera-tiler --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: camtiler + name: camera-tiler rules: - apiGroups: - "" @@ -165,14 +194,14 @@ rules: kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: camtiler + name: camera-tiler subjects: - kind: ServiceAccount - name: camtiler + name: camera-tiler apiGroup: "" roleRef: kind: Role - name: camtiler + name: camera-tiler apiGroup: "" --- apiVersion: networking.k8s.io/v1 @@ -209,7 +238,7 @@ spec: path: "/tiled" backend: service: - name: camtiler + name: camera-tiler port: number: 5001 - pathType: Prefix @@ -237,7 +266,7 @@ metadata: spec: podSelector: matchLabels: - component: camdetect + component: camera-motion-detect policyTypes: - Ingress - Egress @@ -245,7 +274,8 @@ spec: - from: - podSelector: matchLabels: - component: camtiler + app.kubernetes.io/name: camtiler + component: camera-tiler - from: - namespaceSelector: matchLabels: @@ -267,7 +297,7 @@ spec: - to: - podSelector: matchLabels: - v1.min.io/tenant: minio + app.kubernetes.io/name: minio ports: - port: 9000 --- @@ -278,7 +308,8 @@ metadata: spec: podSelector: matchLabels: - component: camtiler + app.kubernetes.io/name: camtiler + component: camera-tiler policyTypes: - Ingress - Egress @@ -286,7 +317,7 @@ spec: - to: - podSelector: matchLabels: - component: camdetect + component: camera-motion-detect ports: - port: 5000 ingress: @@ -312,7 +343,8 @@ metadata: spec: podSelector: matchLabels: - app.kubernetes.io/name: log-viewer-backend + app.kubernetes.io/name: camtiler + component: log-viewer-backend policyTypes: - Ingress - Egress @@ -345,7 +377,8 @@ metadata: spec: podSelector: matchLabels: - app.kubernetes.io/name: log-viewer-frontend + app.kubernetes.io/name: camtiler + component: log-viewer-frontend policyTypes: - Ingress - Egress @@ -358,31 +391,6 @@ spec: matchLabels: app.kubernetes.io/name: traefik --- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: minio - annotations: - kubernetes.io/ingress.class: traefik - traefik.ingress.kubernetes.io/router.entrypoints: websecure - traefik.ingress.kubernetes.io/router.tls: "true" - external-dns.alpha.kubernetes.io/target: traefik.k-space.ee -spec: - rules: - - host: cams-s3.k-space.ee - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: minio - port: - number: 80 - tls: - - hosts: - - "*.k-space.ee" ---- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -473,12 +481,13 @@ spec: metadata: name: foobar labels: - component: camdetect + app.kubernetes.io/name: foobar + component: camera-motion-detect spec: type: ClusterIP selector: app.kubernetes.io/name: foobar - component: camdetect + component: camera-motion-detect ports: - protocol: TCP port: 80 @@ -508,18 +517,25 @@ spec: metadata: labels: app.kubernetes.io/name: foobar - component: camdetect + component: camera-motion-detect spec: containers: - - name: camdetect + - name: camera-motion-detect image: harbor.k-space.ee/k-space/camera-motion-detect:latest + starupProbe: + httpGet: + path: /healthz + port: 5000 + initialDelaySeconds: 2 + periodSeconds: 180 + timeoutSeconds: 60 readinessProbe: httpGet: path: /readyz port: 5000 - initialDelaySeconds: 10 - periodSeconds: 180 - timeoutSeconds: 60 + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 5 ports: - containerPort: 5000 name: "http" @@ -557,13 +573,13 @@ spec: - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: minio-secret - key: secretkey + name: minio-secrets + key: MINIO_ROOT_PASSWORD - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: minio-secret - key: accesskey + name: minio-secrets + key: MINIO_ROOT_USER # Make sure 2+ pods of same camera are scheduled on different hosts affinity: @@ -585,7 +601,7 @@ spec: labelSelector: matchLabels: app.kubernetes.io/name: foobar - component: camdetect + component: camera-motion-detect --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor @@ -597,6 +613,7 @@ spec: - port: http podTargetLabels: - app.kubernetes.io/name + - component --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -607,21 +624,21 @@ spec: - name: cameras rules: - alert: CameraLost - expr: rate(camdetect_rx_frames_total[2m]) < 1 + expr: rate(camtiler_frames_total{stage="downloaded"}[1m]) < 1 for: 2m labels: severity: warning annotations: summary: Camera feed stopped - alert: CameraServerRoomMotion - expr: camdetect_event_active {app="camdetect-server-room"} > 0 + expr: rate(camtiler_events_total{app_kubernetes_io_name="server-room"}[30m]) > 0 for: 1m labels: severity: warning annotations: summary: Motion was detected in server room - alert: CameraSlowUploads - expr: rate(camdetect_upload_dropped_frames_total[2m]) > 1 + expr: camtiler_queue_frames{stage="upload"} > 10 for: 5m labels: severity: warning @@ -629,7 +646,7 @@ spec: summary: Motion detect snapshots are piling up and not getting uploaded to S3 - alert: CameraSlowProcessing - expr: rate(camdetect_download_dropped_frames_total[2m]) > 1 + expr: camtiler_queue_frames{stage="download"} > 10 for: 5m labels: severity: warning @@ -644,6 +661,7 @@ metadata: spec: target: http://user@workshop.cam.k-space.ee:8080/?action=stream secretRef: camera-secrets + replicas: 2 --- apiVersion: k-space.ee/v1alpha1 kind: Camera @@ -652,6 +670,7 @@ metadata: spec: target: http://user@server-room.cam.k-space.ee:8080/?action=stream secretRef: camera-secrets + replicas: 2 --- apiVersion: k-space.ee/v1alpha1 kind: Camera @@ -660,6 +679,7 @@ metadata: spec: target: http://user@printer.cam.k-space.ee:8080/?action=stream secretRef: camera-secrets + replicas: 2 --- apiVersion: k-space.ee/v1alpha1 kind: Camera @@ -668,6 +688,7 @@ metadata: spec: target: http://user@chaos.cam.k-space.ee:8080/?action=stream secretRef: camera-secrets + replicas: 2 --- apiVersion: k-space.ee/v1alpha1 kind: Camera @@ -676,6 +697,7 @@ metadata: spec: target: http://user@cyber.cam.k-space.ee:8080/?action=stream secretRef: camera-secrets + replicas: 2 --- apiVersion: k-space.ee/v1alpha1 kind: Camera @@ -684,6 +706,7 @@ metadata: spec: target: http://user@kitchen.cam.k-space.ee:8080/?action=stream secretRef: camera-secrets + replicas: 2 --- apiVersion: k-space.ee/v1alpha1 kind: Camera @@ -692,6 +715,7 @@ metadata: spec: target: http://user@back-door.cam.k-space.ee:8080/?action=stream secretRef: camera-secrets + replicas: 2 --- apiVersion: k-space.ee/v1alpha1 kind: Camera @@ -700,3 +724,4 @@ metadata: spec: target: http://user@ground-door.cam.k-space.ee:8080/?action=stream secretRef: camera-secrets + replicas: 2 diff --git a/camtiler/minio-support.yml b/camtiler/minio-support.yml deleted file mode 120000 index aef73bb..0000000 --- a/camtiler/minio-support.yml +++ /dev/null @@ -1 +0,0 @@ -../shared/minio-support.yml \ No newline at end of file diff --git a/camtiler/persistence.yml b/camtiler/persistence.yml index bc10f18..95d6af4 100644 --- a/camtiler/persistence.yml +++ b/camtiler/persistence.yml @@ -7,9 +7,10 @@ spec: additionalMongodConfig: systemLog: quiet: true - members: 3 + members: 2 + arbiters: 1 type: ReplicaSet - version: "5.0.9" + version: "6.0.3" security: authentication: modes: ["SCRAM"] @@ -27,7 +28,7 @@ spec: passwordSecretRef: name: mongodb-application-readonly-password roles: - - name: readOnly + - name: read db: application scramCredentialsSecretName: mongodb-application-readonly statefulSet: @@ -35,6 +36,24 @@ spec: logLevel: WARN template: spec: + containers: + - name: mongod + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: journal-volume + mountPath: /data/journal + - name: mongodb-agent + resources: + requests: + cpu: 1m + memory: 100Mi + limits: {} affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -55,8 +74,21 @@ spec: volumeClaimTemplates: - metadata: name: logs-volume + labels: + usecase: logs spec: - storageClassName: local-path + storageClassName: mongo + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi + - metadata: + name: journal-volume + labels: + usecase: journal + spec: + storageClassName: mongo accessModes: - ReadWriteOnce resources: @@ -64,63 +96,44 @@ spec: storage: 512Mi - metadata: name: data-volume + labels: + usecase: data spec: - storageClassName: local-path + storageClassName: mongo accessModes: - ReadWriteOnce resources: requests: storage: 2Gi --- -apiVersion: minio.min.io/v2 -kind: Tenant +apiVersion: apps/v1 +kind: StatefulSet metadata: name: minio - annotations: - prometheus.io/path: /minio/prometheus/metrics - prometheus.io/port: "9000" - prometheus.io/scrape: "true" + labels: + app.kubernetes.io/name: minio spec: - credsSecret: - name: minio-secret - buckets: - - name: application - requestAutoCert: false - users: - - name: minio-user-0 - pools: - - name: pool-0 + selector: + matchLabels: + app.kubernetes.io/name: minio + serviceName: minio-svc + replicas: 4 + podManagementPolicy: Parallel + template: + metadata: + labels: + app.kubernetes.io/name: minio + spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - - key: v1.min.io/tenant + - key: app operator: In values: - minio - - key: v1.min.io/pool - operator: In - values: - - pool-0 topologyKey: kubernetes.io/hostname - resources: - requests: - cpu: '1' - memory: 512Mi - servers: 4 - volumesPerServer: 1 - volumeClaimTemplate: - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: '30Gi' - storageClassName: local-path - status: {} nodeSelector: dedicated: storage tolerations: @@ -128,3 +141,154 @@ spec: operator: Equal value: storage effect: NoSchedule + containers: + - name: minio + env: + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: public + envFrom: + - secretRef: + name: minio-secrets + image: minio/minio:latest + args: + - server + - http://minio-{0...3}.minio-svc.camtiler.svc.cluster.local/data + - --address + - 0.0.0.0:9000 + - --console-address + - 0.0.0.0:9001 + ports: + - containerPort: 9000 + name: http + - containerPort: 9001 + name: console + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + requests: + cpu: 1m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + volumeMounts: + - name: minio-data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: minio-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: '30Gi' + storageClassName: minio +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 9000 + protocol: TCP + name: http + selector: + app.kubernetes.io/name: minio +--- +kind: Service +apiVersion: v1 +metadata: + name: minio-svc +spec: + selector: + app.kubernetes.io/name: minio + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http + port: 9000 +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: minio +spec: + selector: + matchLabels: + app.kubernetes.io/name: minio + podMetricsEndpoints: + - port: http + path: /minio/v2/metrics/node + podTargetLabels: + - app.kubernetes.io/name +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: minio +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: minio + policyTypes: + - Ingress + - Egress + egress: + - ports: + - port: http + to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + ingress: + - ports: + - port: http + from: + - podSelector: {} + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: prometheus-operator + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minio + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + external-dns.alpha.kubernetes.io/target: traefik.k-space.ee +spec: + rules: + - host: cams-s3.k-space.ee + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: minio-svc + port: + number: 9000 + tls: + - hosts: + - "*.k-space.ee"