From 0c142831364805217ecb2e3c5d9eda9931807532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauri=20V=C3=B5sandi?= Date: Sat, 5 Nov 2022 20:55:52 +0200 Subject: [PATCH] Add logmower --- argocd/applications/logmower.yml | 17 ++ logmower/application.yml | 411 +++++++++++++++++++++++++++++++ logmower/mongodb-support.yml | 1 + logmower/mongoexpress.yml | 47 ++++ logmower/networkpolicy-base.yml | 1 + 5 files changed, 477 insertions(+) create mode 100644 argocd/applications/logmower.yml create mode 100644 logmower/application.yml create mode 120000 logmower/mongodb-support.yml create mode 100644 logmower/mongoexpress.yml create mode 120000 logmower/networkpolicy-base.yml diff --git a/argocd/applications/logmower.yml b/argocd/applications/logmower.yml new file mode 100644 index 0000000..bbe4684 --- /dev/null +++ b/argocd/applications/logmower.yml @@ -0,0 +1,17 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: logmower + namespace: argocd +spec: + project: default + source: + repoURL: 'git@git.k-space.ee:k-space/kube.git' + path: logmower + targetRevision: HEAD + destination: + server: 'https://kubernetes.default.svc' + namespace: logmower + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/logmower/application.yml b/logmower/application.yml new file mode 100644 index 0000000..1b3a303 --- /dev/null +++ b/logmower/application.yml @@ -0,0 +1,411 @@ +--- +apiVersion: codemowers.io/v1alpha1 +kind: GeneratedSecret +metadata: + name: logmower-readwrite-password +spec: + mapping: + - key: password + value: "%(password)s" +--- +apiVersion: codemowers.io/v1alpha1 +kind: GeneratedSecret +metadata: + name: logmower-readonly-password +spec: + mapping: + - key: password + value: "%(password)s" +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: logmower-mongodb +spec: + additionalMongodConfig: + systemLog: + quiet: true + members: 3 + type: ReplicaSet + version: "5.0.13" + security: + authentication: + modes: ["SCRAM"] + users: + - name: readwrite + db: application + passwordSecretRef: + name: logmower-readwrite-password + roles: + - name: readWrite + db: application + scramCredentialsSecretName: logmower-readwrite + - name: readonly + db: application + passwordSecretRef: + name: logmower-readonly-password + roles: + - name: read + db: application + scramCredentialsSecretName: logmower-readonly + statefulSet: + spec: + logLevel: WARN + template: + spec: + containers: + - name: mongod + resources: + requests: + cpu: 100m + memory: 2Gi + limits: + cpu: 2000m + memory: 2Gi + - name: mongodb-agent + resources: + requests: + cpu: 1m + memory: 100Mi + limits: {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - logmower-svc + topologyKey: kubernetes.io/hostname + nodeSelector: + dedicated: monitoring + tolerations: + - key: dedicated + operator: Equal + value: monitoring + effect: NoSchedule + volumeClaimTemplates: + - metadata: + name: logs-volume + spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 512Mi + - metadata: + name: data-volume + spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: logmower-shipper +spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 50% + selector: + matchLabels: + app: logmower-shipper + template: + metadata: + labels: + app: logmower-shipper + spec: + serviceAccountName: logmower-shipper + containers: + - name: logmower-shipper + image: harbor.k-space.ee/k-space/logmower-shipper-prototype + securityContext: + runAsUser: 0 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MONGODB_HOST + valueFrom: + secretKeyRef: + name: logmower-mongodb-application-readwrite + key: connectionString.standard + ports: + - containerPort: 8000 + name: metrics + command: + - /log_shipper.py + - "--max-collection-size" + - "10000000000" + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: etcmachineid + mountPath: /etc/machine-id + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + volumes: + - name: etcmachineid + hostPath: + path: /etc/machine-id + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log + tolerations: + - operator: "Exists" + effect: "NoSchedule" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: logging-logmower-shipper +subjects: + - kind: ServiceAccount + name: logmower-shipper + namespace: logmower +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: logmower-shipper + labels: + app: logmower-shipper +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: logmower-shipper +spec: + podSelector: + matchLabels: + app: logmower-shipper + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: prometheus-operator + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + egress: + - to: + - podSelector: + matchLabels: + app: logmower-mongodb-svc + ports: + - port: 27017 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: logmower-eventsource +spec: + podSelector: + matchLabels: + app: logmower-eventsource + policyTypes: + - Ingress + - Egress + egress: + - to: + - podSelector: + matchLabels: + app: logmower-mongodb-svc + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app.kubernetes.io/name: traefik +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: logmower-frontend +spec: + podSelector: + matchLabels: + app: logmower-frontend + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app.kubernetes.io/name: traefik +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: logmower-shipper +spec: + selector: + matchLabels: + app: logmower-shipper + podMetricsEndpoints: + - port: metrics +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: logmower + annotations: + kubernetes.io/ingress.class: traefik + cert-manager.io/cluster-issuer: default + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.middlewares: traefik-sso@kubernetescrd + traefik.ingress.kubernetes.io/router.tls: "true" + external-dns.alpha.kubernetes.io/target: traefik.k-space.ee +spec: + rules: + - host: log.k-space.ee + http: + paths: + - pathType: Prefix + path: "/events" + backend: + service: + name: logmower-eventsource + port: + number: 3002 + - pathType: Prefix + path: "/" + backend: + service: + name: logmower-frontend + port: + number: 8080 + tls: + - hosts: + - "*.k-space.ee" +--- +apiVersion: v1 +kind: Service +metadata: + name: logmower-eventsource +spec: + type: ClusterIP + selector: + app: logmower-eventsource + ports: + - protocol: TCP + port: 3002 +--- +apiVersion: v1 +kind: Service +metadata: + name: logmower-frontend +spec: + type: ClusterIP + selector: + app: logmower-frontend + ports: + - protocol: TCP + port: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logmower-frontend +spec: + selector: + matchLabels: + app: logmower-frontend + template: + metadata: + labels: + app: logmower-frontend + spec: + containers: + - name: logmower-frontend + image: harbor.k-space.ee/k-space/logmower-frontend + ports: + - containerPort: 8080 + name: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logmower-eventsource +spec: + selector: + matchLabels: + app: logmower-eventsource + template: + metadata: + labels: + app: logmower-eventsource + spec: + containers: + - name: logmower-eventsource + image: harbor.k-space.ee/k-space/logmower-eventsource + command: + - npm + - start + ports: + - containerPort: 3002 + name: nodejs + env: + - name: MONGODB_HOST + valueFrom: + secretKeyRef: + name: logmower-mongodb-application-readonly + key: connectionString.standard + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: logmower-mongodb +spec: + podSelector: + matchLabels: + app: logmower-mongodb-svc + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: {} + ports: + - port: 27017 + egress: + - to: + - podSelector: + matchLabels: + app: logmower-mongodb-svc + ports: + - port: 27017 diff --git a/logmower/mongodb-support.yml b/logmower/mongodb-support.yml new file mode 120000 index 0000000..e834dd2 --- /dev/null +++ b/logmower/mongodb-support.yml @@ -0,0 +1 @@ +../mongodb-operator/mongodb-support.yml \ No newline at end of file diff --git a/logmower/mongoexpress.yml b/logmower/mongoexpress.yml new file mode 100644 index 0000000..9a33ac0 --- /dev/null +++ b/logmower/mongoexpress.yml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logmower-mongoexpress +spec: + revisionHistoryLimit: 0 + replicas: 1 + selector: + matchLabels: + app: logmower-mongoexpress + template: + metadata: + labels: + app: logmower-mongoexpress + spec: + containers: + - name: mongoexpress + image: mongo-express + ports: + - name: mongoexpress + containerPort: 8081 + env: + - name: ME_CONFIG_MONGODB_URL + valueFrom: + secretKeyRef: + name: logmower-mongodb-application-readonly + key: connectionString.standard + - name: ME_CONFIG_MONGODB_ENABLE_ADMIN + value: "true" +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: logmower-mongoexpress +spec: + podSelector: + matchLabels: + app: logmower-mongoexpress + policyTypes: + - Egress + egress: + - to: + - podSelector: + matchLabels: + app: logmower-mongodb-svc + ports: + - port: 27017 diff --git a/logmower/networkpolicy-base.yml b/logmower/networkpolicy-base.yml new file mode 120000 index 0000000..e84a698 --- /dev/null +++ b/logmower/networkpolicy-base.yml @@ -0,0 +1 @@ +../shared/networkpolicy-base.yml \ No newline at end of file