diff --git a/README.md b/README.md index 54a643a..8499e84 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,13 @@ Watches log directory for logs, and ships them to mongo. 3. `pkg/file` handles file lifecycle; watches files and tails them, streaming lines to `pkg/lines`. 4. `pkg/lines` processes lines and streams them to `pkg/sender`. 5. `pkg/sender` batches lines and ships them to mongo. + +## Skaffold +```bash +export NS=gitdbd-iig6x +kubectl apply -f k8s/dev/mongodb.yaml -n "$NS" +kubectl apply -f k8s/dev/prom.yaml -n "$NS" +skaffold dev --namespace "$NS" + +hwatch kubectl get pods --namespace "$NS" +``` diff --git a/k8s/dev/logmower.yaml b/k8s/dev/logmower.yaml new file mode 100644 index 0000000..1618090 --- /dev/null +++ b/k8s/dev/logmower.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: log-shipper +spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 50% + selector: + matchLabels: + app: log-shipper + template: + metadata: + labels: + app: log-shipper + spec: + serviceAccountName: log-shipper + containers: + - name: log-shipper + image: harbor.k-space.ee/rasmus/logmower-shipper + securityContext: + runAsUser: 0 + env: + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MONGODB_URI + valueFrom: + secretKeyRef: + name: mongodb-application-readwrite + key: connectionString.standard + ports: + - containerPort: 8000 + name: metrics + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: etcmachineid + mountPath: /etc/machine-id + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + volumes: + - name: etcmachineid + hostPath: + path: /etc/machine-id + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log + tolerations: + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "NoSchedule" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: log-shipper + labels: + app: log-shipper diff --git a/k8s/dev/mongodb.yaml b/k8s/dev/mongodb.yaml new file mode 100644 index 0000000..44f827c --- /dev/null +++ b/k8s/dev/mongodb.yaml @@ -0,0 +1,268 @@ +--- +apiVersion: codemowers.io/v1alpha1 +kind: GeneratedSecret +metadata: + name: mongodb-application-readwrite-password +spec: + mapping: + - key: password + value: "%(password)s" +--- +apiVersion: codemowers.io/v1alpha1 +kind: GeneratedSecret +metadata: + name: mongodb-application-readonly-password +spec: + mapping: + - key: password + value: "%(password)s" +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: mongodb +spec: + additionalMongodConfig: + systemLog: + quiet: true + members: 3 + type: ReplicaSet + version: "5.0.9" + security: + authentication: + modes: ["SCRAM"] + users: + - name: readwrite + db: application + passwordSecretRef: + name: mongodb-application-readwrite-password + roles: + - name: readWrite + db: application + scramCredentialsSecretName: mongodb-application-readwrite + - name: readonly + db: application + passwordSecretRef: + name: mongodb-application-readonly-password + roles: + - name: readOnly + db: application + scramCredentialsSecretName: mongodb-application-readonly + statefulSet: + spec: + logLevel: WARN + template: + spec: + containers: + - name: mongod + resources: + requests: + cpu: 100m + memory: 1Gi + limits: + cpu: 4000m + memory: 1Gi + - name: mongodb-agent + resources: + requests: + cpu: 1m + memory: 100Mi + limits: { } + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mongodb-svc + topologyKey: kubernetes.io/hostname + volumeClaimTemplates: + - metadata: + name: logs-volume + spec: + storageClassName: openebs-hostpath + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 512Mi + - metadata: + name: data-volume + spec: + storageClassName: openebs-hostpath + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongoexpress +spec: + revisionHistoryLimit: 0 + replicas: 1 + selector: + matchLabels: + app: mongoexpress + template: + metadata: + labels: + app: mongoexpress + spec: + containers: + - name: mongoexpress + image: mongo-express + ports: + - name: mongoexpress + containerPort: 8081 + env: + - name: ME_CONFIG_MONGODB_URL + valueFrom: + secretKeyRef: + name: mongodb-application-readwrite + key: connectionString.standard + - name: ME_CONFIG_MONGODB_ENABLE_ADMIN + value: "true" +--- +apiVersion: v1 +kind: Service +metadata: + name: mongoexpress + labels: + app: mongoexpress +spec: + selector: + app: mongoexpress + ports: + - protocol: TCP + port: 80 + targetPort: 8081 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: mongoexpress + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + external-dns.alpha.kubernetes.io/target: traefik-iig6x.codemowers.ee +spec: + rules: + - host: mongoexpress-iig6x.codemowers.ee + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: mongoexpress + port: + number: 80 + tls: + - hosts: + - "*.codemowers.ee" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: mongodb-kubernetes-operator +rules: + - apiGroups: + - "" + resources: + - pods + - services + - configmaps + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - mongodbcommunity.mongodb.com + resources: + - mongodbcommunity + - mongodbcommunity/status + - mongodbcommunity/spec + - mongodbcommunity/finalizers + verbs: + - get + - patch + - list + - update + - watch +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-database +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - apiGroups: + - "" + resources: + - pods + verbs: + - patch + - delete + - get +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator +subjects: + - kind: ServiceAccount + name: mongodb-kubernetes-operator +roleRef: + kind: Role + name: mongodb-kubernetes-operator + apiGroup: rbac.authorization.k8s.io +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-database +subjects: + - kind: ServiceAccount + name: mongodb-database +roleRef: + kind: Role + name: mongodb-database + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-kubernetes-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-database diff --git a/k8s/dev/prom.yaml b/k8s/dev/prom.yaml new file mode 100644 index 0000000..7d4ca80 --- /dev/null +++ b/k8s/dev/prom.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: logmower-shipper +spec: + selector: + matchLabels: + app: logmower-shipper + podMetricsEndpoints: + - port: metrics diff --git a/pkg/mongo/metrics.go b/pkg/mongo/metrics.go index 6316611..5312285 100644 --- a/pkg/mongo/metrics.go +++ b/pkg/mongo/metrics.go @@ -1,7 +1,6 @@ package mongo import ( - "context" "log" "time" @@ -14,27 +13,12 @@ import ( const promSubsystem = "database" -var ( - promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{ - Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem, - Name: "heartbeat_time", - Help: "Time in seconds for succeeded heartbeat, or 0 on failure", - Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50}, - }, []string{"connection_id"}) - - promDbCmd = promauto.NewHistogramVec(prom.HistogramOpts{ - Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem, - Name: "operation_latency", // "command_time", - Help: "Time in seconds of commands", - Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50}, - }, []string{"connection_id", "command_name"}) - - promDbCmdErr = promauto.NewCounterVec(prom.CounterOpts{ - Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem, - Name: "errors", - Help: "Failed commands (also reflected elsewhere)", - }, []string{"connection_id", "command_name"}) -) +var promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{ + Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem, + Name: "heartbeat_time", + Help: "Time in seconds for succeeded heartbeat, or 0 on failure", + Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50}, +}, []string{"connection_id"}) func monitoredClientOptions() *mongoOpt.ClientOptions { return mongoOpt.Client(). @@ -46,15 +30,5 @@ func monitoredClientOptions() *mongoOpt.ClientOptions { promDbHeartbeat.WithLabelValues(ev.ConnectionID).Observe(0) log.Printf("database heartbeat failed on connection %q: %e", ev.ConnectionID, ev.Failure) }, - }). - SetMonitor(&mongoEvent.CommandMonitor{ - Succeeded: func(_ context.Context, ev *mongoEvent.CommandSucceededEvent) { - promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds()) - }, - Failed: func(_ context.Context, ev *mongoEvent.CommandFailedEvent) { - promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds()) - - promDbCmdErr.WithLabelValues(ev.ConnectionID, ev.CommandName).Add(1) - }, }) } diff --git a/pkg/mongo/mongo.go b/pkg/mongo/mongo.go index 1739373..fe737b2 100644 --- a/pkg/mongo/mongo.go +++ b/pkg/mongo/mongo.go @@ -22,11 +22,15 @@ func Initialize(ctx context.Context, uri string) (*mongo.Collection, error) { dbOpt := monitoredClientOptions().ApplyURI(uri) - dbClient, err := mongo.Connect(globals.MongoTimeout(ctx)) + dbClient, err := mongo.Connect(globals.MongoTimeout(ctx), dbOpt) if err != nil { return nil, fmt.Errorf("connecting to %q: %w", dbOpt.GetURI(), err) } + if err := dbClient.Ping(globals.MongoTimeout(ctx), nil); err != nil { + return nil, fmt.Errorf("first ping to database: %w", err) + } + col := dbClient.Database(uriParsed.Path).Collection("logs") if err := InitializeIndexes(globals.MongoTimeout(ctx), col); err != nil { diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index 7bb9a46..a98ebf2 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -32,10 +32,10 @@ var App = &cli.App{ //TODO: &cli.BoolFlag{Name: "normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords"}, //TODO: &cli.BoolFlag{Name: "parse-json"}, // - &cli.StringSliceFlag{Category: "selectors", Name: "pod-namespace", EnvVars: []string{"KUBE_POD_NAMESPACE"}, Usage: "whitelist filter for filenames"}, + &cli.StringSliceFlag{Category: "selectors", Name: "namespace", EnvVars: []string{"KUBE_NAMESPACE"}, Usage: "whitelist filter for filenames"}, &cli.StringSliceFlag{Category: "selectors", Name: "pod-prefix", EnvVars: []string{"KUBE_NODE_NAME"}, Usage: "blacklist filter for filenames"}, // - &cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGO_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true}, + &cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGODB_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true}, }, Before: func(ctx *cli.Context) error { globals.BufferLimitBytes = ctx.Int("max-record-size") @@ -49,7 +49,7 @@ var App = &cli.App{ }, Action: func(ctx *cli.Context) error { - whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("pod-namespace")), ctx.StringSlice("pod-prefix") + whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("namespace")), ctx.StringSlice("pod-prefix") var wg sync.WaitGroup log.Printf("%s %s starting", ctx.App.Name, ctx.App.Version) diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..c2fe4cf --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: skaffold/v3alpha1 +kind: Config +metadata: + name: logmower-shipper +build: + artifacts: + - image: harbor.k-space.ee/rasmus/logmower-shipper + docker: + dockerfile: Dockerfile +deploy: + kubectl: {} +# manifests: + # rawYaml: + # - k8s/staging/deployment.yaml +profiles: + - name: dev + activation: + - command: dev + build: + artifacts: + - image: harbor.k-space.ee/rasmus/logmower-shipper + manifests: + rawYaml: + - k8s/dev/logmower.yaml + # - k8s/dev/mongodb.yaml