add skaffold and fix bugs

main
rasmus 3 months ago
parent a6eba73f0c
commit ff19ca0fff
  1. 10
      README.md
  2. 78
      k8s/dev/logmower.yaml
  3. 268
      k8s/dev/mongodb.yaml
  4. 11
      k8s/dev/prom.yaml
  5. 38
      pkg/mongo/metrics.go
  6. 6
      pkg/mongo/mongo.go
  7. 6
      pkg/watcher/watcher.go
  8. 26
      skaffold.yaml

@ -14,3 +14,13 @@ Watches log directory for logs, and ships them to mongo.
3. `pkg/file` handles file lifecycle; watches files and tails them, streaming lines to `pkg/lines`.
4. `pkg/lines` processes lines and streams them to `pkg/sender`.
5. `pkg/sender` batches lines and ships them to mongo.
## Skaffold
```bash
export NS=gitdbd-iig6x
kubectl apply -f k8s/dev/mongodb.yaml -n "$NS"
kubectl apply -f k8s/dev/prom.yaml -n "$NS"
skaffold dev --namespace "$NS"
hwatch kubectl get pods --namespace "$NS"
```

@ -0,0 +1,78 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: log-shipper
spec:
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 50%
selector:
matchLabels:
app: log-shipper
template:
metadata:
labels:
app: log-shipper
spec:
serviceAccountName: log-shipper
containers:
- name: log-shipper
image: harbor.k-space.ee/rasmus/logmower-shipper
securityContext:
runAsUser: 0
env:
- name: KUBE_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: MONGODB_URI
valueFrom:
secretKeyRef:
name: mongodb-application-readwrite
key: connectionString.standard
ports:
- containerPort: 8000
name: metrics
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: etcmachineid
mountPath: /etc/machine-id
readOnly: true
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: etcmachineid
hostPath:
path: /etc/machine-id
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
tolerations:
- operator: "Exists"
effect: "NoExecute"
- operator: "Exists"
effect: "NoSchedule"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: log-shipper
labels:
app: log-shipper

@ -0,0 +1,268 @@
---
apiVersion: codemowers.io/v1alpha1
kind: GeneratedSecret
metadata:
name: mongodb-application-readwrite-password
spec:
mapping:
- key: password
value: "%(password)s"
---
apiVersion: codemowers.io/v1alpha1
kind: GeneratedSecret
metadata:
name: mongodb-application-readonly-password
spec:
mapping:
- key: password
value: "%(password)s"
---
apiVersion: mongodbcommunity.mongodb.com/v1
kind: MongoDBCommunity
metadata:
name: mongodb
spec:
additionalMongodConfig:
systemLog:
quiet: true
members: 3
type: ReplicaSet
version: "5.0.9"
security:
authentication:
modes: ["SCRAM"]
users:
- name: readwrite
db: application
passwordSecretRef:
name: mongodb-application-readwrite-password
roles:
- name: readWrite
db: application
scramCredentialsSecretName: mongodb-application-readwrite
- name: readonly
db: application
passwordSecretRef:
name: mongodb-application-readonly-password
roles:
- name: readOnly
db: application
scramCredentialsSecretName: mongodb-application-readonly
statefulSet:
spec:
logLevel: WARN
template:
spec:
containers:
- name: mongod
resources:
requests:
cpu: 100m
memory: 1Gi
limits:
cpu: 4000m
memory: 1Gi
- name: mongodb-agent
resources:
requests:
cpu: 1m
memory: 100Mi
limits: { }
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- mongodb-svc
topologyKey: kubernetes.io/hostname
volumeClaimTemplates:
- metadata:
name: logs-volume
spec:
storageClassName: openebs-hostpath
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 512Mi
- metadata:
name: data-volume
spec:
storageClassName: openebs-hostpath
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: mongoexpress
spec:
revisionHistoryLimit: 0
replicas: 1
selector:
matchLabels:
app: mongoexpress
template:
metadata:
labels:
app: mongoexpress
spec:
containers:
- name: mongoexpress
image: mongo-express
ports:
- name: mongoexpress
containerPort: 8081
env:
- name: ME_CONFIG_MONGODB_URL
valueFrom:
secretKeyRef:
name: mongodb-application-readwrite
key: connectionString.standard
- name: ME_CONFIG_MONGODB_ENABLE_ADMIN
value: "true"
---
apiVersion: v1
kind: Service
metadata:
name: mongoexpress
labels:
app: mongoexpress
spec:
selector:
app: mongoexpress
ports:
- protocol: TCP
port: 80
targetPort: 8081
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: mongoexpress
annotations:
kubernetes.io/ingress.class: traefik
traefik.ingress.kubernetes.io/router.entrypoints: websecure
traefik.ingress.kubernetes.io/router.tls: "true"
external-dns.alpha.kubernetes.io/target: traefik-iig6x.codemowers.ee
spec:
rules:
- host: mongoexpress-iig6x.codemowers.ee
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: mongoexpress
port:
number: 80
tls:
- hosts:
- "*.codemowers.ee"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: mongodb-kubernetes-operator
rules:
- apiGroups:
- ""
resources:
- pods
- services
- configmaps
- secrets
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- apps
resources:
- statefulsets
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- mongodbcommunity.mongodb.com
resources:
- mongodbcommunity
- mongodbcommunity/status
- mongodbcommunity/spec
- mongodbcommunity/finalizers
verbs:
- get
- patch
- list
- update
- watch
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mongodb-database
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- apiGroups:
- ""
resources:
- pods
verbs:
- patch
- delete
- get
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mongodb-kubernetes-operator
subjects:
- kind: ServiceAccount
name: mongodb-kubernetes-operator
roleRef:
kind: Role
name: mongodb-kubernetes-operator
apiGroup: rbac.authorization.k8s.io
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mongodb-database
subjects:
- kind: ServiceAccount
name: mongodb-database
roleRef:
kind: Role
name: mongodb-database
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: mongodb-kubernetes-operator
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: mongodb-database

@ -0,0 +1,11 @@
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: logmower-shipper
spec:
selector:
matchLabels:
app: logmower-shipper
podMetricsEndpoints:
- port: metrics

@ -1,7 +1,6 @@
package mongo
import (
"context"
"log"
"time"
@ -14,27 +13,12 @@ import (
const promSubsystem = "database"
var (
promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{
Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
Name: "heartbeat_time",
Help: "Time in seconds for succeeded heartbeat, or 0 on failure",
Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
}, []string{"connection_id"})
promDbCmd = promauto.NewHistogramVec(prom.HistogramOpts{
Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
Name: "operation_latency", // "command_time",
Help: "Time in seconds of commands",
Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
}, []string{"connection_id", "command_name"})
promDbCmdErr = promauto.NewCounterVec(prom.CounterOpts{
Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
Name: "errors",
Help: "Failed commands (also reflected elsewhere)",
}, []string{"connection_id", "command_name"})
)
var promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{
Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
Name: "heartbeat_time",
Help: "Time in seconds for succeeded heartbeat, or 0 on failure",
Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
}, []string{"connection_id"})
func monitoredClientOptions() *mongoOpt.ClientOptions {
return mongoOpt.Client().
@ -46,15 +30,5 @@ func monitoredClientOptions() *mongoOpt.ClientOptions {
promDbHeartbeat.WithLabelValues(ev.ConnectionID).Observe(0)
log.Printf("database heartbeat failed on connection %q: %e", ev.ConnectionID, ev.Failure)
},
}).
SetMonitor(&mongoEvent.CommandMonitor{
Succeeded: func(_ context.Context, ev *mongoEvent.CommandSucceededEvent) {
promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds())
},
Failed: func(_ context.Context, ev *mongoEvent.CommandFailedEvent) {
promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds())
promDbCmdErr.WithLabelValues(ev.ConnectionID, ev.CommandName).Add(1)
},
})
}

@ -22,11 +22,15 @@ func Initialize(ctx context.Context, uri string) (*mongo.Collection, error) {
dbOpt := monitoredClientOptions().ApplyURI(uri)
dbClient, err := mongo.Connect(globals.MongoTimeout(ctx))
dbClient, err := mongo.Connect(globals.MongoTimeout(ctx), dbOpt)
if err != nil {
return nil, fmt.Errorf("connecting to %q: %w", dbOpt.GetURI(), err)
}
if err := dbClient.Ping(globals.MongoTimeout(ctx), nil); err != nil {
return nil, fmt.Errorf("first ping to database: %w", err)
}
col := dbClient.Database(uriParsed.Path).Collection("logs")
if err := InitializeIndexes(globals.MongoTimeout(ctx), col); err != nil {

@ -32,10 +32,10 @@ var App = &cli.App{
//TODO: &cli.BoolFlag{Name: "normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords"},
//TODO: &cli.BoolFlag{Name: "parse-json"},
//
&cli.StringSliceFlag{Category: "selectors", Name: "pod-namespace", EnvVars: []string{"KUBE_POD_NAMESPACE"}, Usage: "whitelist filter for filenames"},
&cli.StringSliceFlag{Category: "selectors", Name: "namespace", EnvVars: []string{"KUBE_NAMESPACE"}, Usage: "whitelist filter for filenames"},
&cli.StringSliceFlag{Category: "selectors", Name: "pod-prefix", EnvVars: []string{"KUBE_NODE_NAME"}, Usage: "blacklist filter for filenames"},
//
&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGO_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGODB_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
},
Before: func(ctx *cli.Context) error {
globals.BufferLimitBytes = ctx.Int("max-record-size")
@ -49,7 +49,7 @@ var App = &cli.App{
},
Action: func(ctx *cli.Context) error {
whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("pod-namespace")), ctx.StringSlice("pod-prefix")
whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("namespace")), ctx.StringSlice("pod-prefix")
var wg sync.WaitGroup
log.Printf("%s %s starting", ctx.App.Name, ctx.App.Version)

@ -0,0 +1,26 @@
---
apiVersion: skaffold/v3alpha1
kind: Config
metadata:
name: logmower-shipper
build:
artifacts:
- image: harbor.k-space.ee/rasmus/logmower-shipper
docker:
dockerfile: Dockerfile
deploy:
kubectl: {}
# manifests:
# rawYaml:
# - k8s/staging/deployment.yaml
profiles:
- name: dev
activation:
- command: dev
build:
artifacts:
- image: harbor.k-space.ee/rasmus/logmower-shipper
manifests:
rawYaml:
- k8s/dev/logmower.yaml
# - k8s/dev/mongodb.yaml
Loading…
Cancel
Save