add skaffold and fix bugs
This commit is contained in:
parent
a6eba73f0c
commit
ff19ca0fff
10
README.md
10
README.md
|
@ -14,3 +14,13 @@ Watches log directory for logs, and ships them to mongo.
|
||||||
3. `pkg/file` handles file lifecycle; watches files and tails them, streaming lines to `pkg/lines`.
|
3. `pkg/file` handles file lifecycle; watches files and tails them, streaming lines to `pkg/lines`.
|
||||||
4. `pkg/lines` processes lines and streams them to `pkg/sender`.
|
4. `pkg/lines` processes lines and streams them to `pkg/sender`.
|
||||||
5. `pkg/sender` batches lines and ships them to mongo.
|
5. `pkg/sender` batches lines and ships them to mongo.
|
||||||
|
|
||||||
|
## Skaffold
|
||||||
|
```bash
|
||||||
|
export NS=gitdbd-iig6x
|
||||||
|
kubectl apply -f k8s/dev/mongodb.yaml -n "$NS"
|
||||||
|
kubectl apply -f k8s/dev/prom.yaml -n "$NS"
|
||||||
|
skaffold dev --namespace "$NS"
|
||||||
|
|
||||||
|
hwatch kubectl get pods --namespace "$NS"
|
||||||
|
```
|
||||||
|
|
|
@ -0,0 +1,78 @@
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: log-shipper
|
||||||
|
spec:
|
||||||
|
updateStrategy:
|
||||||
|
type: RollingUpdate
|
||||||
|
rollingUpdate:
|
||||||
|
maxUnavailable: 50%
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: log-shipper
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: log-shipper
|
||||||
|
spec:
|
||||||
|
serviceAccountName: log-shipper
|
||||||
|
containers:
|
||||||
|
- name: log-shipper
|
||||||
|
image: harbor.k-space.ee/rasmus/logmower-shipper
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 0
|
||||||
|
env:
|
||||||
|
- name: KUBE_NAMESPACE
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: metadata.namespace
|
||||||
|
- name: KUBE_NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
- name: MONGODB_URI
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: mongodb-application-readwrite
|
||||||
|
key: connectionString.standard
|
||||||
|
ports:
|
||||||
|
- containerPort: 8000
|
||||||
|
name: metrics
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: 200Mi
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 100Mi
|
||||||
|
volumeMounts:
|
||||||
|
- name: etcmachineid
|
||||||
|
mountPath: /etc/machine-id
|
||||||
|
readOnly: true
|
||||||
|
- name: varlibdockercontainers
|
||||||
|
mountPath: /var/lib/docker/containers
|
||||||
|
readOnly: true
|
||||||
|
- name: varlog
|
||||||
|
mountPath: /var/log
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: etcmachineid
|
||||||
|
hostPath:
|
||||||
|
path: /etc/machine-id
|
||||||
|
- name: varlibdockercontainers
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/docker/containers
|
||||||
|
- name: varlog
|
||||||
|
hostPath:
|
||||||
|
path: /var/log
|
||||||
|
tolerations:
|
||||||
|
- operator: "Exists"
|
||||||
|
effect: "NoExecute"
|
||||||
|
- operator: "Exists"
|
||||||
|
effect: "NoSchedule"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: log-shipper
|
||||||
|
labels:
|
||||||
|
app: log-shipper
|
|
@ -0,0 +1,268 @@
|
||||||
|
---
|
||||||
|
apiVersion: codemowers.io/v1alpha1
|
||||||
|
kind: GeneratedSecret
|
||||||
|
metadata:
|
||||||
|
name: mongodb-application-readwrite-password
|
||||||
|
spec:
|
||||||
|
mapping:
|
||||||
|
- key: password
|
||||||
|
value: "%(password)s"
|
||||||
|
---
|
||||||
|
apiVersion: codemowers.io/v1alpha1
|
||||||
|
kind: GeneratedSecret
|
||||||
|
metadata:
|
||||||
|
name: mongodb-application-readonly-password
|
||||||
|
spec:
|
||||||
|
mapping:
|
||||||
|
- key: password
|
||||||
|
value: "%(password)s"
|
||||||
|
---
|
||||||
|
apiVersion: mongodbcommunity.mongodb.com/v1
|
||||||
|
kind: MongoDBCommunity
|
||||||
|
metadata:
|
||||||
|
name: mongodb
|
||||||
|
spec:
|
||||||
|
additionalMongodConfig:
|
||||||
|
systemLog:
|
||||||
|
quiet: true
|
||||||
|
members: 3
|
||||||
|
type: ReplicaSet
|
||||||
|
version: "5.0.9"
|
||||||
|
security:
|
||||||
|
authentication:
|
||||||
|
modes: ["SCRAM"]
|
||||||
|
users:
|
||||||
|
- name: readwrite
|
||||||
|
db: application
|
||||||
|
passwordSecretRef:
|
||||||
|
name: mongodb-application-readwrite-password
|
||||||
|
roles:
|
||||||
|
- name: readWrite
|
||||||
|
db: application
|
||||||
|
scramCredentialsSecretName: mongodb-application-readwrite
|
||||||
|
- name: readonly
|
||||||
|
db: application
|
||||||
|
passwordSecretRef:
|
||||||
|
name: mongodb-application-readonly-password
|
||||||
|
roles:
|
||||||
|
- name: readOnly
|
||||||
|
db: application
|
||||||
|
scramCredentialsSecretName: mongodb-application-readonly
|
||||||
|
statefulSet:
|
||||||
|
spec:
|
||||||
|
logLevel: WARN
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: mongod
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
cpu: 4000m
|
||||||
|
memory: 1Gi
|
||||||
|
- name: mongodb-agent
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 1m
|
||||||
|
memory: 100Mi
|
||||||
|
limits: { }
|
||||||
|
affinity:
|
||||||
|
podAntiAffinity:
|
||||||
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
- labelSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: app
|
||||||
|
operator: In
|
||||||
|
values:
|
||||||
|
- mongodb-svc
|
||||||
|
topologyKey: kubernetes.io/hostname
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: logs-volume
|
||||||
|
spec:
|
||||||
|
storageClassName: openebs-hostpath
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 512Mi
|
||||||
|
- metadata:
|
||||||
|
name: data-volume
|
||||||
|
spec:
|
||||||
|
storageClassName: openebs-hostpath
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 2Gi
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: mongoexpress
|
||||||
|
spec:
|
||||||
|
revisionHistoryLimit: 0
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: mongoexpress
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: mongoexpress
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: mongoexpress
|
||||||
|
image: mongo-express
|
||||||
|
ports:
|
||||||
|
- name: mongoexpress
|
||||||
|
containerPort: 8081
|
||||||
|
env:
|
||||||
|
- name: ME_CONFIG_MONGODB_URL
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: mongodb-application-readwrite
|
||||||
|
key: connectionString.standard
|
||||||
|
- name: ME_CONFIG_MONGODB_ENABLE_ADMIN
|
||||||
|
value: "true"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: mongoexpress
|
||||||
|
labels:
|
||||||
|
app: mongoexpress
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: mongoexpress
|
||||||
|
ports:
|
||||||
|
- protocol: TCP
|
||||||
|
port: 80
|
||||||
|
targetPort: 8081
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: mongoexpress
|
||||||
|
annotations:
|
||||||
|
kubernetes.io/ingress.class: traefik
|
||||||
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||||
|
traefik.ingress.kubernetes.io/router.tls: "true"
|
||||||
|
external-dns.alpha.kubernetes.io/target: traefik-iig6x.codemowers.ee
|
||||||
|
spec:
|
||||||
|
rules:
|
||||||
|
- host: mongoexpress-iig6x.codemowers.ee
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- pathType: Prefix
|
||||||
|
path: "/"
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: mongoexpress
|
||||||
|
port:
|
||||||
|
number: 80
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- "*.codemowers.ee"
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: mongodb-kubernetes-operator
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
- services
|
||||||
|
- configmaps
|
||||||
|
- secrets
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- apps
|
||||||
|
resources:
|
||||||
|
- statefulsets
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- mongodbcommunity.mongodb.com
|
||||||
|
resources:
|
||||||
|
- mongodbcommunity
|
||||||
|
- mongodbcommunity/status
|
||||||
|
- mongodbcommunity/spec
|
||||||
|
- mongodbcommunity/finalizers
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- patch
|
||||||
|
- list
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
---
|
||||||
|
kind: Role
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: mongodb-database
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- secrets
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
verbs:
|
||||||
|
- patch
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
---
|
||||||
|
kind: RoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: mongodb-kubernetes-operator
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: mongodb-kubernetes-operator
|
||||||
|
roleRef:
|
||||||
|
kind: Role
|
||||||
|
name: mongodb-kubernetes-operator
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
---
|
||||||
|
kind: RoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: mongodb-database
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: mongodb-database
|
||||||
|
roleRef:
|
||||||
|
kind: Role
|
||||||
|
name: mongodb-database
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: mongodb-kubernetes-operator
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: mongodb-database
|
|
@ -0,0 +1,11 @@
|
||||||
|
---
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PodMonitor
|
||||||
|
metadata:
|
||||||
|
name: logmower-shipper
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: logmower-shipper
|
||||||
|
podMetricsEndpoints:
|
||||||
|
- port: metrics
|
|
@ -1,7 +1,6 @@
|
||||||
package mongo
|
package mongo
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"log"
|
"log"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -14,27 +13,12 @@ import (
|
||||||
|
|
||||||
const promSubsystem = "database"
|
const promSubsystem = "database"
|
||||||
|
|
||||||
var (
|
var promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{
|
||||||
promDbHeartbeat = promauto.NewHistogramVec(prom.HistogramOpts{
|
Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
|
||||||
Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
|
Name: "heartbeat_time",
|
||||||
Name: "heartbeat_time",
|
Help: "Time in seconds for succeeded heartbeat, or 0 on failure",
|
||||||
Help: "Time in seconds for succeeded heartbeat, or 0 on failure",
|
Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
|
||||||
Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
|
}, []string{"connection_id"})
|
||||||
}, []string{"connection_id"})
|
|
||||||
|
|
||||||
promDbCmd = promauto.NewHistogramVec(prom.HistogramOpts{
|
|
||||||
Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
|
|
||||||
Name: "operation_latency", // "command_time",
|
|
||||||
Help: "Time in seconds of commands",
|
|
||||||
Buckets: []float64{0.1, 0.2, 0.5, 1, 5, 10, 50},
|
|
||||||
}, []string{"connection_id", "command_name"})
|
|
||||||
|
|
||||||
promDbCmdErr = promauto.NewCounterVec(prom.CounterOpts{
|
|
||||||
Namespace: globals.PrometheusPrefix, Subsystem: promSubsystem,
|
|
||||||
Name: "errors",
|
|
||||||
Help: "Failed commands (also reflected elsewhere)",
|
|
||||||
}, []string{"connection_id", "command_name"})
|
|
||||||
)
|
|
||||||
|
|
||||||
func monitoredClientOptions() *mongoOpt.ClientOptions {
|
func monitoredClientOptions() *mongoOpt.ClientOptions {
|
||||||
return mongoOpt.Client().
|
return mongoOpt.Client().
|
||||||
|
@ -46,15 +30,5 @@ func monitoredClientOptions() *mongoOpt.ClientOptions {
|
||||||
promDbHeartbeat.WithLabelValues(ev.ConnectionID).Observe(0)
|
promDbHeartbeat.WithLabelValues(ev.ConnectionID).Observe(0)
|
||||||
log.Printf("database heartbeat failed on connection %q: %e", ev.ConnectionID, ev.Failure)
|
log.Printf("database heartbeat failed on connection %q: %e", ev.ConnectionID, ev.Failure)
|
||||||
},
|
},
|
||||||
}).
|
|
||||||
SetMonitor(&mongoEvent.CommandMonitor{
|
|
||||||
Succeeded: func(_ context.Context, ev *mongoEvent.CommandSucceededEvent) {
|
|
||||||
promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds())
|
|
||||||
},
|
|
||||||
Failed: func(_ context.Context, ev *mongoEvent.CommandFailedEvent) {
|
|
||||||
promDbCmd.WithLabelValues(ev.ConnectionID, ev.CommandName).Observe(time.Duration(ev.DurationNanos).Seconds())
|
|
||||||
|
|
||||||
promDbCmdErr.WithLabelValues(ev.ConnectionID, ev.CommandName).Add(1)
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,11 +22,15 @@ func Initialize(ctx context.Context, uri string) (*mongo.Collection, error) {
|
||||||
|
|
||||||
dbOpt := monitoredClientOptions().ApplyURI(uri)
|
dbOpt := monitoredClientOptions().ApplyURI(uri)
|
||||||
|
|
||||||
dbClient, err := mongo.Connect(globals.MongoTimeout(ctx))
|
dbClient, err := mongo.Connect(globals.MongoTimeout(ctx), dbOpt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("connecting to %q: %w", dbOpt.GetURI(), err)
|
return nil, fmt.Errorf("connecting to %q: %w", dbOpt.GetURI(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := dbClient.Ping(globals.MongoTimeout(ctx), nil); err != nil {
|
||||||
|
return nil, fmt.Errorf("first ping to database: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
col := dbClient.Database(uriParsed.Path).Collection("logs")
|
col := dbClient.Database(uriParsed.Path).Collection("logs")
|
||||||
|
|
||||||
if err := InitializeIndexes(globals.MongoTimeout(ctx), col); err != nil {
|
if err := InitializeIndexes(globals.MongoTimeout(ctx), col); err != nil {
|
||||||
|
|
|
@ -32,10 +32,10 @@ var App = &cli.App{
|
||||||
//TODO: &cli.BoolFlag{Name: "normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords"},
|
//TODO: &cli.BoolFlag{Name: "normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords"},
|
||||||
//TODO: &cli.BoolFlag{Name: "parse-json"},
|
//TODO: &cli.BoolFlag{Name: "parse-json"},
|
||||||
//
|
//
|
||||||
&cli.StringSliceFlag{Category: "selectors", Name: "pod-namespace", EnvVars: []string{"KUBE_POD_NAMESPACE"}, Usage: "whitelist filter for filenames"},
|
&cli.StringSliceFlag{Category: "selectors", Name: "namespace", EnvVars: []string{"KUBE_NAMESPACE"}, Usage: "whitelist filter for filenames"},
|
||||||
&cli.StringSliceFlag{Category: "selectors", Name: "pod-prefix", EnvVars: []string{"KUBE_NODE_NAME"}, Usage: "blacklist filter for filenames"},
|
&cli.StringSliceFlag{Category: "selectors", Name: "pod-prefix", EnvVars: []string{"KUBE_NODE_NAME"}, Usage: "blacklist filter for filenames"},
|
||||||
//
|
//
|
||||||
&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGO_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
|
&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGODB_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
|
||||||
},
|
},
|
||||||
Before: func(ctx *cli.Context) error {
|
Before: func(ctx *cli.Context) error {
|
||||||
globals.BufferLimitBytes = ctx.Int("max-record-size")
|
globals.BufferLimitBytes = ctx.Int("max-record-size")
|
||||||
|
@ -49,7 +49,7 @@ var App = &cli.App{
|
||||||
},
|
},
|
||||||
|
|
||||||
Action: func(ctx *cli.Context) error {
|
Action: func(ctx *cli.Context) error {
|
||||||
whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("pod-namespace")), ctx.StringSlice("pod-prefix")
|
whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("namespace")), ctx.StringSlice("pod-prefix")
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
log.Printf("%s %s starting", ctx.App.Name, ctx.App.Version)
|
log.Printf("%s %s starting", ctx.App.Name, ctx.App.Version)
|
||||||
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
---
|
||||||
|
apiVersion: skaffold/v3alpha1
|
||||||
|
kind: Config
|
||||||
|
metadata:
|
||||||
|
name: logmower-shipper
|
||||||
|
build:
|
||||||
|
artifacts:
|
||||||
|
- image: harbor.k-space.ee/rasmus/logmower-shipper
|
||||||
|
docker:
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
deploy:
|
||||||
|
kubectl: {}
|
||||||
|
# manifests:
|
||||||
|
# rawYaml:
|
||||||
|
# - k8s/staging/deployment.yaml
|
||||||
|
profiles:
|
||||||
|
- name: dev
|
||||||
|
activation:
|
||||||
|
- command: dev
|
||||||
|
build:
|
||||||
|
artifacts:
|
||||||
|
- image: harbor.k-space.ee/rasmus/logmower-shipper
|
||||||
|
manifests:
|
||||||
|
rawYaml:
|
||||||
|
- k8s/dev/logmower.yaml
|
||||||
|
# - k8s/dev/mongodb.yaml
|
Loading…
Reference in New Issue