logmower-shipper/pkg/watcher/app.go

160 lines
5.1 KiB
Go
Raw Normal View History

2022-11-09 16:07:28 +00:00
package watcher
import (
"fmt"
"log"
2022-11-06 01:43:18 +00:00
"path/filepath"
"sync"
2022-11-09 16:07:28 +00:00
"git.k-space.ee/k-space/logmower-shipper/pkg/file"
"git.k-space.ee/k-space/logmower-shipper/pkg/globals"
2022-11-11 15:09:12 +00:00
"git.k-space.ee/k-space/logmower-shipper/pkg/lines"
2022-11-09 16:07:28 +00:00
m "git.k-space.ee/k-space/logmower-shipper/pkg/mongo"
2022-11-11 15:09:12 +00:00
"git.k-space.ee/k-space/logmower-shipper/pkg/sender"
2022-11-09 16:07:28 +00:00
"git.k-space.ee/k-space/logmower-shipper/pkg/util"
"github.com/fsnotify/fsnotify"
"github.com/urfave/cli/v2"
2022-11-11 15:09:12 +00:00
mongoOpt "go.mongodb.org/mongo-driver/mongo/options"
)
var App = &cli.App{
2022-11-09 16:07:28 +00:00
Name: globals.AppName,
Version: "1.0.0",
2022-11-09 16:07:28 +00:00
Authors: []*cli.Author{{Name: "jtagcat"}, {Name: "codemowers.io"}},
Description: "Collect and ship kubernetes logs",
2022-11-03 10:42:59 +00:00
// TODO: #2: yaml
Flags: []cli.Flag{
2022-11-11 19:12:31 +00:00
// in Action
&cli.StringFlag{Name: "log-directory", Usage: "Directory to watch for logs", Value: "/var/log/containers"},
2022-11-11 19:12:31 +00:00
&cli.Uint64Flag{Name: "max-connection-pool-size", EnvVars: []string{"MAX_CONNECTION_POOL_SIZE"}, Value: 1, Usage: "Max MongoDB connection pool size"},
// in Before
&cli.BoolFlag{Name: "simulate", Aliases: []string{"dry-run"}, Usage: "Do not write to database"},
2022-11-11 15:09:12 +00:00
&cli.IntFlag{Name: "max-record-size", EnvVars: []string{"MAX_RECORD_SIZE"}, Value: 128 * 1024, Usage: "Maximum record size in bytes"},
&cli.IntFlag{Name: "bulk-insertion-size", EnvVars: []string{"BULK_INSERTION_SIZE"}, Value: 1000, Usage: "MongoDB bulk insertion size in records"},
2022-11-11 19:12:31 +00:00
&cli.IntFlag{Name: "max-upload-queue-size", EnvVars: []string{"MAX_UPLOAD_QUEUE_SIZE"}, Value: 1024, Usage: "Max upload queue size (before batching) in records"},
2022-11-09 18:45:44 +00:00
//
2022-11-11 19:12:31 +00:00
//TODO: &cli.BoolFlag{Name: "heuristic-normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords", Value: false},
//TODO: &cli.BoolFlag{Name: "heuristic-parse-json", Usage: "Attempt automatically unwrapping JSON records", Value: false},
2022-11-11 15:09:12 +00:00
&cli.StringSliceFlag{Category: "selectors", Name: "namespace", EnvVars: []string{"NAMESPACE"}, Usage: "whitelist filter for filenames"},
&cli.StringSliceFlag{Category: "selectors", Name: "exclude-pod-prefixes", EnvVars: []string{"EXCLUDE_POD_PREFIXES"}, Usage: "blacklist filter for filenames", Value: cli.NewStringSlice("logmower-")},
2022-11-09 18:13:50 +00:00
//
2022-11-09 19:55:54 +00:00
&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGODB_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
},
2022-11-06 20:02:29 +00:00
Before: func(ctx *cli.Context) error {
2022-11-11 15:09:12 +00:00
sender.Simulate = ctx.Bool("simulate")
2022-11-11 19:12:31 +00:00
lines.BufferLimitBytes = ctx.Int("max-record-size")
2022-11-11 15:09:12 +00:00
sender.MaxBatchItems = ctx.Int("bulk-insertion-size")
2022-11-11 19:12:31 +00:00
if sender.MaxBatchItems < 1 {
return fmt.Errorf("bulk-insertion-size minimum is 1")
}
file.SendQueueLimit = ctx.Int("max-upload-queue-size")
2022-11-06 20:02:29 +00:00
return nil
},
Action: func(ctx *cli.Context) error {
2022-11-11 15:09:12 +00:00
whitelistNamespaces, blacklistPodPrefixes := sliceToMap(ctx.StringSlice("namespace")), ctx.StringSlice("exclude-pod-prefixes")
var wg sync.WaitGroup
2022-11-06 00:16:54 +00:00
log.Printf("%s %s starting", ctx.App.Name, ctx.App.Version)
2022-11-11 15:09:12 +00:00
db, err := m.Initialize(ctx.Context, ctx.String("mongo-uri"), mongoOpt.Client().
SetMaxPoolSize(ctx.Uint64("max-connection-pool-size")))
2022-11-06 14:52:22 +00:00
if err != nil {
return fmt.Errorf("initializing database connection: %w", err)
2022-11-06 14:52:22 +00:00
}
2022-11-09 16:07:28 +00:00
hostinfo, err := util.Hostinfo(ctx.String("node-name"))
if err != nil {
return fmt.Errorf("populating host info: %w", err)
}
watcher, err := fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("initializing log directory watcher: %w", err)
}
defer watcher.Close()
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-ctx.Context.Done():
return
case event, ok := <-watcher.Events:
if !ok {
return
}
promWatcherEvents.Add(1)
if event.Op != fsnotify.Create {
continue
}
// TODO: #1: || if not in filterset
2022-11-09 16:07:28 +00:00
kubeInfo, ok := util.ParseLogFilename(event.Name)
if !ok {
promWatcherFilesSkipped.Add(1)
log.Printf("skipped %q: filename not parsable in kubernetes log format", filepath.Base(event.Name))
continue
}
2022-11-09 19:15:17 +00:00
if _, ok := whitelistNamespaces[kubeInfo.Namespace]; !ok {
2022-11-09 19:09:15 +00:00
continue
}
2022-11-09 19:15:17 +00:00
if ok := hasSlicePrefix(kubeInfo.Pod, blacklistPodPrefixes); ok {
2022-11-09 19:09:15 +00:00
continue
}
promWatcherFilesStarted.Add(1)
wg.Add(1)
go func() {
2022-11-09 16:07:28 +00:00
file := file.File{
File: &m.File{
Host: &hostinfo,
KubeInfo: kubeInfo,
Path: event.Name,
},
2022-11-09 16:07:28 +00:00
MetricsName: filepath.Base(event.Name),
}
file.Process(ctx.Context, db)
wg.Done()
}()
case err, ok := <-watcher.Errors:
if !ok {
return
}
promWatcherErr.Add(1)
log.Printf("watching for new logs: %e", err)
}
}
}()
logDir := ctx.String("log-directory")
2022-11-06 02:04:32 +00:00
// simulate create events to pick up files already created
if err := simulateInitialCreates(logDir, watcher.Events); err != nil {
return fmt.Errorf("listing log directory %q: %w", logDir, err)
2022-11-06 02:04:32 +00:00
}
2022-11-06 01:43:18 +00:00
if err := watcher.Add(logDir); err != nil {
return fmt.Errorf("watching for new logs in %q: %w", logDir, err)
}
promWatcherOnline.Set(1)
// waiting indefinitely for interrupt
wg.Wait() // wait for watch and file processors to cleanup
return ctx.Err()
},
}