k-space/logmower-shipper
10
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Activity

mongo: configuration, sync with py args

Browse Source 
+ tidy
This commit is contained in:
rasmus 2022-11-12 15:27:14 +02:00
parent 6f6ce23b37
commit 1c31b19fd2
2 changed files with 39 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
pkg/mongo/mongo.go
View File

@ -17,7 +17,18 @@ func GlobalTimeout(ctx context.Context) context.Context {
return ctx
}
func Initialize(ctx context.Context, uri string, opts *mongoOpt.ClientOptions) (*mongo.Collection, error) {
type InitializeOptions struct {
MaxPoolSize uint64
CapSizeBytes int64
ExpireAfterSeconds int64
}
func Initialize(ctx context.Context, uri string, opt *InitializeOptions) (*mongo.Collection, error) {
collectionName := "logs"
if opt == nil {
opt = &InitializeOptions{}
}
uriParsed, err := url.ParseRequestURI(uri)
if err != nil {
return nil, fmt.Errorf("parsing URI for database name: %w", err)
@ -28,7 +39,7 @@ func Initialize(ctx context.Context, uri string, opts *mongoOpt.ClientOptions) (
return nil, fmt.Errorf("URI must include database name (as database to authenticate against)")
}
dbOpt := attachMetrics(opts).ApplyURI(uri)
dbOpt := attachMetrics(mongoOpt.Client()).ApplyURI(uri).SetMaxPoolSize(opt.MaxPoolSize)
dbClient, err := mongo.Connect(GlobalTimeout(ctx), dbOpt)
if err != nil {
@ -39,7 +50,19 @@ func Initialize(ctx context.Context, uri string, opts *mongoOpt.ClientOptions) (
return nil, fmt.Errorf("first ping to database: %w", err)
}
col := dbClient.Database(uriParsed.Path).Collection("logs")
db := dbClient.Database(uriParsed.Path)
capped := opt.CapSizeBytes > 0
if err := db.CreateCollection(GlobalTimeout(ctx), collectionName, &mongoOpt.CreateCollectionOptions{
Capped: &capped,
SizeInBytes: &opt.CapSizeBytes,
ExpireAfterSeconds: &opt.ExpireAfterSeconds,
}); err != nil {
return nil, fmt.Errorf("initializing collection")
}
col := db.Collection(collectionName)
if err := InitializeIndexes(GlobalTimeout(ctx), col); err != nil {
return nil, fmt.Errorf("initializing indexes: %w", err)

23
pkg/watcher/app.go
View File

@ -14,7 +14,6 @@ import (
"git.k-space.ee/k-space/logmower-shipper/pkg/util"
"github.com/fsnotify/fsnotify"
"github.com/urfave/cli/v2"
mongoOpt "go.mongodb.org/mongo-driver/mongo/options"
)
var App = &cli.App{
@ -27,20 +26,22 @@ var App = &cli.App{
Flags: []cli.Flag{
// in Action
&cli.StringFlag{Name: "log-directory", Usage: "Directory to watch for logs", Value: "/var/log/containers"},
&cli.Uint64Flag{Name: "max-connection-pool-size", EnvVars: []string{"MAX_CONNECTION_POOL_SIZE"}, Value: 1, Usage: "Max MongoDB connection pool size"},
&cli.Uint64Flag{Category: "mongo", Name: "max-connection-pool-size", EnvVars: []string{"MAX_CONNECTION_POOL_SIZE"}, Value: 1, Usage: "Max MongoDB connection pool size"},
&cli.Int64Flag{Category: "mongo", Name: "max-collection-size", EnvVars: []string{"MAX_COLLECTION_SIZE"}, Usage: "MongoDB collection size limit in bytes"},
&cli.Int64Flag{Category: "mongo", Name: "max-record-retention", EnvVars: []string{"MAX_RECORD_RETENTION"}, Usage: "Record retention in seconds"},
&cli.StringFlag{Category: "mongo", Name: "mongo-uri", EnvVars: []string{"MONGODB_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
// in Before
&cli.BoolFlag{Name: "simulate", Aliases: []string{"dry-run"}, Usage: "Do not write to database"},
&cli.BoolFlag{Category: "mongo", Name: "simulate", Aliases: []string{"dry-run"}, Usage: "Writes to database are simulate as successful"},
&cli.IntFlag{Name: "max-record-size", EnvVars: []string{"MAX_RECORD_SIZE"}, Value: 128 * 1024, Usage: "Maximum record size in bytes"},
&cli.IntFlag{Name: "bulk-insertion-size", EnvVars: []string{"BULK_INSERTION_SIZE"}, Value: 1000, Usage: "MongoDB bulk insertion size in records"},
&cli.IntFlag{Name: "max-upload-queue-size", EnvVars: []string{"MAX_UPLOAD_QUEUE_SIZE"}, Value: 1024, Usage: "Max upload queue size (before batching) in records"},
//
//TODO: &cli.BoolFlag{Name: "heuristic-normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords", Value: false},
//TODO: &cli.BoolFlag{Name: "heuristic-parse-json", Usage: "Attempt automatically unwrapping JSON records", Value: false},
// TODO: &cli.BoolFlag{Category: "parsing", Name: "heuristic-normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords", Value: false},
// TODO: &cli.BoolFlag{Category: "parsing", Name: "heuristic-parse-json", Usage: "Attempt automatically unwrapping JSON records", Value: false},
&cli.StringSliceFlag{Category: "selectors", Name: "namespace", EnvVars: []string{"NAMESPACE"}, Usage: "whitelist filter for filenames"},
&cli.StringSliceFlag{Category: "selectors", Name: "exclude-pod-prefixes", EnvVars: []string{"EXCLUDE_POD_PREFIXES"}, Usage: "blacklist filter for filenames", Value: cli.NewStringSlice("logmower-")},
//
&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGODB_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
},
Before: func(ctx *cli.Context) error {
sender.Simulate = ctx.Bool("simulate")
@ -60,8 +61,11 @@ var App = &cli.App{
log.Printf("%s %s starting", ctx.App.Name, ctx.App.Version)
db, err := m.Initialize(ctx.Context, ctx.String("mongo-uri"), mongoOpt.Client().
SetMaxPoolSize(ctx.Uint64("max-connection-pool-size")))
db, err := m.Initialize(ctx.Context, ctx.String("mongo-uri"), &m.InitializeOptions{
MaxPoolSize: ctx.Uint64("max-connection-pool-size"),
CapSizeBytes: ctx.Int64("max-collection-size"),
ExpireAfterSeconds: ctx.Int64("max-record-retention"),
})
if err != nil {
return fmt.Errorf("initializing database connection: %w", err)
}
@ -95,7 +99,6 @@ var App = &cli.App{
continue
}
// TODO: #1: || if not in filterset
kubeInfo, ok := util.ParseLogFilename(event.Name)
if !ok {
promWatcherFilesSkipped.Add(1)