From 1cb1bdfbd8bb479396c5c1d3853124f5c48b03b3 Mon Sep 17 00:00:00 2001
From: rasmus <rasmus@k-space.ee>
Date: Fri, 11 Nov 2022 21:12:31 +0200
Subject: [PATCH] continue work on vars

---
 pkg/file/file.go   |  2 +-
 pkg/watcher/app.go | 23 +++++++++++++----------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/pkg/file/file.go b/pkg/file/file.go
index 5641059..fdaac0a 100644
--- a/pkg/file/file.go
+++ b/pkg/file/file.go
@@ -19,7 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
-const SendQueueLimit = 1024
+var SendQueueLimit = 1024
 
 // wrapper to force copying before use
 func backoff() wait.Backoff {
diff --git a/pkg/watcher/app.go b/pkg/watcher/app.go
index 737e23a..67d99ab 100644
--- a/pkg/watcher/app.go
+++ b/pkg/watcher/app.go
@@ -25,28 +25,31 @@ var App = &cli.App{
 	Description: "Collect and ship kubernetes logs",
 	// TODO: #2: yaml
 	Flags: []cli.Flag{
-		&cli.BoolFlag{Name: "simulate", Aliases: []string{"dry-run"}, Usage: "Do not write to database"},
+		// in Action
 		&cli.StringFlag{Name: "log-directory", Usage: "Directory to watch for logs", Value: "/var/log/containers"},
+		&cli.Uint64Flag{Name: "max-connection-pool-size", EnvVars: []string{"MAX_CONNECTION_POOL_SIZE"}, Value: 1, Usage: "Max MongoDB connection pool size"},
+		// in Before
+		&cli.BoolFlag{Name: "simulate", Aliases: []string{"dry-run"}, Usage: "Do not write to database"},
 		&cli.IntFlag{Name: "max-record-size", EnvVars: []string{"MAX_RECORD_SIZE"}, Value: 128 * 1024, Usage: "Maximum record size in bytes"},
 		&cli.IntFlag{Name: "bulk-insertion-size", EnvVars: []string{"BULK_INSERTION_SIZE"}, Value: 1000, Usage: "MongoDB bulk insertion size in records"},
-		&cli.Uint64Flag{Name: "max-connection-pool-size", EnvVars: []string{"MAX_CONNECTION_POOL_SIZE"}, Value: 1, Usage: "Max MongoDB connection pool size"},
-		//
-		//TODO: &cli.BoolFlag{Name: "normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords"},
-		//TODO: &cli.BoolFlag{Name: "parse-json"},
+		&cli.IntFlag{Name: "max-upload-queue-size", EnvVars: []string{"MAX_UPLOAD_QUEUE_SIZE"}, Value: 1024, Usage: "Max upload queue size (before batching) in records"},
 		//
+		//TODO: &cli.BoolFlag{Name: "heuristic-normalize-log-level", Usage: "Normalize log.level values to Syslog defined keywords", Value: false},
+		//TODO: &cli.BoolFlag{Name: "heuristic-parse-json", Usage: "Attempt automatically unwrapping JSON records", Value: false},
+
 		&cli.StringSliceFlag{Category: "selectors", Name: "namespace", EnvVars: []string{"NAMESPACE"}, Usage: "whitelist filter for filenames"},
 		&cli.StringSliceFlag{Category: "selectors", Name: "exclude-pod-prefixes", EnvVars: []string{"EXCLUDE_POD_PREFIXES"}, Usage: "blacklist filter for filenames", Value: cli.NewStringSlice("logmower-")},
 		//
 		&cli.StringFlag{Category: "secrets", Name: "mongo-uri", EnvVars: []string{"MONGODB_URI"}, Usage: "mongodb://foo:bar@host:27017/database", Required: true},
 	},
 	Before: func(ctx *cli.Context) error {
-		lines.BufferLimitBytes = ctx.Int("max-record-size")
-		if lines.BufferLimitBytes < 1 {
-			return fmt.Errorf("max-record-size must be positive")
-		}
-
 		sender.Simulate = ctx.Bool("simulate")
+		lines.BufferLimitBytes = ctx.Int("max-record-size")
 		sender.MaxBatchItems = ctx.Int("bulk-insertion-size")
+		if sender.MaxBatchItems < 1 {
+			return fmt.Errorf("bulk-insertion-size minimum is 1")
+		}
+		file.SendQueueLimit = ctx.Int("max-upload-queue-size")
 
 		return nil
 	},