diff --git a/.gitignore b/.gitignore index 4c49bd7..85f79cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +.overnodebundle .env diff --git a/README.md b/README.md new file mode 100644 index 0000000..2596521 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# Mongo logger + +This is Mongo backed logging system that makes use of the `fluentd` driver found in Docker. + + +# Running in production + +To collect container logs on a Docker host +populate `.env` with `MONGO_URI` and +deploy `docker-compose.yml` on all Docker hosts. + +In Docker host adjust `/etc/docker/daemon.json` + +``` +{ + "log-driver": "fluentd", + "log-opts": { + "fluentd-address": "127.0.0.1:24224", + "fluentd-sub-second-precision": true + } +} +``` + +Proceed to reload Docker daemon. + +Enable syslog forwarding: + +``` +echo '*.* @127.0.0.1:5140' | tee /etc/rsyslog.d/fwd.conf +systemctl restart rsyslog +``` + + +# Dumping logs + +To dump logs on console: + +``` +pip3 install motor termcolor +source .env +export MONGO_URI +python3 tailer.py +``` + + +# Why not Loki? + +Loki is ridiculously complex to set up for HA setup. + + +# Why not Fluentd? + +After spending great deal of time working around bugs and design flaws in Fluentd +we gave up. For reasoning see [https://github.com/fluent/fluentd/pull/2808](here) diff --git a/docker-compose.yml b/docker-compose.yml index 28eb343..21eb100 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,6 @@ services: app: restart: always image: kspaceee/mongo-logger - build: - context: . network_mode: host env_file: .env logging: diff --git a/overnode.yml b/overnode.yml new file mode 100644 index 0000000..4ad51cc --- /dev/null +++ b/overnode.yml @@ -0,0 +1,6 @@ +id: logger + +version: 3.7 + +logger: + docker-compose.yml: 1,2,3 diff --git a/tailer.py b/tailer.py new file mode 100644 index 0000000..6f29e23 --- /dev/null +++ b/tailer.py @@ -0,0 +1,53 @@ +import argparse +import asyncio +import re +import os +import sys +from motor.motor_asyncio import AsyncIOMotorClient +from termcolor import colored + +MONGO_URI = os.getenv("MONGO_URI") + +if not MONGO_URI: + print("MONGO_URI not specified") + sys.exit(1) + +db = AsyncIOMotorClient(MONGO_URI).get_default_database() + +parser = argparse.ArgumentParser() +parser.add_argument('--include-container', action='append', default=[]) +parser.add_argument('--exclude-container', action='append', default=[]) +parser.add_argument('--exclude-host', action='append', default=[]) +args = parser.parse_args() + +async def main(): + async with db.log.watch([{'$match': {'operationType': 'insert'}}]) as stream: + print("Connected") + async for event in stream: + doc = event["fullDocument"] + host = doc.get("host", "-") + if host in args.exclude_host: + continue + container_name = doc.get("container_name", "/-")[1:] + if args.include_container and container_name not in args.include_container: + continue + if container_name in args.exclude_container: + continue + + msg = doc.get("message") + if msg: + container_name = "system:%s" % doc.get("ident") + else: + msg = doc.get("log") + + # TODO: Do this mangling in fluentd.conf + m = re.match("time=\".*?\" level=[a-z]+ msg=\"(.*?)\"", msg) + if m: + msg, = m.groups() + m = re.match("\[\d+.*? \d+\] (.*?)\"", msg) + if m: + msg, = m.groups() + print(doc["time"], colored(doc.get("host", "-"), "blue"), "\t", "% 30s" % colored(container_name, "yellow"), msg) + +loop = asyncio.get_event_loop() +loop.run_until_complete(main())