Add missing bits

.gitignore

@@ -1 +1,2 @@
+.overnodebundle
 .env

README.md

@@ -0,0 +1,54 @@
+# Mongo logger
+
+This is Mongo backed logging system that makes use of the `fluentd` driver found in Docker.
+
+
+# Running in production
+
+To collect container logs on a Docker host
+populate `.env` with `MONGO_URI` and
+deploy `docker-compose.yml` on all Docker hosts.
+
+In Docker host adjust `/etc/docker/daemon.json`
+
+```
+{
+  "log-driver": "fluentd",
+  "log-opts": {
+    "fluentd-address": "127.0.0.1:24224",
+    "fluentd-sub-second-precision": true
+  }
+}
+```
+
+Proceed to reload Docker daemon.
+
+Enable syslog forwarding:
+
+```
+echo '*.* @127.0.0.1:5140' | tee /etc/rsyslog.d/fwd.conf
+systemctl restart rsyslog
+```
+
+
+# Dumping logs
+
+To dump logs on console:
+
+```
+pip3 install motor termcolor
+source .env
+export MONGO_URI
+python3 tailer.py
+```
+
+
+# Why not Loki?
+
+Loki is ridiculously complex to set up for HA setup.
+
+
+# Why not Fluentd?
+
+After spending great deal of time working around bugs and design flaws in Fluentd
+we gave up. For reasoning see [https://github.com/fluent/fluentd/pull/2808](here)

docker-compose.yml

@@ -4,8 +4,6 @@ services:
   app:
     restart: always
     image: kspaceee/mongo-logger
-    build:
-      context: .
     network_mode: host
     env_file: .env
     logging:

overnode.yml

@@ -0,0 +1,6 @@
+id: logger
+
+version: 3.7
+
+logger:
+    docker-compose.yml: 1,2,3

tailer.py

@@ -0,0 +1,53 @@
+import argparse
+import asyncio
+import re
+import os
+import sys
+from motor.motor_asyncio import AsyncIOMotorClient
+from termcolor import colored
+
+MONGO_URI = os.getenv("MONGO_URI")
+
+if not MONGO_URI:
+    print("MONGO_URI not specified")
+    sys.exit(1)
+
+db = AsyncIOMotorClient(MONGO_URI).get_default_database()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--include-container', action='append', default=[])
+parser.add_argument('--exclude-container', action='append', default=[])
+parser.add_argument('--exclude-host', action='append', default=[])
+args = parser.parse_args()
+
+async def main():
+    async with db.log.watch([{'$match': {'operationType': 'insert'}}]) as stream:
+        print("Connected")
+        async for event in stream:
+            doc = event["fullDocument"]
+            host = doc.get("host", "-")
+            if host in args.exclude_host:
+                continue
+            container_name = doc.get("container_name", "/-")[1:]
+            if args.include_container and container_name not in args.include_container:
+                continue
+            if container_name in args.exclude_container:
+                continue
+
+            msg = doc.get("message")
+            if msg:
+                container_name = "system:%s" % doc.get("ident")
+            else:
+                msg = doc.get("log")
+
+            # TODO: Do this mangling in fluentd.conf
+            m = re.match("time=\".*?\" level=[a-z]+ msg=\"(.*?)\"", msg)
+            if m:
+                msg, = m.groups()
+            m = re.match("\[\d+.*? \d+\] (.*?)\"", msg)
+            if m:
+                msg, = m.groups()
+            print(doc["time"], colored(doc.get("host", "-"), "blue"), "\t", "% 30s" % colored(container_name, "yellow"), msg)
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(main())