Implement frameskip and add log viewer

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "log-viewer"]
+	path = log-viewer
+	url = https://git.k-space.ee/k-space/log-viewer.git
+	branch = wip

camdetect.py

@@ -31,6 +31,7 @@ MONGO_COLLECTION = os.getenv("MONGO_COLLETION", "eventlog")
 SOURCE_NAME = os.environ["SOURCE_NAME"]
 SLIDE_WINDOW = 2
 DCT_BLOCK_SIZE = 8
+UPLOAD_FRAMESKIP = 3
 
 # Percentage of blocks active to consider movement in whole frame
 THRESHOLD_RATIO = int(os.getenv("THRESHOLD_RATIO", "5"))
@@ -193,6 +194,11 @@ async def motion_detector(reference_frame, download_queue, upload_queue):
     """
     event_id = None
     differing_blocks = []
+    uploads_skipped = 0
+
+    # Hold queue keeps frames that we have before motion event start timestamp
+    hold_queue = asyncio.Queue(2 ** SLIDE_WINDOW)
+
     while True:
         dt, blob, dct, thumb = await download_queue.get()
         app.ctx.last_frame, app.ctx.dct = blob, dct
@@ -250,14 +256,32 @@ async def motion_detector(reference_frame, download_queue, upload_queue):
             app.ctx.event_id = event_id = result.inserted_id
             gauge_event_active.set(1)
 
+        # Handle buffering frames prior event start
+        if hold_queue.full():
+            await hold_queue.get()
+        hold_queue.put_nowait((blob, thumb))
+
         # Handle image upload
         if motion_detected and event_id:
             counter_movement_frames.inc()
-            try:
+            while True:
-                # Push JPEG blob into upload queue
+                if not uploads_skipped:
-                upload_queue.put_nowait((blob, thumb, event_id))
+                    uploads_skipped = UPLOAD_FRAMESKIP
-            except asyncio.QueueFull:
+                else:
-                counter_upload_dropped_frames.inc()
+                    uploads_skipped -= 1
+                    continue
+
+                # Drain queue of frames prior event start
+                try:
+                    blob, thumb = hold_queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    break
+
+                try:
+                    # Push JPEG blob into upload queue
+                    upload_queue.put_nowait((blob, thumb, event_id))
+                except asyncio.QueueFull:
+                    counter_upload_dropped_frames.inc()
             gauge_upload_queue_size.set(upload_queue.qsize())
 
         # Handle event end
@@ -303,7 +327,7 @@ def generate_thumbnail(dct):
         np.array(
             (tr.repeat(2, 1).repeat(2, 0) >> 3) + 127, dtype=np.uint8)[:dm]))
     _, jpeg = cv2.imencode(".jpg",
-                           cv2.cvtColor(m, cv2.COLOR_YCrCb2RGB),
+                           cv2.cvtColor(m, cv2.COLOR_YCrCb2BGR),
                            (cv2.IMWRITE_JPEG_QUALITY, 80))
     return jpeg
 
@@ -379,6 +403,7 @@ app = Sanic("camdetect")
 async def bypass_stream_wrapper(request):
     # Desired frame interval, by default 500ms
     interval = float(request.args.get("interval", 500)) / 1000.0
+
     async def stream_camera(response):
         ts = 0
         while True:

docker-compose.yml

@@ -1,6 +1,21 @@
 version: '3.7'
 
 # All keys here are for dev instance only, do not put prod keys here
+x-common: &common
+  AWS_ACCESS_KEY_ID: camdetect
+  AWS_SECRET_ACCESS_KEY: 2mSI6HdbJ8
+  ME_CONFIG_MONGODB_ENABLE_ADMIN: 'true'
+  ME_CONFIG_MONGODB_SERVER: '127.0.0.1'
+  ME_CONFIG_MONGODB_AUTH_DATABASE: admin
+  MINIO_ACCESS_KEY: camdetect
+  MINIO_SECRET_KEY: 2mSI6HdbJ8
+  MINIO_DEFAULT_BUCKETS: camdetect
+  MINIO_URI: 'http://camdetect:2mSI6HdbJ8@127.0.0.1:9000/camdetect'
+  S3_ENDPOINT_URL: http://127.0.0.1:9000
+  MINIO_DEFAULT_BUCKETS: camdetect
+  MINIO_CONSOLE_PORT_NUMBER: 9001
+  MJPEGSTREAMER_CREDENTIALS: user:123456
+  SOURCE_NAME: dummy
 
 services:
   camdetect:
@@ -10,20 +25,13 @@ services:
       context: .
     entrypoint: /app/camdetect.py
     command: http://user:123456@127.0.0.1:8080?action=stream
-    environment:
+    environment: *common
-      - MJPEGSTREAMER_CREDENTIALS=user:123456
-      - AWS_SECRET_ACCESS_KEY=2mSI6HdbJ8
-      - S3_ENDPOINT_URL=http://127.0.0.1:9000
-      - SOURCE_NAME=dummy
 
   mongoexpress:
     restart: always
     image: mongo-express
     network_mode: host
-    environment:
+    environment: *common
-      - ME_CONFIG_MONGODB_ENABLE_ADMIN=true
-      - ME_CONFIG_MONGODB_SERVER=127.0.0.1
-      - ME_CONFIG_MONGODB_AUTH_DATABASE=admin
     logging:
       driver: none
 
@@ -50,11 +58,7 @@ services:
     restart: always
     network_mode: host
     image: bitnami/minio:latest
-    environment:
+    environment: *common
-      - MINIO_ACCESS_KEY=camdetect
-      - MINIO_SECRET_KEY=2mSI6HdbJ8
-      - MINIO_DEFAULT_BUCKETS=camdetect
-      - MINIO_CONSOLE_PORT_NUMBER=9001
     logging:
       driver: none
 
@@ -64,4 +68,17 @@ services:
     image: kvaps/mjpg-streamer
     devices:
       - /dev/video0
-    command: -i "/usr/lib64/input_uvc.so -y -d /dev/video0 -r 1280x720 -f 30" -o "output_http.so -c user:123456"
+    command: -i "/usr/lib64/input_uvc.so -y -d /dev/video0 -r 1280x720 -f 5" -o "output_http.so -c user:123456"
+
+  log-viewer-backend:
+    restart: always
+    network_mode: host
+    build:
+      context: ./log-viewer/backend
+    environment: *common
+
+  log-viewer-frontend:
+    restart: always
+    network_mode: host
+    build:
+      context: ./log-viewer/frontend

log-viewer

@@ -0,0 +1 @@
+Subproject commit b8e9f03b86ff63a5cdc77bae18caf7d7efd077b9