From 33f3e6171cdb49b30b684ac791cda35d51bfa204 Mon Sep 17 00:00:00 2001 From: songmeo Date: Tue, 23 Mar 2021 09:34:08 +0000 Subject: [PATCH 1/6] add dev.yml --- dev.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 dev.yml diff --git a/dev.yml b/dev.yml new file mode 100644 index 0000000..1c3e4c9 --- /dev/null +++ b/dev.yml @@ -0,0 +1,26 @@ +version: '3.7' + +services: + mongoexpress: + image: mongo-express + network_mode: host + environment: + - ME_CONFIG_MONGODB_ENABLE_ADMIN=true + - ME_CONFIG_MONGODB_SERVER=127.0.0.1 + - ME_CONFIG_MONGODB_PORT=27017 + - ME_CONFIG_MONGODB_AUTH_DATABASE=admin + mongo: + network_mode: host + restart: unless-stopped + image: mongo:latest + environment: + - MONGODB_INITDB_ROOT_USERNAME=doorboy + - MONGODB_INITDB_ROOT_PASSWORD=password + - MONGODB_INITDB_DATABASE=kspace_accounting + doorboy_proxy: + hostname: doorboy.infra.k-space.ee + restart: unless-stopped + network_mode: host + env_file: .env + build: + context: . From 35cc25848c91a9de3935ee25682acf97287b285a Mon Sep 17 00:00:00 2001 From: songmeo Date: Wed, 31 Mar 2021 10:23:48 +0300 Subject: [PATCH 2/6] change db to get_default_database --- const.py | 2 ++ main.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 const.py diff --git a/const.py b/const.py new file mode 100644 index 0000000..24b2f0e --- /dev/null +++ b/const.py @@ -0,0 +1,2 @@ +import os +MONGO_URI = os.getenv("MONGO_URI", "mongodb://127.0.0.1:27017/kspace_accounting") diff --git a/main.py b/main.py index a42013f..b27d8b9 100644 --- a/main.py +++ b/main.py @@ -8,9 +8,12 @@ import json import os import pymongo import smtplib +import const app = Flask(__name__) -mongodb = MongoClient('mongodb://172.21.27.1,172.21.27.2,172.21.27.3:27017/', replicaSet="kspace-mongo-set").kspace_accounting +#mongodb = MongoClient('mongodb://172.21.27.1,172.21.27.2,172.21.27.3:27017/', replicaSet="kspace-mongo-set").kspace_accounting +mongodb = MongoClient(const.MONGO_URI) +mongodb = mongodb.get_default_database() mongodb.authenticate("kspace_accounting", os.environ["MONGO_PASSWORD"]) DOORBOY_SECRET = os.environ["DOORBOY_SECRET"] From 9adf2695284376c043848c10ea6adba359eafb18 Mon Sep 17 00:00:00 2001 From: songmeo Date: Wed, 31 Mar 2021 20:59:20 +0300 Subject: [PATCH 3/6] rm db authentication --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index b27d8b9..29025a3 100644 --- a/main.py +++ b/main.py @@ -14,7 +14,6 @@ app = Flask(__name__) #mongodb = MongoClient('mongodb://172.21.27.1,172.21.27.2,172.21.27.3:27017/', replicaSet="kspace-mongo-set").kspace_accounting mongodb = MongoClient(const.MONGO_URI) mongodb = mongodb.get_default_database() -mongodb.authenticate("kspace_accounting", os.environ["MONGO_PASSWORD"]) DOORBOY_SECRET = os.environ["DOORBOY_SECRET"] From 661a704d8f9db85ec7a0a02ad2565db6700c85dd Mon Sep 17 00:00:00 2001 From: songmeo Date: Fri, 2 Apr 2021 09:10:34 +0000 Subject: [PATCH 4/6] change to sanic and motor --- main.py | 38 +++++++++++++++++++------------------- requirements.txt | 2 ++ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/main.py b/main.py index 29025a3..26785b4 100644 --- a/main.py +++ b/main.py @@ -1,18 +1,20 @@ from datetime import datetime -from flask import Flask, request, redirect, render_template, Blueprint, make_response, jsonify -from pymongo import MongoClient -import flask +from sanic import Sanic, response +import uvloop, asyncio +from motor.motor_asyncio import AsyncIOMotorClient +import pymongo import hashlib import jinja2 import json import os -import pymongo import smtplib import const -app = Flask(__name__) +app = Sanic(__name__) +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) # swap default event loop to uvloop + #mongodb = MongoClient('mongodb://172.21.27.1,172.21.27.2,172.21.27.3:27017/', replicaSet="kspace-mongo-set").kspace_accounting -mongodb = MongoClient(const.MONGO_URI) +mongodb = AsyncIOMotorClient(const.MONGO_URI) mongodb = mongodb.get_default_database() DOORBOY_SECRET = os.environ["DOORBOY_SECRET"] @@ -20,25 +22,25 @@ DOORBOY_SECRET = os.environ["DOORBOY_SECRET"] assert len(DOORBOY_SECRET) > 10 @app.route("/allowed") -def view_doorboy_uids(): +async def view_doorboy_uids(request): if request.headers.get('KEY') != DOORBOY_SECRET: return "how about no" - allowed_names = [o["_id"] for o in mongodb.member.find({"enabled": True})] + allowed_names = [o["_id"] for o in await mongodb.member.find({"enabled": True}).to_list()] allowed_uids = [] - for obj in mongodb.inventory.find({"token.uid_hash": {"$exists":True}, "inventory.owner_id": {"$exists":True}, "token.enabled": True}, {"inventory.owner_id": True, "token.uid_hash": True }): + for obj in await mongodb.inventory.find({"token.uid_hash": {"$exists":True}, "inventory.owner_id": {"$exists":True}, "token.enabled": True}, {"inventory.owner_id": True, "token.uid_hash": True }): if obj["inventory"].pop("owner_id") in allowed_names: - obj.pop("_id") - obj.pop("inventory") + del obj["_id"] + del obj["inventory"] allowed_uids.append(obj) - return jsonify(allowed_uids=allowed_uids) + return response.json(allowed_uids=allowed_uids) @app.route("/longpoll") -def view_longpoll(): +async def view_longpoll(request): if request.headers.get('KEY') != DOORBOY_SECRET: return "how about no" - def g(): + async def g(): yield "data: response-generator-started\n\n" pipeline = [ { @@ -50,16 +52,14 @@ def view_longpoll(): } ] try: - with mongodb.eventlog.watch(pipeline) as stream: + async with mongodb.eventlog.watch(pipeline) as stream: yield "data: watch-stream-opened\n\n" - for event in stream: + async for event in stream: if event["fullDocument"].get("type") == "open-door": yield "data: %s\n\n" % event["fullDocument"]["door"] except pymongo.errors.PyMongoError: return "urror" - return flask.Response(g(), - mimetype="text/event-stream") + return stream(g, content_type="text/event-stream") if __name__ == '__main__': app.run(debug=False, host='::') - diff --git a/requirements.txt b/requirements.txt index 2df0789..5e3178d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ Flask Flask-WTF pymongo gunicorn +sanic +motor From c1cf91ee1dca14fcf9dd0c92a249e88dbd72b16c Mon Sep 17 00:00:00 2001 From: songmeo Date: Fri, 2 Apr 2021 09:17:39 +0000 Subject: [PATCH 5/6] rewrite with sanic, motor --- Dockerfile | 2 +- main.py | 38 +++++++++++++++++++------------------- requirements.txt | 2 ++ 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index f5320b1..2dda3fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,4 @@ COPY requirements.txt ./ RUN pip install -r requirements.txt COPY *.py ./ -CMD ["gunicorn", "--workers", "10", "--bind", "[::]:5000", "main:app", "--timeout", "90"] +CMD ["gunicorn", "--worker-class", "sanic.worker.GunicornWorker", "--workers", "10", "--bind", "[::]:5000", "main:app", "--timeout", "90"] diff --git a/main.py b/main.py index 29025a3..26785b4 100644 --- a/main.py +++ b/main.py @@ -1,18 +1,20 @@ from datetime import datetime -from flask import Flask, request, redirect, render_template, Blueprint, make_response, jsonify -from pymongo import MongoClient -import flask +from sanic import Sanic, response +import uvloop, asyncio +from motor.motor_asyncio import AsyncIOMotorClient +import pymongo import hashlib import jinja2 import json import os -import pymongo import smtplib import const -app = Flask(__name__) +app = Sanic(__name__) +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) # swap default event loop to uvloop + #mongodb = MongoClient('mongodb://172.21.27.1,172.21.27.2,172.21.27.3:27017/', replicaSet="kspace-mongo-set").kspace_accounting -mongodb = MongoClient(const.MONGO_URI) +mongodb = AsyncIOMotorClient(const.MONGO_URI) mongodb = mongodb.get_default_database() DOORBOY_SECRET = os.environ["DOORBOY_SECRET"] @@ -20,25 +22,25 @@ DOORBOY_SECRET = os.environ["DOORBOY_SECRET"] assert len(DOORBOY_SECRET) > 10 @app.route("/allowed") -def view_doorboy_uids(): +async def view_doorboy_uids(request): if request.headers.get('KEY') != DOORBOY_SECRET: return "how about no" - allowed_names = [o["_id"] for o in mongodb.member.find({"enabled": True})] + allowed_names = [o["_id"] for o in await mongodb.member.find({"enabled": True}).to_list()] allowed_uids = [] - for obj in mongodb.inventory.find({"token.uid_hash": {"$exists":True}, "inventory.owner_id": {"$exists":True}, "token.enabled": True}, {"inventory.owner_id": True, "token.uid_hash": True }): + for obj in await mongodb.inventory.find({"token.uid_hash": {"$exists":True}, "inventory.owner_id": {"$exists":True}, "token.enabled": True}, {"inventory.owner_id": True, "token.uid_hash": True }): if obj["inventory"].pop("owner_id") in allowed_names: - obj.pop("_id") - obj.pop("inventory") + del obj["_id"] + del obj["inventory"] allowed_uids.append(obj) - return jsonify(allowed_uids=allowed_uids) + return response.json(allowed_uids=allowed_uids) @app.route("/longpoll") -def view_longpoll(): +async def view_longpoll(request): if request.headers.get('KEY') != DOORBOY_SECRET: return "how about no" - def g(): + async def g(): yield "data: response-generator-started\n\n" pipeline = [ { @@ -50,16 +52,14 @@ def view_longpoll(): } ] try: - with mongodb.eventlog.watch(pipeline) as stream: + async with mongodb.eventlog.watch(pipeline) as stream: yield "data: watch-stream-opened\n\n" - for event in stream: + async for event in stream: if event["fullDocument"].get("type") == "open-door": yield "data: %s\n\n" % event["fullDocument"]["door"] except pymongo.errors.PyMongoError: return "urror" - return flask.Response(g(), - mimetype="text/event-stream") + return stream(g, content_type="text/event-stream") if __name__ == '__main__': app.run(debug=False, host='::') - diff --git a/requirements.txt b/requirements.txt index 2df0789..5e3178d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ Flask Flask-WTF pymongo gunicorn +sanic +motor From c9018003ecdbd208d0806a2af9503586cbbcfd64 Mon Sep 17 00:00:00 2001 From: songmeo Date: Sat, 3 Apr 2021 17:09:51 +0300 Subject: [PATCH 6/6] clean up requirements, small tweak --- main.py | 34 +++++++++++++++++----------------- requirements.txt | 3 --- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/main.py b/main.py index 26785b4..886d0fe 100644 --- a/main.py +++ b/main.py @@ -1,17 +1,13 @@ from datetime import datetime from sanic import Sanic, response -import uvloop, asyncio +from sanic.response import text, json, stream from motor.motor_asyncio import AsyncIOMotorClient +import uvloop, asyncio import pymongo -import hashlib -import jinja2 -import json import os -import smtplib import const app = Sanic(__name__) -asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) # swap default event loop to uvloop #mongodb = MongoClient('mongodb://172.21.27.1,172.21.27.2,172.21.27.3:27017/', replicaSet="kspace-mongo-set").kspace_accounting mongodb = AsyncIOMotorClient(const.MONGO_URI) @@ -24,24 +20,26 @@ assert len(DOORBOY_SECRET) > 10 @app.route("/allowed") async def view_doorboy_uids(request): if request.headers.get('KEY') != DOORBOY_SECRET: - return "how about no" - allowed_names = [o["_id"] for o in await mongodb.member.find({"enabled": True}).to_list()] + return text("how about no") + allowed_names = [] + async for obj in mongodb.member.find({"enabled": True}): + allowed_names.append(obj["_id"]) allowed_uids = [] - for obj in await mongodb.inventory.find({"token.uid_hash": {"$exists":True}, "inventory.owner_id": {"$exists":True}, "token.enabled": True}, {"inventory.owner_id": True, "token.uid_hash": True }): + async for obj in mongodb.inventory.find({"token.uid_hash": {"$exists":True}, "inventory.owner_id": {"$exists":True}, "token.enabled": True}, {"inventory.owner_id": True, "token.uid_hash": True }): if obj["inventory"].pop("owner_id") in allowed_names: del obj["_id"] del obj["inventory"] allowed_uids.append(obj) - return response.json(allowed_uids=allowed_uids) + return json({"allowed_uids": allowed_uids}) @app.route("/longpoll") async def view_longpoll(request): if request.headers.get('KEY') != DOORBOY_SECRET: - return "how about no" + return text("how about no") - async def g(): - yield "data: response-generator-started\n\n" + async def g(response): + await response.write("data: response-generator-started\n\n") pipeline = [ { '$match': { @@ -53,13 +51,15 @@ async def view_longpoll(request): ] try: async with mongodb.eventlog.watch(pipeline) as stream: - yield "data: watch-stream-opened\n\n" + await response.write("data: watch-stream-opened\n\n") async for event in stream: if event["fullDocument"].get("type") == "open-door": - yield "data: %s\n\n" % event["fullDocument"]["door"] + await response.write("data: %s\n\n" % event["fullDocument"]["door"]) except pymongo.errors.PyMongoError: - return "urror" + return return stream(g, content_type="text/event-stream") if __name__ == '__main__': - app.run(debug=False, host='::') + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) # swap default event loop to uvloop + loop = asyncio.get_event_loop() + app.run(host='::', debug=False, loop=loop) diff --git a/requirements.txt b/requirements.txt index 5e3178d..ce59ef1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,3 @@ -Flask -Flask-WTF -pymongo gunicorn sanic motor