Separated codebase from development repo

.flake8

@@ -0,0 +1,6 @@
+[flake8]
+inline-quotes = "
+multiline-quotes = """
+indent-size = 4
+max-line-length = 160
+ignore = Q003 E128 E704

.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+*.swp

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 3.9.2
     hooks:
     -   id: flake8
-        additional_dependencies: [flake8-typing-imports==1.10.0]
+        additional_dependencies: [flake8-typing-imports==1.10.0,flake8-quotes==3.2.0]
 
 -   repo: https://github.com/jorisroovers/gitlint
     rev: v0.15.1

Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:20.04 as base
+ENV container docker
+ENV PYTHONUNBUFFERED=1
+ENV LC_ALL C.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+RUN echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \
+ && echo "Dpkg::Use-Pty=0;" > /etc/apt/apt.conf.d/99quieter \
+ && apt-get update -qq \
+ && apt-get install -y -qq \
+    bash build-essential python3-dev cython3 libffi-dev libssl-dev \
+    libkrb5-dev ldap-utils libsasl2-modules-gssapi-mit libsasl2-dev libldap2-dev \
+    python python3-pip python3-cffi iptables ipset \
+    strongswan libstrongswan-extra-plugins libcharon-extra-plugins \
+    openvpn  libncurses5-dev gawk wget unzip git rsync \
+ && apt-get clean \
+ && rm /etc/dpkg/dpkg.cfg.d/docker-apt-speedup
+
+WORKDIR /src
+COPY requirements.txt /src/
+RUN pip3 install --no-cache-dir -r requirements.txt
+COPY config/strongswan.conf /etc/strongswan.conf
+COPY pinecrypt/. /src/pinecrypt/
+COPY helpers /helpers/
+COPY MANIFEST.in setup.py README.md /src/
+COPY misc/. /src/misc/
+RUN python3 -m compileall .
+RUN pip3 install --no-cache-dir .
+RUN rm -Rfv /src

MANIFEST.in

@@ -0,0 +1,6 @@
+include README.md
+include pinecrypt/server/templates/mail/*.md
+include pinecrypt/server/builder/overlay/usr/bin/pinecrypt.server-*
+include pinecrypt/server/builder/overlay/etc/uci-defaults/*
+include pinecrypt/server/builder/overlay/etc/profile
+include pinecrypt/server/builder/*.sh

config/strongswan.conf

@@ -0,0 +1,45 @@
+charon {
+  load_modular = yes
+  plugins {
+    include /etc/strongswan.d/charon/ccm.conf
+    include /etc/strongswan.d/charon/nonce.conf
+    include /etc/strongswan.d/charon/updown.conf
+    include /etc/strongswan.d/charon/pkcs11.conf
+    include /etc/strongswan.d/charon/connmark.conf
+    include /etc/strongswan.d/charon/pkcs1.conf
+    include /etc/strongswan.d/charon/constraints.conf
+    include /etc/strongswan.d/charon/revocation.conf
+    include /etc/strongswan.d/charon/openssl.conf
+    include /etc/strongswan.d/charon/aes.conf
+    include /etc/strongswan.d/charon/resolve.conf
+    include /etc/strongswan.d/charon/curve25519.conf
+    include /etc/strongswan.d/charon/gcm.conf
+    include /etc/strongswan.d/charon/random.conf
+    include /etc/strongswan.d/charon/pkcs7.conf
+    include /etc/strongswan.d/charon/af-alg.conf
+    include /etc/strongswan.d/charon/x509.conf
+    include /etc/strongswan.d/charon/rdrand.conf
+    include /etc/strongswan.d/charon/hmac.conf
+    include /etc/strongswan.d/charon/gmp.conf
+    include /etc/strongswan.d/charon/pubkey.conf
+    include /etc/strongswan.d/charon/ctr.conf
+    include /etc/strongswan.d/charon/certexpire.conf
+    include /etc/strongswan.d/charon/socket-default.conf
+    include /etc/strongswan.d/charon/lookip.conf
+    include /etc/strongswan.d/charon/mgf1.conf
+    include /etc/strongswan.d/charon/unity.conf
+    include /etc/strongswan.d/charon/sha2.conf
+    include /etc/strongswan.d/charon/stroke.conf
+    include /etc/strongswan.d/charon/aesni.conf
+    include /etc/strongswan.d/charon/agent.conf
+    include /etc/strongswan.d/charon/kernel-libipsec.conf
+    include /etc/strongswan.d/charon/curl.conf
+    include /etc/strongswan.d/charon/pkcs8.conf
+    include /etc/strongswan.d/charon/cmac.conf
+    include /etc/strongswan.d/charon/attr.conf
+    include /etc/strongswan.d/charon/error-notify.conf
+    include /etc/strongswan.d/charon/pem.conf
+    include /etc/strongswan.d/charon/pkcs12.conf
+    include /etc/strongswan.d/charon/kernel-netlink.conf
+  }
+}

helpers/openvpn-client-connect.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python3
+import os
+import sys
+from pinecrypt.server import db
+
+# This implements OCSP like functionality
+
+obj = db.certificates.find_one({
+    # TODO: use digest instead
+    "serial_number": "%x" % int(os.environ["tls_serial_0"]),
+    "status":"signed",
+})
+
+if not obj:
+    sys.exit(1)

helpers/openvpn-learn-address.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+import os
+import sys
+from pinecrypt.server import db
+from datetime import datetime
+
+operation, addr = sys.argv[1:3]
+if operation == "delete":
+    pass
+else:
+    common_name = sys.argv[3]
+    db.certificates.update_one({
+        # TODO: use digest instead
+        "serial_number": "%x" % int(os.environ["tls_serial_0"]),
+        "status":"signed",
+    }, {
+        "$set": {
+            "last_seen": datetime.utcnow(),
+            "instance": os.environ["instance"],
+            "remote": {
+                "port": int(os.environ["untrusted_port"]),
+                "addr": os.environ["untrusted_ip"],
+            }
+        },
+        "$addToSet": {
+            "ip": addr
+        }
+    })

helpers/updown.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+import sys
+import os
+from pinecrypt.server import db
+from datetime import datetime
+
+addrs = set()
+for key, value in os.environ.items():
+    if key.startswith("PLUTO_PEER_SOURCEIP"):
+        addrs.add(value)
+
+with open("/instance") as fh:
+    instance = fh.read().strip()
+
+db.certificates.update_one({
+    "distinguished_name": os.environ["PLUTO_PEER_ID"],
+    "status":"signed",
+}, {
+    "$set": {
+        "last_seen": datetime.utcnow(),
+        "instance": instance,
+        "remote": {
+            "addr": os.environ["PLUTO_PEER"]
+        }
+    },
+    "$addToSet": {
+        "ip": {
+            "$each": list(addrs)
+        }
+    }
+})

misc/pinecone

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+from pinecrypt.server.cli import entry_point
+
+if __name__ == "__main__":
+    entry_point()

pinecrypt/server/api/bootstrap.py

@@ -0,0 +1,44 @@
+import hashlib
+import logging
+from pinecrypt.server import authority, const, config
+from pinecrypt.server.common import cert_to_dn
+from pinecrypt.server.decorators import serialize
+
+logger = logging.getLogger(__name__)
+
+class BootstrapResource(object):
+    @serialize
+    def on_get(self, req, resp):
+        """
+        Return publicly accessible info unlike /api/session
+        """
+        return dict(
+            hostname=const.FQDN,
+            namespace=const.AUTHORITY_NAMESPACE,
+            replicas=[doc["common_name"] for doc in authority.list_replicas()],
+            globals=list(config.get_all("Globals")),
+            openvpn=dict(
+                tls_version_min=config.get("Globals", "OPENVPN_TLS_VERSION_MIN")["value"],
+                tls_ciphersuites=config.get("Globals", "OPENVPN_TLS_CIPHERSUITES")["value"],
+                tls_cipher=config.get("Globals", "OPENVPN_TLS_CIPHER")["value"],
+                cipher=config.get("Globals", "OPENVPN_CIPHER")["value"],
+                auth=config.get("Globals", "OPENVPN_AUTH")["value"]
+            ),
+            strongswan=dict(
+                dhgroup=config.get("Globals", "STRONGSWAN_DHGROUP")["value"],
+                ike=config.get("Globals", "STRONGSWAN_IKE")["value"],
+                esp=config.get("Globals", "STRONGSWAN_ESP")["value"],
+            ),
+            certificate=dict(
+                algorithm=authority.public_key.algorithm,
+                common_name=authority.certificate.subject.native["common_name"],
+                distinguished_name=cert_to_dn(authority.certificate),
+                md5sum=hashlib.md5(authority.certificate_buf).hexdigest(),
+                blob=authority.certificate_buf.decode("ascii"),
+                organization=authority.certificate["tbs_certificate"]["subject"].native.get("organization_name"),
+                signed=authority.certificate["tbs_certificate"]["validity"]["not_before"].native.replace(tzinfo=None),
+                expires=authority.certificate["tbs_certificate"]["validity"]["not_after"].native.replace(tzinfo=None)
+            ),
+            user_enrollment_allowed=const.USER_ENROLLMENT_ALLOWED,
+            user_multiple_certificates=const.USER_MULTIPLE_CERTIFICATES,
+        )

pinecrypt/server/api/builder.py

@@ -0,0 +1,55 @@
+
+import click
+import os
+import asyncio
+from sanic import Sanic
+from sanic.exceptions import ServerError
+from sanic.response import file_stream
+from pinecrypt.server import const
+
+app = Sanic("builder")
+app.config.RESPONSE_TIMEOUT = 300
+
+@app.route("/api/build/")
+async def view_build(request):
+    build_script_path = "/builder/script/mfp.sh"
+    suffix = "-glinet_gl-ar150-squashfs-sysupgrade.bin"
+    suggested_filename = "mfp%s" % suffix
+    build = "/builder/src"
+    log_path = build + "/build.log"
+
+    proc = await asyncio.create_subprocess_exec(
+        build_script_path,
+        stdout=open(log_path, "w"),
+        close_fds=True,
+        shell=False,
+        cwd=os.path.dirname(os.path.realpath(build_script_path)),
+        env={
+            "PROFILE": "glinet_gl-ar150",
+            "PATH": "/usr/sbin:/usr/bin:/sbin:/bin",
+            "AUTHORITY_NAMESPACE": const.AUTHORITY_NAMESPACE,
+            "BUILD": build,
+            "OVERLAY": build + "/overlay/"
+        },
+        startupinfo=None,
+        creationflags=0,
+    )
+
+    stdout, stderr = await proc.communicate()
+
+    if proc.returncode:
+        raise ServerError("Build script finished with non-zero exitcode, see %s for more information" % log_path)
+
+    for root, dirs, files in os.walk("/builder/src/bin/targets"):
+        for filename in files:
+            if filename.endswith(suffix):
+                path = os.path.join(root, filename)
+                click.echo("Serving: %s" % path)
+                return await file_stream(
+                        path,
+                        headers={
+                           "Content-Disposition": "attachment; filename=%s" % suggested_filename
+                        }
+                    )
+    raise ServerError("Failed to find image builder directory in %s" % build)
+

pinecrypt/server/api/events.py

@@ -0,0 +1,121 @@
+from datetime import datetime
+from functools import wraps
+from oscrypto import asymmetric
+from json import dumps
+from motor.motor_asyncio import AsyncIOMotorClient
+from pinecrypt.server import const
+from prometheus_client import Counter
+from sanic import Sanic
+from sanic.response import stream
+from sanic_prometheus import monitor
+from bson.objectid import ObjectId
+
+
+streams_opened = Counter("pinecrypt_events_stream_opened",
+    "Event stream opened count")
+events_emitted = Counter("pinecrypt_events_emitted",
+    "Events emitted count")
+
+
+app = Sanic("events")
+monitor(app).expose_endpoint()
+app.config.RESPONSE_TIMEOUT = 999
+
+def cookie_login(func):
+    @wraps(func)
+    async def wrapped(request, *args, **kwargs):
+        if request.method != "GET":
+            raise ValueError("For now stick with read-only operations for cookie auth")
+        value = request.cookies.get(const.SESSION_COOKIE)
+        now = datetime.utcnow()
+        await app.db.certidude_sessions.update_one({
+            "secret": value,
+            "started": {
+                "$lte": now
+            },
+            "expires": {
+                "$gte": now
+            },
+        }, {
+            "$set": {
+                "last_seen": now,
+           }
+        })
+        return await func(request, *args, **kwargs)
+    return wrapped
+
+
+@app.listener("before_server_start")
+async def setup_db(app, loop):
+    # TODO: find cleaner way to do this, for more see
+    # https://github.com/sanic-org/sanic/issues/919
+    app.db = AsyncIOMotorClient(const.MONGO_URI).get_default_database()
+
+
+# TODO: Change to /api/event/log and simplify nginx config to /api/event
+@app.route("/api/event/")
+@cookie_login
+async def view_event(request):
+    async def g(resp):
+        await resp.write("data: response-generator-started\n\n")
+        streams_opened.inc()
+
+        async with app.db.watch(full_document="updateLookup") as stream:
+            await resp.write("data: watch-stream-opened\n\n")
+            async for event in stream:
+
+                if event.get("ns").get("coll") == "certidude_certificates":
+                    if event.get("operationType") == "insert" and event["fullDocument"].get("status") == "csr":
+                        await resp.write("event: request-submitted\ndata: %s\n\n" % str(event["documentKey"].get("_id")))
+                        events_emitted.inc()
+
+                    if event.get("operationType") == "update" and event["updateDescription"].get("updatedFields").get("status") == "signed":
+                        await resp.write("event: request-signed\ndata: %s\n\n" % str(event["documentKey"].get("_id")))
+                        events_emitted.inc()
+
+                    if event.get("operationType") == "insert" and event["fullDocument"].get("status") == "signed":
+                        await resp.write("event: request-signed\ndata: %s\n\n" % event["fullDocument"].get("common_name"))
+                        events_emitted.inc()
+
+                    if event.get("operationType") == "update" and event["fullDocument"].get("status") == "revoked":
+                        await resp.write("event: certificate-revoked\ndata: %s\n\n" % str(event["documentKey"].get("_id")))
+                        events_emitted.inc()
+
+                    if event.get("operationType") == "delete":
+                        await resp.write("event: request-deleted\ndata: %s\n\n" % str(event["documentKey"].get("_id")))
+                        events_emitted.inc()
+
+                    if event.get("operationType") == "update" and "tags" in event.get("updateDescription").get("updatedFields"):
+                        await resp.write("event: tag-update\ndata: %s\n\n" % event["fullDocument"].get("common_name"))
+                        events_emitted.inc()
+
+                    if event.get("operationType") == "update" and "attributes" in event.get("updateDescription").get("updatedFields"):
+                        await resp.write("event: attribute-update\ndata: %s\n\n" % str(event["documentKey"].get("_id")))
+                        events_emitted.inc()
+
+
+                if event.get("ns").get("coll") == "certidude_logs":
+
+                    from pinecrypt.server.decorators import MyEncoder
+
+                    obj=dict(
+                        created=event["fullDocument"].get("created"),
+                        message=event["fullDocument"].get("message"),
+                        severity=event["fullDocument"].get("severity")
+                    )
+
+                    await resp.write("event: log-entry\ndata: %s\n\n" % dumps(obj, cls=MyEncoder))
+                    events_emitted.inc()
+    return stream(g, content_type="text/event-stream")
+
+
+@app.route("/api/event/request-signed/<id>")
+async def publish(request, id):
+    pipeline = [{"$match": { "operationType": "update", "fullDocument.status": "signed", "documentKey._id": ObjectId(id)}}]
+    resp = await request.respond(content_type="application/x-x509-user-cert")
+    async with app.db["certidude_certificates"].watch(pipeline, full_document="updateLookup") as stream:
+        async for event in stream:
+            cert_der = event["fullDocument"].get("cert_buf")
+            cert_pem = asymmetric.dump_certificate(asymmetric.load_certificate(cert_der))
+            await resp.send(cert_pem, True)
+    return resp

pinecrypt/server/api/log.py

@@ -0,0 +1,13 @@
+from pinecrypt.server.decorators import serialize
+from pinecrypt.server import db
+from .utils.firewall import cookie_login
+
+class LogResource(object):
+    @serialize
+    @cookie_login
+    def on_get(self, req, resp):
+        def g():
+            for log in db.eventlog.find({}).limit(req.get_param_as_int("limit", required=True)).sort("created", -1):
+                log.pop("_id")
+                yield log
+        return tuple(g())

pinecrypt/server/api/ocsp.py

@@ -0,0 +1,127 @@
+import pytz
+from asn1crypto.util import timezone
+from asn1crypto import ocsp
+from pinecrypt.server import const, authority
+from datetime import datetime, timedelta
+from math import inf
+from motor.motor_asyncio import AsyncIOMotorClient
+from oscrypto import asymmetric
+from prometheus_client import Counter, Histogram
+from sanic import Sanic, response
+from sanic_prometheus import monitor
+
+ocsp_request_valid = Counter("pinecrypt_ocsp_request_valid",
+    "Valid OCSP requests")
+ocsp_request_list_size = Histogram("pinecrypt_ocsp_request_list_size",
+    "Histogram of OCSP request list size",
+    buckets=(1, 2, 3, inf))
+ocsp_request_size_bytes = Histogram("pinecrypt_ocsp_request_size_bytes",
+    "Histogram of OCSP request size in bytes",
+    buckets=(100, 200, 500, 1000, 2000, 5000, 10000, inf))
+ocsp_request_nonces = Histogram("pinecrypt_ocsp_request_nonces",
+    "Histogram of nonce count per request",
+    buckets=(1, 2, 3, inf))
+ocsp_response_status = Counter("pinecrypt_ocsp_response_status",
+    "Status responses", ["status"])
+
+app = Sanic("events")
+monitor(app).expose_endpoint()
+
+
+@app.listener("before_server_start")
+async def setup_db(app, loop):
+    # TODO: find cleaner way to do this, for more see
+    # https://github.com/sanic-org/sanic/issues/919
+    app.ctx.db = AsyncIOMotorClient(const.MONGO_URI).get_default_database()
+
+
+@app.route("/api/ocsp/", methods=["POST"])
+async def view_ocsp_responder(request):
+    ocsp_request_size_bytes.observe(len(request.body))
+    ocsp_req = ocsp.OCSPRequest.load(request.body)
+
+    server_certificate = authority.get_ca_cert()
+
+    now = datetime.now(timezone.utc).replace(microsecond=0)
+    response_extensions = []
+
+    nonces = 0
+    for ext in ocsp_req["tbs_request"]["request_extensions"]:
+        if ext["extn_id"].native == "nonce":
+            nonces += 1
+            response_extensions.append(
+                ocsp.ResponseDataExtension({
+                    "extn_id": "nonce",
+                    "critical": False,
+                    "extn_value": ext["extn_value"]
+                })
+            )
+
+    ocsp_request_nonces.observe(nonces)
+    ocsp_request_valid.inc()
+
+    responses = []
+
+    ocsp_request_list_size.observe(len(ocsp_req["tbs_request"]["request_list"]))
+    for item in ocsp_req["tbs_request"]["request_list"]:
+        serial = item["req_cert"]["serial_number"].native
+        assert serial > 0, "Serial number correctness check failed"
+
+        doc = await app.ctx.db.certidude_certificates.find_one({"serial_number": "%x" % serial})
+        if doc:
+            if doc["status"] == "signed":
+                status = ocsp.CertStatus(name="good", value=None)
+                ocsp_response_status.labels("good").inc()
+            elif doc["status"] == "revoked":
+                status = ocsp.CertStatus(
+                    name="revoked",
+                    value={
+                        "revocation_time": doc["revoked"].replace(tzinfo=pytz.UTC),
+                        "revocation_reason": doc["revocation_reason"],
+                    })
+                ocsp_response_status.labels("revoked").inc()
+            else:
+                # This should not happen, if it does database is mangled
+                raise ValueError("Invalid/unknown certificate status '%s'" % doc["status"])
+        else:
+            status = ocsp.CertStatus(name="unknown", value=None)
+            ocsp_response_status.labels("unknown").inc()
+
+        responses.append({
+            "cert_id": {
+                "hash_algorithm": {
+                    "algorithm": "sha1"
+                },
+                "issuer_name_hash": server_certificate.asn1.subject.sha1,
+                "issuer_key_hash": server_certificate.public_key.asn1.sha1,
+                "serial_number": serial,
+            },
+            "cert_status": status,
+            "this_update": now - const.CLOCK_SKEW_TOLERANCE,
+            "next_update": now + timedelta(minutes=15) + const.CLOCK_SKEW_TOLERANCE,
+            "single_extensions": []
+        })
+
+    response_data = ocsp.ResponseData({
+        "responder_id": ocsp.ResponderId(name="by_key", value=server_certificate.public_key.asn1.sha1),
+        "produced_at": now,
+        "responses": responses,
+        "response_extensions": response_extensions
+    })
+
+    return response.raw(ocsp.OCSPResponse({
+        "response_status": "successful",
+        "response_bytes": {
+            "response_type": "basic_ocsp_response",
+            "response": {
+                "tbs_response_data": response_data,
+                "certs": [server_certificate.asn1],
+                "signature_algorithm": {"algorithm": "sha1_ecdsa" if authority.public_key.algorithm == "ec" else "sha1_rsa" },
+                "signature": (asymmetric.ecdsa_sign if authority.public_key.algorithm == "ec" else asymmetric.rsa_pkcs1v15_sign)(
+                    authority.private_key,
+                    response_data.dump(),
+                    "sha1"
+                )
+            }
+        }
+    }).dump(), headers={"Content-Type": "application/ocsp-response"})

pinecrypt/server/api/request.py

@@ -0,0 +1,268 @@
+import click
+import falcon
+import logging
+import json
+import hashlib
+from asn1crypto import pem
+from asn1crypto.csr import CertificationRequest
+from pinecrypt.server import const, errors, authority
+from pinecrypt.server.decorators import csrf_protection, MyEncoder
+from pinecrypt.server.user import DirectoryConnection
+from oscrypto import asymmetric
+from .utils.firewall import whitelist_subnets, whitelist_content_types, \
+    login_required, login_optional, authorize_admin, validate_clock_skew
+
+logger = logging.getLogger(__name__)
+
+"""
+openssl genrsa -out test.key 1024
+openssl req -new -sha256 -key test.key -out test.csr -subj "/CN=test"
+curl -f -L -H "Content-type: application/pkcs10" --data-binary @test.csr \
+  http://ca.example.lan/api/request/?wait=yes
+"""
+
+class RequestListResource(object):
+    @login_optional
+    @whitelist_subnets(const.REQUEST_SUBNETS)
+    @whitelist_content_types("application/pkcs10")
+    @validate_clock_skew
+    def on_post(self, req, resp):
+        """
+        Validate and parse certificate signing request, the RESTful way
+        Endpoint urls
+        /request/?wait=yes
+        /request/autosign=1
+        /request
+        """
+        reasons = []
+        body = req.stream.read(req.content_length)
+
+        try:
+            header, _, der_bytes = pem.unarmor(body)
+            csr = CertificationRequest.load(der_bytes)
+        except ValueError:
+            logger.info("Malformed certificate signing request submission from %s blocked", req.context["remote"]["addr"])
+            raise falcon.HTTPBadRequest(
+                "Bad request",
+                "Malformed certificate signing request")
+        else:
+            req_public_key = asymmetric.load_public_key(csr["certification_request_info"]["subject_pk_info"])
+            if authority.public_key.algorithm != req_public_key.algorithm:
+                logger.info("Attempt to submit %s based request from %s blocked, only %s allowed" % (
+                    req_public_key.algorithm.upper(),
+                    req.context["remote"]["addr"],
+                    authority.public_key.algorithm.upper()))
+                raise falcon.HTTPBadRequest(
+                    "Bad request",
+                    "Unsupported key algorithm %s, expected %s" % (req_public_key.algorithm, authority.public_key.algorithm))
+
+        try:
+            common_name = csr["certification_request_info"]["subject"].native["common_name"]
+        except KeyError:
+            logger.info("Malformed certificate signing request without common name submitted from %s" % req.context["remote"]["addr"])
+            raise falcon.HTTPBadRequest(title="Bad request",description="Common name missing from certificate signing request")
+
+        """
+        Determine whether autosign is allowed to overwrite already issued
+        certificates automatically
+        """
+
+        overwrite_allowed = False
+        for subnet in const.OVERWRITE_SUBNETS:
+            if req.context["remote"]["addr"] in subnet:
+                overwrite_allowed = True
+                break
+
+
+        """
+        Handle domain computer automatic enrollment
+        """
+        machine = req.context.get("machine")
+        if machine:
+            reasons.append("machine enrollment not allowed from %s" % req.context["remote"]["addr"])
+            for subnet in const.MACHINE_ENROLLMENT_SUBNETS:
+                if req.context["remote"]["addr"] in subnet:
+                    if common_name != machine:
+                        raise falcon.HTTPBadRequest(
+                            "Bad request",
+                            "Common name %s differs from Kerberos credential %s!" % (common_name, machine))
+
+                    hit = False
+                    with DirectoryConnection() as conn:
+                        ft = const.LDAP_COMPUTER_FILTER % ("%s$" % machine)
+                        attribs = "cn",
+                        r = conn.search_s(const.LDAP_BASE, 2, ft, attribs)
+                        for dn, entry in r:
+                            if not dn:
+                                continue
+                            else:
+                                hit = True
+                                break
+
+                    if hit:
+                        # Automatic enroll with Kerberos machine cerdentials
+                        resp.set_header("Content-Type", "application/x-pem-file")
+                        try:
+                            mongo_doc = authority.store_request(body,address=str(req.context["remote"]["addr"]))
+                            cert, resp.text = authority.sign(mongo_id=str(mongo_doc["_id"]),
+                                profile="Roadwarrior", overwrite=overwrite_allowed) # TODO: handle thrown exception
+                            logger.info("Automatically enrolled Kerberos authenticated machine %s (%s) from %s",
+                                machine, dn, req.context["remote"]["addr"])
+                            return
+                        except errors.RequestExists:
+                            reasons.append("same request already uploaded exists")
+                            # We should still redirect client to long poll URL below
+                        except errors.DuplicateCommonNameError:
+                            logger.warning("rejected signing request with overlapping common name from %s",
+                                req.context["remote"]["addr"])
+                            raise falcon.HTTPConflict(
+                                "CSR with such CN already exists",
+                                 "Will not overwrite existing certificate signing request, explicitly delete CSR and try again")
+
+                    else:
+                        logger.error("Kerberos authenticated machine %s didn't fit the 'ldap computer filter' criteria %s" % (machine, ft))
+
+
+        """
+        Process automatic signing if the IP address is whitelisted,
+        autosigning was requested and certificate can be automatically signed
+        """
+
+        if req.get_param_as_bool("autosign"):
+            for subnet in const.AUTOSIGN_SUBNETS:
+                if req.context["remote"]["addr"] in subnet:
+                    try:
+                        resp.set_header("Content-Type", "application/x-pem-file")
+                        mongo_doc = authority.store_request(body,address=str(req.context["remote"]["addr"]))
+                        _, resp.text = authority.sign(mongo_id=str(mongo_doc["_id"]),
+                            overwrite=overwrite_allowed, profile="Roadwarrior")
+
+                        logger.info("Signed %s as %s is whitelisted for autosign", common_name, req.context["remote"]["addr"])
+                        return
+                    except EnvironmentError:
+                        logger.info("Autosign for %s from %s failed, signed certificate already exists",
+                            common_name, req.context["remote"]["addr"])
+                        reasons.append("autosign failed, signed certificate already exists")
+                    break
+            else:
+                reasons.append("IP address not whitelisted for autosign")
+        else:
+            reasons.append("autosign not requested")
+
+        # Attempt to save the request otherwise
+        try:
+            mongo_doc = authority.store_request(body,
+                address=str(req.context["remote"]["addr"]))
+        except errors.RequestExists:
+            reasons.append("same request already uploaded exists")
+            # We should still redirect client to long poll URL below
+        except errors.DuplicateCommonNameError:
+            logger.warning("rejected signing request with overlapping common name from %s",
+                req.context["remote"]["addr"])
+            raise falcon.HTTPConflict(
+                "CSR with such CN already exists",
+                "Will not overwrite existing certificate signing request, explicitly delete CSR and try again")
+
+        # Wait the certificate to be signed if waiting is requested
+        logger.info("Signing request %s from %s put on hold,  %s", common_name, req.context["remote"]["addr"], ", ".join(reasons))
+
+        if req.get_param("wait"):
+            header, _, der_bytes = pem.unarmor(body)
+            url = "https://%s/api/event/request-signed/%s" % (const.AUTHORITY_NAMESPACE, str(mongo_doc["_id"]))
+            click.echo("Redirecting to: %s"  % url)
+            resp.status = falcon.HTTP_SEE_OTHER
+            resp.set_header("Location", url)
+        else:
+            # Request was accepted, but not processed
+            resp.status = falcon.HTTP_202
+            resp.text = ". ".join(reasons)
+
+            if req.client_accepts("application/json"):
+                resp.text = json.dumps({"title":"Accepted", "description":resp.text, "id":str(mongo_doc["_id"])},
+                    cls=MyEncoder)
+
+
+class RequestDetailResource(object):
+    def on_get(self, req, resp, id):
+        """
+        Fetch certificate signing request as PEM
+        """
+
+        try:
+            csr, csr_doc, buf = authority.get_request(id)
+        except errors.RequestDoesNotExist:
+            logger.warning("Failed to serve non-existant request %s to %s",
+                id, req.context["remote"]["addr"])
+            raise falcon.HTTPNotFound()
+
+        resp.set_header("Content-Type", "application/pkcs10")
+        logger.debug("Signing request %s was downloaded by %s",
+            csr_doc["common_name"], req.context["remote"]["addr"])
+
+        preferred_type = req.client_prefers(("application/json", "application/x-pem-file"))
+
+        if preferred_type == "application/x-pem-file":
+            # For certidude client, curl scripts etc
+            resp.set_header("Content-Type", "application/x-pem-file")
+            resp.set_header("Content-Disposition", ("attachment; filename=%s.pem" % csr_doc["common_name"]))
+            resp.text = buf
+        elif preferred_type == "application/json":
+            # For web interface events
+            resp.set_header("Content-Type", "application/json")
+            resp.set_header("Content-Disposition", ("attachment; filename=%s.json" % csr_doc["common_name"]))
+            resp.text = json.dumps(dict(
+                submitted=csr_doc["submitted"],
+                common_name=csr_doc["common_name"],
+                id=str(csr_doc["_id"]),
+                address=csr_doc["user"]["request_addresss"],
+                md5sum=hashlib.md5(buf).hexdigest(),
+                sha1sum=hashlib.sha1(buf).hexdigest(),
+                sha256sum=hashlib.sha256(buf).hexdigest(),
+                sha512sum=hashlib.sha512(buf).hexdigest()), cls=MyEncoder)
+        else:
+            raise falcon.HTTPUnsupportedMediaType(
+                "Client did not accept application/json or application/x-pem-file")
+
+
+    @csrf_protection
+    @login_required
+    @authorize_admin
+    def on_post(self, req, resp, id):
+        """
+        Sign a certificate signing request
+        """
+        try:
+            cert, buf = authority.sign(mongo_id=id,
+                profile=req.get_param("profile", default="Roadwarrior"),
+                overwrite=True,
+                signer=req.context.get("user").name)  #if user is cached in browser then there is no name
+            # Mailing and long poll publishing implemented in the function above
+        except EnvironmentError: # no such CSR
+            raise falcon.HTTPNotFound(title="Not found",description="CSR not found with id %s" %  id)
+
+        resp.text = "Certificate successfully signed"
+        resp.status = falcon.HTTP_201
+        resp.location = req.forwarded_uri.replace("request","sign")
+
+        cn = cert.subject.native.get("common_name")
+        logger.info("Signing request %s signed by %s from %s", cn,
+            req.context.get("user"), req.context["remote"]["addr"])
+
+
+    @csrf_protection
+    @login_required
+    @authorize_admin
+    def on_delete(self, req, resp, id):
+        try:
+            authority.delete_request(id, user=req.context.get("user"))
+            # Logging implemented in the function above
+        except errors.RequestDoesNotExist as e:
+            resp.text = "No certificate signing request for with id %s not found" % id
+            logger.warning("User %s failed to delete signing request %s from %s, reason: %s",
+                req.context["user"], id, req.context["remote"]["addr"], e)
+            raise falcon.HTTPNotFound()
+        except ValueError as e:
+            resp.text = "No ID specified %s" % id
+            logger.warning("User %s wanted to delete invalid signing request %s from %s, reason: %s",
+                req.context["user"], id, req.context["remote"]["addr"], e)
+            raise falcon.HTTPBadRequest()

pinecrypt/server/api/revoked.py

@@ -0,0 +1,46 @@
+import falcon
+import logging
+from pinecrypt.server import authority, const, errors
+from .utils.firewall import whitelist_subnets
+
+logger = logging.getLogger(__name__)
+
+class RevocationListResource(object):
+    @whitelist_subnets(const.CRL_SUBNETS)
+    def on_get(self, req, resp):
+        # Primarily offer DER encoded CRL as per RFC5280
+        # This is also what StrongSwan expects
+        if req.client_accepts("application/x-pkcs7-crl"):
+            resp.set_header("Content-Type", "application/x-pkcs7-crl")
+            resp.append_header(
+                "Content-Disposition",
+                ("attachment; filename=%s.crl" % const.HOSTNAME))
+            # Convert PEM to DER
+            logger.debug("Serving revocation list (DER) to %s", req.context["remote"]["addr"])
+            resp.text = authority.export_crl(pem=False)
+        elif req.client_accepts("application/x-pem-file"):
+            resp.set_header("Content-Type", "application/x-pem-file")
+            resp.append_header(
+                "Content-Disposition",
+                ("attachment; filename=%s-crl.pem" % const.HOSTNAME))
+            logger.debug("Serving revocation list (PEM) to %s", req.context["remote"]["addr"])
+            resp.text = authority.export_crl()
+        else:
+            logger.debug("Client %s asked revocation list in unsupported format" % req.context["remote"]["addr"])
+            raise falcon.HTTPUnsupportedMediaType(
+                "Client did not accept application/x-pkcs7-crl or application/x-pem-file")
+
+
+class RevokedCertificateDetailResource(object):
+    def on_get(self, req, resp, serial_number):
+        try:
+            cert_doc, buf = authority.get_revoked(serial_number)
+        except errors.CertificateDoesNotExist:
+            logger.warning("Failed to serve non-existant revoked certificate with serial %s to %s",
+                serial_number, req.context["remote"]["addr"])
+            raise falcon.HTTPNotFound()
+        resp.set_header("Content-Type", "application/x-pem-file")
+        resp.set_header("Content-Disposition", ("attachment; filename=%s.pem" % cert_doc["serial_number"]))
+        resp.text = buf
+        logger.debug("Served revoked certificate with serial %s to %s",
+            cert_doc["serial_number"], req.context["remote"]["addr"])

pinecrypt/server/api/script.py

@@ -0,0 +1,29 @@
+import logging
+import os
+from pinecrypt.server import authority, const
+from jinja2 import Environment, FileSystemLoader
+from .utils.firewall import whitelist_subject
+
+logger = logging.getLogger(__name__)
+env = Environment(loader=FileSystemLoader(const.SCRIPT_DIR), trim_blocks=True)
+
+class ScriptResource(object):
+    @whitelist_subject
+    def on_get(self, req, resp, id):
+        path, buf, cert, attribs = authority.get_attributes(id)
+        # TODO: are keys unique?
+        named_tags = {}
+        other_tags = []
+        cn = cert["common_name"]
+
+        script = named_tags.get("script", "default.sh")
+        assert script in os.listdir(const.SCRIPT_DIR)
+        resp.set_header("Content-Type", "text/x-shellscript")
+        resp.body = env.get_template(os.path.join(script)).render(
+            authority_name=const.FQDN,
+            common_name=cn,
+            other_tags=other_tags,
+            named_tags=named_tags,
+            attributes=attribs.get("user").get("machine"))
+        logger.info("Served script %s for %s at %s" % (script, cn, req.context["remote"]["addr"]))
+        # TODO: Assert time is within reasonable range

pinecrypt/server/api/session.py

@@ -0,0 +1,185 @@
+import hashlib
+import logging
+from pinecrypt.server import authority, const, config
+from pinecrypt.server.decorators import serialize, csrf_protection
+from pinecrypt.server.user import User
+from .utils.firewall import login_required, authorize_admin, register_session
+
+logger = logging.getLogger(__name__)
+
+
+class CertificateAuthorityResource(object):
+    def on_get(self, req, resp):
+        logger.info("Served CA certificate to %s", req.context["remote"]["addr"])
+        resp.stream = open(const.AUTHORITY_CERTIFICATE_PATH, "rb")
+        resp.append_header("Content-Type", "application/x-x509-ca-cert")
+        resp.append_header("Content-Disposition", "attachment; filename=%s.crt" %
+            const.HOSTNAME)
+
+
+class SessionResource(object):
+
+    def __init__(self, manager):
+        self.token_manager = manager
+
+    @csrf_protection
+    @serialize
+    @login_required
+    @register_session
+    @authorize_admin
+    def on_get(self, req, resp):
+        def serialize_requests(g):
+            for csr, request, server in g():
+                try:
+                    submission_address = request["user"]["request_address"]
+                except KeyError:
+                    submission_address = None
+                try:
+                    submission_hostname = request["user"]["request_hostname"]
+                except KeyError:
+                    submission_hostname = None
+                yield dict(
+                    id=str(request["_id"]),
+                    submitted=request["submitted"],
+                    common_name=request["common_name"],
+                    address=submission_address,
+                    hostname=submission_hostname if submission_hostname != submission_address else None,
+                    md5sum=hashlib.md5(request["request_buf"]).hexdigest(),
+                    sha1sum=hashlib.sha1(request["request_buf"]).hexdigest(),
+                    sha256sum=hashlib.sha256(request["request_buf"]).hexdigest(),
+                    sha512sum=hashlib.sha512(request["request_buf"]).hexdigest()
+                )
+
+        def serialize_revoked(g):
+            for cert_obj, cert in g(limit=5):
+                yield dict(
+                    id=str(cert_obj["_id"]),
+                    serial="%x" % cert.serial_number,
+                    common_name=cert_obj["common_name"],
+                    # TODO: key type, key length, key exponent, key modulo
+                    signed=cert_obj["signed"],
+                    expired=cert_obj["expires"],
+                    revoked=cert_obj["revoked"],
+                    reason=cert_obj["revocation_reason"],
+                    sha256sum=hashlib.sha256(cert_obj["cert_buf"]).hexdigest())
+
+        def serialize_certificates(g):
+            for cert_doc, cert in g():
+                try:
+                    tags = cert_doc["tags"]
+                except KeyError:  # No tags
+                    tags = None
+
+
+                # TODO: Load attributes from databse
+                attributes = {}
+
+                try:
+                    lease = dict(
+                        inner_address=cert_doc["ip"],
+                        outer_address=cert_doc["remote"]["addr"],
+                        last_seen=cert_doc["last_seen"],
+                    )
+                except KeyError: # No such attribute(s)
+                    lease = None
+
+                try:
+                    #signer_username = getxattr(path, "user.signature.username").decode("ascii")
+                    signer_username = cert_doc["user"]["signature"]["username"]
+                except KeyError:
+                    signer_username = None
+
+                # TODO: dedup
+                serialized = dict(
+                    id=str(cert_doc["_id"]),
+                    serial="%x" % cert.serial_number,
+                    organizational_unit=cert.subject.native.get("organizational_unit_name"),
+                    common_name=cert_doc["common_name"],
+                    # TODO: key type, key length, key exponent, key modulo
+                    signed=cert_doc["signed"],
+                    expires=cert_doc["expires"],
+                    sha256sum=hashlib.sha256(cert_doc["cert_buf"]).hexdigest(),
+                    signer=signer_username,
+                    lease=lease,
+                    tags=tags,
+                    attributes=attributes or None,
+                    responder_url=None
+                )
+
+                for e in cert["tbs_certificate"]["extensions"].native:
+                    if e["extn_id"] == "key_usage":
+                        serialized["key_usage"] = e["extn_value"]
+                    elif e["extn_id"] == "extended_key_usage":
+                        serialized["extended_key_usage"] = e["extn_value"]
+                    elif e["extn_id"] == "basic_constraints":
+                        serialized["basic_constraints"] = e["extn_value"]
+                    elif e["extn_id"] == "crl_distribution_points":
+                        for c in e["extn_value"]:
+                            serialized["revoked_url"] = c["distribution_point"]
+                            break
+                        serialized["extended_key_usage"] = e["extn_value"]
+                    elif e["extn_id"] == "authority_information_access":
+                        for a in e["extn_value"]:
+                            if a["access_method"] == "ocsp":
+                                serialized["responder_url"] = a["access_location"]
+                            else:
+                                raise NotImplementedError("Don't know how to handle AIA access method %s" % a["access_method"])
+                    elif e["extn_id"] == "authority_key_identifier":
+                        pass
+                    elif e["extn_id"] == "key_identifier":
+                        pass
+                    elif e["extn_id"] == "subject_alt_name":
+                        serialized["subject_alt_name"] = e["extn_value"][0]
+                    else:
+                        raise NotImplementedError("Don't know how to handle extension %s" % e["extn_id"])
+                yield serialized
+
+        logger.info("Logged in authority administrator %s from %s with %s" % (
+            req.context.get("user"), req.context["remote"]["addr"], req.context["remote"]["user_agent"]))
+        return dict(
+            user=dict(
+                name=req.context.get("user").name,
+                gn=req.context.get("user").given_name,
+                sn=req.context.get("user").surname,
+                mail=req.context.get("user").mail
+            ),
+            request_submission_allowed=const.REQUEST_SUBMISSION_ALLOWED,
+            service=dict(
+                protocols=const.SERVICE_PROTOCOLS,
+            ),
+            builder=dict(
+                profiles=const.IMAGE_BUILDER_PROFILES or None
+            ),
+            tokens=self.token_manager.list() if self.token_manager else None,
+            tagging=[dict(name=t[0], type=t[1], title=t[0]) for t in const.TAG_TYPES],
+
+            mailer=dict(
+               name=const.SMTP_SENDER_NAME,
+               address=const.SMTP_SENDER_ADDR
+            ) if const.SMTP_SENDER_ADDR else None,
+            events="/api/event/",
+            requests=serialize_requests(authority.list_requests),
+            signed=serialize_certificates(authority.list_signed),
+            revoked=serialize_revoked(authority.list_revoked),
+            signature=dict(
+                revocation_list_lifetime=const.REVOCATION_LIST_LIFETIME,
+                profiles=config.options("SignatureProfile"),
+            ),
+            authorization=dict(
+                admin_users=User.objects.filter_admins(),
+
+                user_subnets=const.USER_SUBNETS or None,
+                autosign_subnets=const.AUTOSIGN_SUBNETS or None,
+                request_subnets=const.REQUEST_SUBNETS or None,
+                machine_enrollment_subnets=const.MACHINE_ENROLLMENT_SUBNETS or None,
+                admin_subnets=const.ADMIN_SUBNETS or None,
+
+                ocsp_subnets=const.OCSP_SUBNETS or None,
+                crl_subnets=const.CRL_SUBNETS or None,
+            ),
+            features=dict(
+                token=True,
+                tagging=True,
+                leases=True,
+                logging=True)
+        )

pinecrypt/server/api/signed.py

@@ -0,0 +1,84 @@
+
+import falcon
+import logging
+import json
+import hashlib
+from pinecrypt.server import authority, errors
+from pinecrypt.server.decorators import csrf_protection
+from .utils.firewall import login_required, authorize_admin
+
+logger = logging.getLogger(__name__)
+
+class SignedCertificateDetailResource(object):
+    def on_get_cn(self, req, resp, cn):
+        try:
+            id = authority.get_common_name_id(cn)
+        except ValueError:
+            raise falcon.HTTPNotFound("Unknown Common name",
+            "Object not found with common name %s" % cn)
+
+        id = authority.get_common_name_id(cn)
+        url = req.forwarded_uri.replace(cn,"id/%s" % id)
+
+        resp.status = falcon.HTTP_307
+        resp.location = url
+
+
+    def on_get(self, req, resp, id):
+        preferred_type = req.client_prefers(("application/json", "application/x-pem-file"))
+        try:
+            cert, cert_doc, pem_buf = authority.get_signed(mongo_id=id)
+        except errors.CertificateDoesNotExist:
+            logger.warning("Failed to serve non-existant certificate %s to %s",
+                id, req.context["remote"]["addr"])
+            raise falcon.HTTPNotFound()
+
+        cn = cert_doc["common_name"]
+
+        if preferred_type == "application/x-pem-file":
+            resp.set_header("Content-Type", "application/x-pem-file")
+            resp.set_header("Content-Disposition", ("attachment; filename=%s.pem" % cn))
+            resp.text = pem_buf
+            logger.debug("Served certificate %s to %s as application/x-pem-file",
+                cn, req.context["remote"]["addr"])
+        elif preferred_type == "application/json":
+            resp.set_header("Content-Type", "application/json")
+            resp.set_header("Content-Disposition", ("attachment; filename=%s.json" % cn))
+            try:
+                signer_username = cert_doc["user"]["signature"]["username"]
+            except KeyError:
+                signer_username = None
+
+            resp.text = json.dumps(dict(
+                common_name=cn,
+                id=str(cert_doc["_id"]),
+                signer=signer_username,
+                serial="%040x" % cert.serial_number,
+                organizational_unit=cert.subject.native.get("organizational_unit_name"),
+                signed=cert["tbs_certificate"]["validity"]["not_before"].native.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
+                expires=cert["tbs_certificate"]["validity"]["not_after"].native.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
+                sha256sum=hashlib.sha256(pem_buf).hexdigest(),
+                attributes=None,
+                lease=None,
+                extensions=dict([
+                    (e["extn_id"].native, e["extn_value"].native)
+                    for e in cert["tbs_certificate"]["extensions"]
+                    if e["extn_id"].native in ("extended_key_usage",)])
+
+            ))
+            logger.debug("Served certificate %s to %s as application/json",
+                cn, req.context["remote"]["addr"])
+        else:
+            logger.debug("Client did not accept application/json or application/x-pem-file")
+            raise falcon.HTTPUnsupportedMediaType(
+                "Client did not accept application/json or application/x-pem-file")
+
+    @csrf_protection
+    @login_required
+    @authorize_admin
+    def on_delete(self, req, resp, id):
+        authority.revoke(id,
+            reason=req.get_param("reason", default="key_compromise"),
+            user=req.context.get("user")
+        )
+

pinecrypt/server/api/tag.py

@@ -0,0 +1,64 @@
+from pinecrypt.server import db
+from pinecrypt.server.decorators import serialize, csrf_protection
+from .utils.firewall import login_required, authorize_admin
+
+logger = logging.getLogger(__name__)
+
+class TagResource(object):
+    @serialize
+    @login_required
+    @authorize_admin
+    def on_get(self, req, resp, id):
+        tags = db.certificates.find_one({"_id": db.ObjectId(id), "status": "signed"}).get("tags")
+        return tags
+
+    @csrf_protection
+    @login_required
+    @authorize_admin
+    def on_post(self, req, resp, id):
+        # TODO: Sanitize input
+        key, value = req.get_param("key", required=True), req.get_param("value", required=True)
+        db.certificates.update_one({
+            "_id": db.ObjectId(id),
+            "status": "signed"
+        }, {
+            "$addToSet": {"tags": "%s=%s" % (key, value)}
+        })
+
+
+class TagDetailResource(object):
+    @csrf_protection
+    @login_required
+    @authorize_admin
+    def on_put(self, req, resp, id, tag):
+        key = tag
+        if "=" in tag:
+            key, prev_value = tag.split("=")
+
+        value = req.get_param("value", required=True)
+        # TODO: Make atomic https://docs.mongodb.com/manual/reference/operator/update-array/
+        db.certificates.find_one_and_update({
+            "_id": db.ObjectId(id),
+            "status": "signed"
+        }, {
+            "$pull": {"tags": tag}
+        })
+
+        db.certificates.find_one_and_update({
+            "_id": db.ObjectId(id),
+            "status": "signed"
+        }, {
+            "$addToSet": {"tags": "%s=%s" % (key, value)}
+        })
+
+
+    @csrf_protection
+    @login_required
+    @authorize_admin
+    def on_delete(self, req, resp, id, tag):