lauri
/
certidude
mirror of https://github.com/laurivosandi/certidude
1
0
Fork 0
Code Issues Activity

Move to pre-forking model for backend API-s

This commit is contained in:
Lauri Võsandi 2018-10-05 10:41:40 +03:00
parent 2f301d4fec
commit 6e50c85c85
17 changed files with 289 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
certidude/api/__init__.py
View File

@ -2,10 +2,13 @@
import falcon
import ipaddress
import logging
import os
from certidude import config
from certidude.common import drop_privileges
from user_agents import parse
from wsgiref.simple_server import make_server, WSGIServer
from setproctitle import setproctitle
class NormalizeMiddleware(object):
def process_request(self, req, resp, *args):
@ -15,7 +18,66 @@ class NormalizeMiddleware(object):
else:
req.context["user_agent"] = "Unknown user agent"
def certidude_app(log_handlers=[]):
class App(object):
PORT = 8080
FORKS = None
DROP_PRIVILEGES = True
def __init__(self):
app = falcon.API(middleware=NormalizeMiddleware())
app.req_options.auto_parse_form_urlencoded = True
self.attach(app)
# Set up log handlers
log_handlers = []
if config.LOGGING_BACKEND == "sql":
from certidude.mysqllog import LogHandler
from certidude.api.log import LogResource
uri = config.cp.get("logging", "database")
log_handlers.append(LogHandler(uri))
elif config.LOGGING_BACKEND == "syslog":
from logging.handlers import SysLogHandler
log_handlers.append(SysLogHandler())
# Browsing syslog via HTTP is obviously not possible out of the box
elif config.LOGGING_BACKEND:
raise ValueError("Invalid logging.backend = %s" % config.LOGGING_BACKEND)
from certidude.push import EventSourceLogHandler
log_handlers.append(EventSourceLogHandler())
for j in logging.Logger.manager.loggerDict.values():
if isinstance(j, logging.Logger): # PlaceHolder is what?
if j.name.startswith("certidude."):
j.setLevel(logging.DEBUG)
for handler in log_handlers:
j.addHandler(handler)
self.server = make_server("127.0.1.1", self.PORT, app, WSGIServer)
setproctitle("certidude: %s" % self.NAME)
def run(self):
if self.DROP_PRIVILEGES:
drop_privileges()
try:
self.server.serve_forever()
except KeyboardInterrupt:
return
else:
return
def fork(self):
for j in range(self.FORKS):
if not os.fork():
self.run()
return True
return False
class ReadWriteApp(App):
NAME = "backend server"
def attach(self, app):
from certidude import authority, config
from certidude.tokens import TokenManager
from .signed import SignedCertificateDetailResource
@ -26,17 +88,15 @@ def certidude_app(log_handlers=[]):
from .attrib import AttributeResource
from .bootstrap import BootstrapResource
from .token import TokenResource
from .builder import ImageBuilderResource
from .session import SessionResource, CertificateAuthorityResource
app = falcon.API(middleware=NormalizeMiddleware())
app.req_options.auto_parse_form_urlencoded = True
from .revoked import RevokedCertificateDetailResource
# Certificate authority API calls
app.add_route("/api/certificate/", CertificateAuthorityResource())
app.add_route("/api/signed/{cn}/", SignedCertificateDetailResource(authority))
app.add_route("/api/request/{cn}/", RequestDetailResource(authority))
app.add_route("/api/request/", RequestListResource(authority))
app.add_route("/api/revoked/{serial_number}/", RevokedCertificateDetailResource(authority))
token_resource = None
token_manager = None
@ -69,35 +129,57 @@ def certidude_app(log_handlers=[]):
# Bootstrap resource
app.add_route("/api/bootstrap/", BootstrapResource(authority))
# LEDE image builder resource
app.add_route("/api/build/{profile}/{suggested_filename}", ImageBuilderResource())
# Add CRL handler if we have any whitelisted subnets
if config.CRL_SUBNETS:
from .revoked import RevocationListResource
app.add_route("/api/revoked/", RevocationListResource(authority))
# Add SCEP handler if we have any whitelisted subnets
if config.SCEP_SUBNETS:
from .scep import SCEPResource
app.add_route("/api/scep/", SCEPResource(authority))
return app
if config.OCSP_SUBNETS:
class ResponderApp(App):
PORT = 8081
FORKS = 4
NAME = "ocsp responder"
def attach(self, app):
from certidude import authority
from .ocsp import OCSPResource
app.add_sink(OCSPResource(authority), prefix="/api/ocsp")
return app
# Set up log handlers
if config.LOGGING_BACKEND == "sql":
from certidude.mysqllog import LogHandler
class RevocationListApp(App):
PORT = 8082
FORKS = 2
NAME = "crl server"
def attach(self, app):
from certidude import authority
from .revoked import RevocationListResource
app.add_route("/api/revoked/", RevocationListResource(authority))
return app
class BuilderApp(App):
PORT = 8083
FORKS = 1
NAME = "image builder"
def attach(self, app):
# LEDE image builder resource
from certidude import authority
from .builder import ImageBuilderResource
app.add_route("/api/build/{profile}/{suggested_filename}", ImageBuilderResource())
return app
class LogApp(App):
PORT = 8084
FORKS = 2
NAME = "log server"
def attach(self, app):
from certidude.api.log import LogResource
uri = config.cp.get("logging", "database")
log_handlers.append(LogHandler(uri))
app.add_route("/api/log/", LogResource(uri))
elif config.LOGGING_BACKEND == "syslog":
from logging.handlers import SysLogHandler
log_handlers.append(SysLogHandler())
# Browsing syslog via HTTP is obviously not possible out of the box
elif config.LOGGING_BACKEND:
raise ValueError("Invalid logging.backend = %s" % config.LOGGING_BACKEND)
return app

2
certidude/api/log.py
View File

@ -12,4 +12,4 @@ class LogResource(RelationalMixin):
def on_get(self, req, resp):
# TODO: Add last id parameter
return self.iterfetch("select * from log order by created desc limit ?",
req.get_param_as_int("limit"))
req.get_param_as_int("limit", required=True))

15
certidude/api/revoked.py
View File

@ -30,3 +30,18 @@ class RevocationListResource(AuthorityHandler):
logger.debug("Client %s asked revocation list in unsupported format" % req.context.get("remote_addr"))
raise falcon.HTTPUnsupportedMediaType(
"Client did not accept application/x-pkcs7-crl or application/x-pem-file")
class RevokedCertificateDetailResource(AuthorityHandler):
def on_get(self, req, resp, serial_number):
try:
path, buf, cert, signed, expires, revoked, reason = self.authority.get_revoked(serial_number)
except EnvironmentError:
logger.warning("Failed to serve non-existant revoked certificate with serial %s to %s",
serial_number, req.context.get("remote_addr"))
raise falcon.HTTPNotFound()
resp.set_header("Content-Type", "application/x-pem-file")
resp.set_header("Content-Disposition", ("attachment; filename=%x.pem" % cert.serial_number))
resp.body = buf
logger.debug("Served revoked certificate with serial %s to %s",
serial_number, req.context.get("remote_addr"))

8
certidude/api/session.py
View File

@ -85,12 +85,10 @@ class SessionResource(AuthorityHandler):
# Extract lease information from filesystem
try:
last_seen = datetime.strptime(getxattr(path, "user.lease.last_seen").decode("ascii"), "%Y-%m-%dT%H:%M:%S.%fZ")
lease = dict(
inner_address = getxattr(path, "user.lease.inner_address").decode("ascii"),
outer_address = getxattr(path, "user.lease.outer_address").decode("ascii"),
last_seen = last_seen,
age = datetime.utcnow() - last_seen
last_seen = datetime.strptime(getxattr(path, "user.lease.last_seen").decode("ascii"), "%Y-%m-%dT%H:%M:%S.%fZ")
)
except IOError: # No such attribute(s)
lease = None
@ -166,10 +164,6 @@ class SessionResource(AuthorityHandler):
hostname = const.FQDN,
tokens = self.token_manager.list() if self.token_manager else None,
tagging = [dict(name=t[0], type=t[1], title=t[2]) for t in config.TAG_TYPES],
lease = dict(
offline = 600, # Seconds from last seen activity to consider lease offline, OpenVPN reneg-sec option
dead = 604800 # Seconds from last activity to consider lease dead, X509 chain broken or machine discarded
),
certificate = dict(
algorithm = self.authority.public_key.algorithm,
common_name = self.authority.certificate.subject.native["common_name"],

2
certidude/authority.py
View File

@ -13,7 +13,7 @@ from asn1crypto.csr import CertificationRequest
from certbuilder import CertificateBuilder
from certidude import config, push, mailer, const
from certidude import errors
from certidude.common import cn_to_dn, generate_serial, random
from certidude.common import cn_to_dn, generate_serial
from crlbuilder import CertificateListBuilder, pem_armor_crl
from csrbuilder import CSRBuilder, pem_armor_csr
from datetime import datetime, timedelta

144
certidude/cli.py
View File

@ -23,6 +23,7 @@ from datetime import datetime, timedelta
from glob import glob
from ipaddress import ip_network
from oscrypto import asymmetric
from setproctitle import setproctitle
try:
import coverage
@ -43,6 +44,18 @@ logger = logging.getLogger(__name__)
NOW = datetime.utcnow()
def make_runtime_dirs(func):
def wrapped(**args):
# systemd doesn't support RuntimeDirectoryPreserve=yes on Xenial
# otherwise this should be part of service files
# with RuntimeDirectory=certidude
if not os.path.exists(const.RUN_DIR):
click.echo("Creating: %s" % const.RUN_DIR)
os.makedirs(const.RUN_DIR)
os.chmod(const.RUN_DIR, 0o755)
return func(**args)
return wrapped
def fqdn_required(func):
def wrapped(**args):
common_name = args.get("common_name")
@ -1024,6 +1037,12 @@ def certidude_provision_authority(username, kerberos_keytab, nginx_config, tls_c
assert subprocess.check_output(["/usr/bin/lsb_release", "-cs"]) in (b"trusty\n", b"xenial\n", b"bionic\n"), "Only Ubuntu 16.04 supported at the moment"
assert os.getuid() == 0 and os.getgid() == 0, "Authority can be set up only by root"
session = dict(
authority = dict(
hostname = common_name
)
)
def verbose_symlink(name, target):
if not os.path.islink(name):
click.echo("Symlinking %s to %s" % (name, target))
@ -1173,14 +1192,9 @@ def certidude_provision_authority(username, kerberos_keytab, nginx_config, tls_c
def verbose_render_systemd_service(template, target, context):
target_path = "/etc/systemd/system/%s" % target
if os.path.exists(target_path):
click.echo("File %s already exists, remove to regenerate" % target_path)
else:
buf = env.get_template(template).render(context)
with open(target_path, "w") as fh:
fh.write(buf)
click.echo("File %s created" % target_path)
os.system("systemctl daemon-reload")
verbose_symlink("/etc/nginx/sites-enabled/certidude.conf", "../sites-available/certidude.conf")
@ -1192,6 +1206,9 @@ def certidude_provision_authority(username, kerberos_keytab, nginx_config, tls_c
verbose_render_systemd_service("server/ldap-kinit.timer", "certidude-ldap-kinit.timer", vars())
verbose_render_systemd_service("snippets/nginx-ocsp-cache.service", "certidude-ocsp-cache.service", vars())
verbose_render_systemd_service("snippets/nginx-ocsp-cache.timer", "certidude-ocsp-cache.timer", vars())
verbose_render_systemd_service("server/housekeeping-daily.service", "certidude-housekeeping-daily.service", vars())
verbose_render_systemd_service("server/housekeeping-daily.timer", "certidude-housekeeping-daily.timer", vars())
os.system("systemctl daemon-reload")
else:
raise NotImplementedError("Not systemd based OS, don't know how to set up initscripts")
@ -1409,14 +1426,6 @@ def certidude_provision_authority(username, kerberos_keytab, nginx_config, tls_c
assert os.stat("/etc/nginx/sites-available/certidude.conf").st_mode == 0o100600
assert os.stat("/etc/certidude/server.conf").st_mode == 0o100600
# Disable legacy garbage
if os.path.exists("/etc/cron.hourly/certidude"):
os.unlink("/etc/cron.hourly/certidude")
if os.path.exists("/etc/cron.daily/certidude"):
os.unlink("/etc/cron.daily/certidude")
if os.path.exists("/etc/systemd/system/certidude.service"):
os.unlink("/etc/systemd/system/certidude.service")
click.echo("To enable e-mail notifications install Postfix as sattelite system and set mailer address in %s" % const.SERVER_CONFIG_PATH)
click.echo()
click.echo("Use following commands to inspect the newly created files:")
@ -1434,6 +1443,8 @@ def certidude_provision_authority(username, kerberos_keytab, nginx_config, tls_c
os.system("systemctl enable certidude-backend.service")
os.system("systemctl enable nginx")
os.system("systemctl enable certidude-ocsp-cache.timer")
os.system("systemctl enable certidude-housekeeping-daily.timer")
os.system("systemctl start certidude-housekeeping-daily.timer")
os.system("systemctl start certidude-ocsp-cache.timer")
if realm:
os.system("systemctl enable certidude-ldap-kinit.timer")
@ -1561,21 +1572,24 @@ def certidude_revoke(common_name, reason):
@click.command("kinit", help="Initialize Kerberos credential cache for LDAP")
@make_runtime_dirs
def certidude_housekeeping_kinit():
from certidude import config
# Update LDAP service ticket if Certidude is joined to domain
if os.path.exists("/etc/krb5.keytab"):
if not os.path.exists("/run/certidude"):
os.makedirs("/run/certidude")
if not os.path.exists("/etc/krb5.keytab"):
raise click.ClickException("No Kerberos keytab configured")
_, kdc = config.LDAP_ACCOUNTS_URI.rsplit("/", 1)
cmd = "KRB5CCNAME=/run/certidude/krb5cc.part kinit -k %s$ -S ldap/%s@%s -t /etc/krb5.keytab" % (
cmd = "KRB5CCNAME=%s.part kinit -k %s$ -S ldap/%s@%s -t /etc/krb5.keytab" % (
config.LDAP_GSSAPI_CRED_CACHE,
const.HOSTNAME.upper(), kdc, config.KERBEROS_REALM
)
click.echo("Executing: %s" % cmd)
os.system(cmd)
os.system("chown certidude:certidude /run/certidude/krb5cc.part")
os.rename("/run/certidude/krb5cc.part", "/run/certidude/krb5cc")
if os.system(cmd):
raise click.ClickException("Failed to initialize Kerberos credential cache!")
os.system("chown certidude:certidude %s.part" % config.LDAP_GSSAPI_CRED_CACHE)
os.rename("%s.part" % config.LDAP_GSSAPI_CRED_CACHE, config.LDAP_GSSAPI_CRED_CACHE)
@click.command("daily", help="Send notifications about expired certificates")
@ -1614,44 +1628,18 @@ def certidude_housekeeping_expiration():
# TODO: Send separate e-mails to subjects
@click.command("serve", help="Run server")
@click.option("-p", "--port", default=8080, help="Listen port")
@click.option("-l", "--listen", default="127.0.1.1", help="Listen address")
@click.command("serve", help="Run API backend server")
@click.option("-f", "--fork", default=False, is_flag=True, help="Fork to background")
def certidude_serve(port, listen, fork):
from certidude import authority, const, push
if port == 80:
click.echo("WARNING: Please run Certidude behind nginx, remote address is assumed to be forwarded by nginx!")
@make_runtime_dirs
def certidude_serve(fork):
from certidude import const, push, config
click.echo("Using configuration from: %s" % const.SERVER_CONFIG_PATH)
log_handlers = []
from certidude import config
click.echo("OCSP responder subnets: %s" % config.OCSP_SUBNETS)
click.echo("CRL subnets: %s" % config.CRL_SUBNETS)
click.echo("SCEP subnets: %s" % config.SCEP_SUBNETS)
click.echo("Loading signature profiles:")
for profile in config.PROFILES.values():
click.echo("- %s" % profile)
click.echo()
# Rebuild reverse mapping
for cn, path, buf, cert, signed, expires in authority.list_signed():
by_serial = os.path.join(config.SIGNED_BY_SERIAL_DIR, "%040x.pem" % cert.serial_number)
if not os.path.exists(by_serial):
click.echo("Linking %s to ../%s.pem" % (by_serial, cn))
os.symlink("../%s.pem" % cn, by_serial)
# Process directories
if not os.path.exists(const.RUN_DIR):
click.echo("Creating: %s" % const.RUN_DIR)
os.makedirs(const.RUN_DIR)
os.chmod(const.RUN_DIR, 0o755)
click.echo("Users subnets: %s" %
", ".join([str(j) for j in config.USER_SUBNETS]))
click.echo("Administrative subnets: %s" %
@ -1661,46 +1649,42 @@ def certidude_serve(port, listen, fork):
click.echo("Request submissions allowed from following subnets: %s" %
", ".join([str(j) for j in config.REQUEST_SUBNETS]))
click.echo("Serving API at %s:%d" % (listen, port))
from wsgiref.simple_server import make_server, WSGIServer
from certidude.api import certidude_app
click.echo("Listening on %s:%d" % (listen, port))
app = certidude_app(log_handlers)
httpd = make_server(listen, port, app, WSGIServer)
"""
Drop privileges
"""
from certidude.push import EventSourceLogHandler
log_handlers.append(EventSourceLogHandler())
for j in logging.Logger.manager.loggerDict.values():
if isinstance(j, logging.Logger): # PlaceHolder is what?
if j.name.startswith("certidude."):
j.setLevel(logging.DEBUG)
for handler in log_handlers:
j.addHandler(handler)
from certidude.api import ReadWriteApp, BuilderApp, ResponderApp, RevocationListApp, LogApp
if not fork or not os.fork():
pid = os.getpid()
with open(const.SERVER_PID_PATH, "w") as pidfile:
pidfile.write("%d\n" % pid)
# Rebuild reverse mapping
from certidude import authority
for cn, path, buf, cert, signed, expires in authority.list_signed():
by_serial = os.path.join(config.SIGNED_BY_SERIAL_DIR, "%040x.pem" % cert.serial_number)
if not os.path.exists(by_serial):
click.echo("Linking %s to ../%s.pem" % (by_serial, cn))
os.symlink("../%s.pem" % cn, by_serial)
push.publish("server-started")
logger.debug("Started Certidude at %s", const.FQDN)
drop_privileges()
try:
httpd.serve_forever()
except KeyboardInterrupt:
click.echo("Caught Ctrl-C, exiting...")
if fork and config.OCSP_SUBNETS:
click.echo("OCSP responder subnets: %s" % config.OCSP_SUBNETS)
if ResponderApp().fork():
return
if fork and config.CRL_SUBNETS:
click.echo("CRL subnets: %s" % config.CRL_SUBNETS)
if RevocationListApp().fork():
return
if fork:
if BuilderApp().fork():
return
if fork and config.LOGGING_BACKEND == "sql":
if LogApp().fork():
return
ReadWriteApp().run()
push.publish("server-stopped")
logger.debug("Shutting down Certidude")
logger.debug("Shutting down Certidude API backend")
return

5
certidude/common.py
View File

@ -2,6 +2,7 @@
import os
import click
import subprocess
from setproctitle import getproctitle
from random import SystemRandom
random = SystemRandom()
@ -98,8 +99,8 @@ def drop_privileges():
os.setgroups(restricted_groups)
os.setgid(gid)
os.setuid(uid)
click.echo("Switched to user %s (uid=%d, gid=%d); member of groups %s" %
("certidude", os.getuid(), os.getgid(), ", ".join([str(j) for j in os.getgroups()])))
click.echo("Switched %s (pid=%d) to user %s (uid=%d, gid=%d); member of groups %s" %
(getproctitle(), os.getpid(), "certidude", os.getuid(), os.getgid(), ", ".join([str(j) for j in os.getgroups()])))
os.umask(0o007)
def apt(packages):

1
certidude/const.py
View File

@ -22,7 +22,6 @@ PROFILE_CONFIG_PATH = os.path.join(CONFIG_DIR, "profile.conf")
CLIENT_CONFIG_PATH = os.path.join(CONFIG_DIR, "client.conf")
SERVICES_CONFIG_PATH = os.path.join(CONFIG_DIR, "services.conf")
SERVER_PID_PATH = os.path.join(RUN_DIR, "server.pid")
SERVER_LOG_PATH = "/var/log/certidude-server.log"
STORAGE_PATH = "/var/lib/certidude/"
try:

2
certidude/static/502.json
View File

@ -1,4 +1,4 @@
{
"title": "502 Bad Gateway",
"description": "It seems the server had bit of a hiccup, perhaps this helps: systemctl restart certidude && journalctl -f"
"description": "It seems the server had bit of a hiccup, perhaps this helps: systemctl restart certidude-backend && journalctl -f"
}

7
certidude/templates/server/housekeeping-daily.service Normal file
View File

@ -0,0 +1,7 @@
[Unit]
Description=Run daily housekeeping jobs, eg certificate expiration notifications
[Service]
Type=oneshot
ExecStart={{ certidude_path }} housekeeping daily

6
certidude/templates/server/housekeeping-daily.timer Normal file
View File

@ -0,0 +1,6 @@
[Timer]
Persistent=true
OnCalendar=daily
[Install]
WantedBy=multi-user.target

20
certidude/templates/server/nginx.conf
View File

@ -31,6 +31,16 @@ server {
server_name {{ common_name }};
listen 80 default_server;
# Proxy pass CRL server
location /api/revoked/ {
proxy_pass http://127.0.1.1:8082/api/revoked/;
}
# Proxy pass OCSP responder
location /api/ocsp/ {
proxy_pass http://127.0.1.1:8081/api/ocsp/;
}
# Proxy pass to backend
location /api/ {
proxy_pass http://127.0.1.1:8080/api/;
@ -90,6 +100,16 @@ server {
# once it has been configured
add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;";
# Proxy pass image builder
location /api/log/ {
proxy_pass http://127.0.1.1:8084/api/log/;
}
# Proxy pass image builder
location /api/builder/ {
proxy_pass http://127.0.1.1:8083/api/builder/;
}
# Proxy pass to backend
location /api/ {
proxy_pass http://127.0.1.1:8080/api/;

0
certidude/templates/server/responder.service Normal file
View File

17
certidude/templates/server/systemd.service
View File

@ -1,17 +0,0 @@
[Unit]
Description=Certidude server
After=network.target
[Service]
Type=simple
EnvironmentFile=/etc/environment
Environment=LANG=C.UTF-8
Environment=PYTHON_EGG_CACHE=/tmp/.cache
PIDFile=/run/certidude/server.pid
KillSignal=SIGINT
ExecStart={{ certidude_path }} serve
TimeoutSec=15
[Install]
WantedBy=multi-user.target

4
certidude/templates/snippets/nginx-ocsp-cache.service
View File

@ -1,7 +1,7 @@
[Unit]
Description=Cache OCSP responses for nginx OCSP stapling
Requires=nginx.service
[Service]
Type=oneshot
Requires=nginx.service
ExecStart=-/usr/bin/curl --cert-status https://{{ common_name }}:8443/ --cacert /etc/certidude/authority/{{ session.authority.hostname }}/ca_cert.pem
ExecStart=-/usr/bin/curl --cert-status https://{{ session.authority.hostname }}:8443/ --cacert /etc/certidude/authority/{{ session.authority.hostname }}/ca_cert.pem

1
requirements.txt
View File

@ -7,3 +7,4 @@ oscrypto
requests
jinja2
ipsecparse
setproctitle

10
tests/test_cli.py
View File

@ -7,6 +7,7 @@ import re
import shutil
import sys
from asn1crypto import pem, x509
from glob import glob
from oscrypto import asymmetric
from csrbuilder import CSRBuilder, pem_armor_csr
from asn1crypto.util import OrderedDict
@ -159,13 +160,11 @@ def clean_server():
files = [
"/etc/krb5.keytab",
"/etc/samba/smb.conf",
"/etc/certidude/server.conf",
"/etc/certidude/builder.conf",
"/etc/certidude/profile.conf",
"/etc/certidude/*.conf",
"/var/log/certidude.log",
"/etc/cron.daily/certidude",
"/etc/cron.hourly/certidude",
"/etc/systemd/system/certidude.service",
"/etc/systemd/system/certidude*",
"/etc/nginx/sites-available/ca.conf",
"/etc/nginx/sites-enabled/ca.conf",
"/etc/nginx/sites-available/certidude.conf",
@ -179,7 +178,8 @@ def clean_server():
"/usr/bin/node",
]
for filename in files:
for pattern in files:
for filename in glob(pattern):
try:
os.unlink(filename)
except: