diff --git a/.travis.yml b/.travis.yml
index 2b78140..f2acf9a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,9 +8,6 @@ after_success:
 virtualenv:
   system_site_packages: true
 install:
-  - sudo add-apt-repository ppa:nginx/stable  -y
-  - sudo apt update
-  - sudo apt install libnginx-mod-nchan
   - echo "127.0.0.1 localhost" | sudo tee /etc/hosts
   - echo "127.0.1.1 ca.example.lan ca" | sudo tee -a /etc/hosts
   - echo "127.0.0.1 vpn.example.lan vpn" | sudo tee -a /etc/hosts
diff --git a/certidude/cli.py b/certidude/cli.py
index e4980d5..881696c 100755
--- a/certidude/cli.py
+++ b/certidude/cli.py
@@ -729,9 +729,18 @@ def certidude_setup_openvpn_networkmanager(authority, remote, common_name, **pat
 def certidude_setup_authority(username, kerberos_keytab, nginx_config, country, state, locality, organization, organizational_unit, common_name, directory, authority_lifetime, push_server, outbox, server_flags):
     # Install only rarely changing stuff from OS package management
     apt("python-setproctitle cython python-dev libkrb5-dev libldap2-dev libffi-dev libssl-dev")
-    apt("python-mimeparse python-markdown python-xattr python-jinja2 python-cffi python-openssl")
+    apt("python-mimeparse python-markdown python-xattr python-jinja2 python-cffi python-openssl software-properties-common")
     pip("gssapi falcon cryptography humanize ipaddress simplepam humanize requests")
     click.echo("Software dependencies installed")
+
+    if not os.path.exists("/etc/apt/sources.list.d/nginx-stable-trusty.list"):
+        os.system("add-apt-repository -y ppa:nginx/stable")
+        os.system("apt update")
+    if not os.path.exists("/usr/lib/nginx/modules/ngx_nchan_module.so"):
+        os.system("apt install -y libnginx-mod-nchan")
+    if not os.path.exists("/usr/sbin/nginx"):
+        os.system("apt install -y nginx")
+
     from cryptography import x509
     from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID
     from cryptography.hazmat.backends import default_backend
diff --git a/tests/test_cli.py b/tests/test_cli.py
index bbccdba..c3c6c0b 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -79,12 +79,7 @@ def clean_client():
         pass
 
 
-def test_cli_setup_authority():
-    import os
-    import sys
-
-    assert os.getuid() == 0, "Run tests as root in a clean VM or container"
-
+def clean_server():
     if os.path.exists("/run/certidude/signer.pid"):
         with open("/run/certidude/signer.pid") as fh:
             try:
@@ -131,6 +126,18 @@ def test_cli_setup_authority():
         if os.path.exists("/etc/openvpn/keys"):
             shutil.rmtree("/etc/openvpn/keys")
 
+    # System packages
+    os.system("apt purge -y nginx libnginx-mod-nchan openvpn strongswan")
+    os.system("apt-get -y autoremove")
+
+
+def test_cli_setup_authority():
+    import os
+    import sys
+
+    assert os.getuid() == 0, "Run tests as root in a clean VM or container"
+
+    clean_server()
     clean_client()
 
     from certidude.cli import entry_point as cli
@@ -146,7 +153,8 @@ def test_cli_setup_authority():
 
     assert not result.exception, result.output
     assert os.getuid() == 0 and os.getgid() == 0, "Serve dropped permissions incorrectly!"
-
+    assert os.system("nginx -t") == 0, "invalid nginx configuration"
+    assert os.path.exists("/run/nginx.pid"), "nginx wasn't started up properly"
 
     from certidude import config, authority
     assert authority.ca_cert.serial_number >= 0x100000000000000000000000000000000000000
@@ -788,3 +796,5 @@ def test_cli_setup_authority():
     os.system("service nginx stop")
     os.system("service openvpn stop")
     os.system("ipsec stop")
+
+    clean_server()