use NGINX with acme.sh

.drone/common.sh

@@ -8,6 +8,9 @@ rm /tmp/resolv.conf
 
 # mock out systemctl
 ln -s /bin/true /usr/local/bin/systemctl
+# mock out acme.sh
+mkdir -p /root/.acme.sh
+ln -s /bin/true /root/.acme.sh/acme.sh
 
 get_ip_addr() {
     getent hosts $1 | cut -d' ' -f1

VERSION.txt

@@ -1 +1 @@
-1.0.12
+1.0.13

ceo_common/errors.py

@@ -83,3 +83,7 @@ class InvalidDomainError(Exception):
 class InvalidIPError(Exception):
     def __init__(self):
         super().__init__('IP address is invalid')
+
+
+class RateLimitError(Exception):
+    pass

ceod/api/error_handlers.py

@@ -8,7 +8,7 @@ from werkzeug.exceptions import HTTPException
 from ceo_common.errors import UserNotFoundError, GroupNotFoundError, \
     UserAlreadyExistsError, GroupAlreadyExistsError, BadRequest, \
     UserAlreadySubscribedError, InvalidMembershipError, \
-    CloudStackAPIError, InvalidDomainError, InvalidIPError
+    CloudStackAPIError, InvalidDomainError, InvalidIPError, RateLimitError
 from ceo_common.logger_factory import logger_factory
 
 __all__ = ['register_error_handlers']
@@ -28,8 +28,10 @@ def generic_error_handler(err: Exception):
         BadRequest, InvalidDomainError, InvalidIPError
     ]):
         status_code = 400
-    elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult) \
+    elif any(isinstance(err, cls) for cls in [
-            or isinstance(err, InvalidMembershipError):
+        ldap3.core.exceptions.LDAPStrongerAuthRequiredResult,
+        InvalidMembershipError, RateLimitError,
+    ]):
         status_code = 403
     elif isinstance(err, UserNotFoundError) or isinstance(err, GroupNotFoundError):
         status_code = 404

ceod/model/CloudService.py

@@ -14,7 +14,8 @@ from zope import component
 from zope.interface import implementer
 
 from .VHostManager import VHostManager
-from ceo_common.errors import CloudStackAPIError, InvalidDomainError, InvalidIPError
+from ceo_common.errors import CloudStackAPIError, InvalidDomainError, \
+    InvalidIPError, RateLimitError
 from ceo_common.logger_factory import logger_factory
 from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \
     IMailService
@@ -34,10 +35,10 @@ class CloudService:
         self.secret_key = cfg.get('cloudstack_secret_key')
         self.base_url = cfg.get('cloudstack_base_url')
         self.members_domain = 'Members'
-        self.vhost_mgr = VHostManager(
+
-            vhost_dir=cfg.get('cloud vhosts_config_dir'),
+        self.vhost_mgr = VHostManager()
-        )
         self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account')
+        self.vhost_rate_limit_secs = cfg.get('cloud vhosts_rate_limit_seconds')
         self.vhost_domain = cfg.get('cloud vhosts_members_domain')
         self.vhost_ip_min = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_min'))
         self.vhost_ip_max = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_max'))
@@ -46,6 +47,7 @@ class CloudService:
         if not os.path.isdir(state_dir):
             os.mkdir(state_dir)
         self.pending_deletions_file = os.path.join(state_dir, 'cloudstack_pending_account_deletions.json')
+        self.vhost_rate_limit_file = os.path.join(state_dir, 'vhost_rate_limits.json')
 
     def _create_url(self, params: Dict[str, str]) -> str:
         # See https://docs.cloudstack.apache.org/en/latest/developersguide/dev.html#the-cloudstack-api
@@ -203,6 +205,17 @@ class CloudService:
             return False
         return self.vhost_ip_min <= addr <= self.vhost_ip_max
 
+    def _check_rate_limit(self, username: str):
+        if os.path.exists(self.vhost_rate_limit_file):
+            d = json.load(open(self.vhost_rate_limit_file))
+        else:
+            d = {}
+        now = int(utils.get_current_datetime().timestamp())
+        if now - d.get(username, 0) < self.vhost_rate_limit_secs:
+            raise RateLimitError(f'Please wait {self.vhost_rate_limit_secs} seconds')
+        d[username] = now
+        json.dump(d, open(self.vhost_rate_limit_file, 'w'))
+
     def create_vhost(self, username: str, domain: str, ip_address: str):
         if self.vhost_mgr.get_num_vhosts(username) >= self.max_vhosts_per_account:
             raise Exception(f'Only {self.max_vhosts_per_account} vhosts '
@@ -211,6 +224,7 @@ class CloudService:
             raise InvalidDomainError()
         if not self._is_valid_ip_address(ip_address):
             raise InvalidIPError()
+        self._check_rate_limit(username)
         self.vhost_mgr.create_vhost(username, domain, ip_address)
 
     def delete_vhost(self, username: str, domain: str):

ceod/model/VHostManager.py

@@ -1,24 +1,41 @@
 import glob
 import os
 import re
+import shutil
 import subprocess
-from typing import List, Dict
+from typing import List, Dict, Tuple
 
 import jinja2
+from zope import component
 from zope.interface import implementer
 
 from ceo_common.logger_factory import logger_factory
-from ceo_common.interfaces import IVHostManager
+from ceo_common.interfaces import IVHostManager, IConfig
 
-REVERSE_PROXY_IP_RE = re.compile(r'^\s+reverse_proxy\s+http://(?P<ip_address>[\d.]+)$')
+PROXY_PASS_IP_RE = re.compile(r'^\s+proxy_pass\s+http://(?P<ip_address>[\d.]+);$')
 VHOST_FILENAME_RE = re.compile(r'^(?P<username>[0-9a-z-]+)_(?P<domain>[0-9a-z.-]+)$')
 logger = logger_factory(__name__)
 
 
 @implementer(IVHostManager)
 class VHostManager:
-    def __init__(self, vhost_dir: str):
+    def __init__(self):
-        self.vhost_dir = vhost_dir
+        cfg = component.getUtility(IConfig)
+
+        self.vhost_dir = cfg.get('cloud vhosts_vhost_dir')
+        self.ssl_dir = cfg.get('cloud vhosts_ssl_dir')
+        if not os.path.exists(self.vhost_dir):
+            os.makedirs(self.vhost_dir)
+        if not os.path.exists(self.ssl_dir):
+            os.makedirs(self.ssl_dir)
+
+        self.default_ssl_cert = cfg.get('cloud vhosts_default_ssl_cert')
+        self.default_ssl_key = cfg.get('cloud vhosts_default_ssl_key')
+
+        self.acme_challenge_dir = cfg.get('cloud vhosts_acme_challenge_dir')
+        self.acme_dir = '/root/.acme.sh'
+        self.acme_sh = os.path.join(self.acme_dir, 'acme.sh')
+
         self.jinja_env = jinja2.Environment(
             loader=jinja2.PackageLoader('ceod.model'),
         )
@@ -38,14 +55,49 @@ class VHostManager:
         """Return a list of all vhost files for this user."""
         return glob.glob(os.path.join(self.vhost_dir, username + '_*'))
 
+    def _run(self, args: List[str]):
+        subprocess.run(args, check=True)
+
     def _reload_web_server(self):
-        logger.debug('Reloading Caddy')
+        logger.debug('Reloading NGINX')
-        subprocess.run(['systemctl', 'reload', 'caddy'], check=True)
+        self._run(['systemctl', 'reload', 'nginx'])
+
+    def _get_cert_and_key_path(self, domain: str) -> Tuple[str, str]:
+        cert_path = f'{self.ssl_dir}/{domain}.chain'
+        key_path = f'{self.ssl_dir}/{domain}.key'
+        return cert_path, key_path
+
+    def _acquire_new_cert(self, domain: str, cert_path: str, key_path: str):
+        logger.info(f'issuing new certificate for {domain}')
+        self._run([
+            self.acme_sh, '--issue', '-d', domain,
+            '-w', self.acme_challenge_dir,
+        ])
+        logger.info(f'installing new certificate for {domain}')
+        self._run([
+            self.acme_sh, '--install-cert', '-d', domain,
+            '--key-file', key_path,
+            '--fullchain-file', cert_path,
+            '--reloadcmd', 'systemctl reload nginx',
+        ])
+
+    def _delete_cert(self, domain: str, cert_path: str, key_path: str):
+        logger.info(f'removing certificate for {domain}')
+        self._run([self.acme_sh, '--remove', '-d', domain])
+        if os.path.exists(os.path.join(self.acme_dir, domain)):
+            shutil.rmtree(os.path.join(self.acme_dir, domain))
+        os.unlink(cert_path)
+        os.unlink(key_path)
 
     def create_vhost(self, username: str, domain: str, ip_address: str):
-        template = self.jinja_env.get_template('caddy_cloud_vhost_config.j2')
+        cert_path, key_path = self._get_cert_and_key_path(domain)
+        if not (os.path.exists(cert_path) and os.path.exists(key_path)):
+            self._acquire_new_cert(domain, cert_path, key_path)
+
+        template = self.jinja_env.get_template('nginx_cloud_vhost_config.j2')
         body = template.render(
-            username=username, domain=domain, ip_address=ip_address)
+            username=username, domain=domain, ip_address=ip_address,
+            ssl_cert_path=cert_path, ssl_key_path=key_path)
         filepath = self._vhost_filepath(username, domain)
         logger.info(f'Writing a new vhost ({domain} -> {ip_address}) to {filepath}')
         with open(filepath, 'w') as fo:
@@ -53,6 +105,10 @@ class VHostManager:
         self._reload_web_server()
 
     def delete_vhost(self, username: str, domain: str):
+        cert_path, key_path = self._get_cert_and_key_path(domain)
+        if os.path.exists(cert_path) and os.path.exists(key_path):
+            self._delete_cert(domain, cert_path, key_path)
+
         filepath = self._vhost_filepath(username, domain)
         logger.info(f'Deleting {filepath}')
         os.unlink(filepath)
@@ -70,7 +126,7 @@ class VHostManager:
             domain = match.group('domain')
             ip_address = None
             for line in open(filepath):
-                match = REVERSE_PROXY_IP_RE.match(line)
+                match = PROXY_PASS_IP_RE.match(line)
                 if match is None:
                     continue
                 ip_address = match.group('ip_address')

ceod/model/templates/caddy_cloud_vhost_config.j2

@@ -1,22 +0,0 @@
-# This file is automatically managed by ceod.
-# DO NOT EDIT THIS FILE MANUALLY UNLESS YOU KNOW WHAT YOU ARE DOING.
-
-{{ domain }} {
-  reverse_proxy http://{{ ip_address }}
-  log {
-    output file /var/log/caddy/member_{{ username }}.log {
-      roll_size 5MiB
-      roll_keep 2
-    }
-    format filter {
-      wrap json
-      fields {
-        request>headers delete
-        request>tls delete
-        resp_headers delete
-        user_id delete
-        common_log delete
-      }
-    }
-  }
-}

ceod/model/templates/nginx_cloud_vhost_config.j2

@@ -0,0 +1,24 @@
+# This file is automatically managed by ceod.
+# DO NOT EDIT THIS FILE MANUALLY UNLESS YOU KNOW WHAT YOU ARE DOING.
+
+server {
+  listen 80;
+  listen [::]:80;
+  server_name {{ domain }};
+  return 301 https://$host$request_uri;
+}
+
+server {
+  listen 443 ssl;
+  listen [::]:443 ssl;
+  server_name {{ domain }};
+  ssl_certificate {{ ssl_cert_path }};
+  ssl_certificate_key {{ ssl_key_path }};
+
+  location / {
+    proxy_pass http://{{ ip_address }};
+  }
+
+  access_log /var/log/nginx/member-{{ username }}-access.log;
+  error_log /var/log/nginx/member-{{ username }}-error.log;
+}

debian/ceod.service

@@ -8,6 +8,8 @@ After=network.target
 Type=exec
 EnvironmentFile=/etc/default/ceod
 WorkingDirectory=/var/lib/ceo
+RuntimeDirectory=ceod
+RuntimeDirectoryPreserve=yes
 ExecStart=/var/lib/ceo/venv/bin/gunicorn $GUNICORN_ARGS 'ceod.api:create_app()'
 # TODO: once the mail container is no longer running in LXC, we should add
 # some security protections here, like ProtectSystem.

etc/ceod.ini

@@ -81,7 +81,12 @@ secret_key = REPLACE_ME
 base_url = http://localhost:8080/client/api
 
 [cloud vhosts]
-config_dir = /etc/caddy/ceod-member-vhosts
+acme_challenge_dir = /var/www
+vhost_dir = /etc/nginx/ceod/member-vhosts
+ssl_dir = /etc/nginx/ceod/member-ssl
+default_ssl_cert = /etc/ssl/private/csclub.cloud.chain
+default_ssl_key = /etc/ssl/private/csclub.cloud.key
+rate_limit_seconds = 10
 max_vhosts_per_account = 10
 members_domain = csclub.cloud
 ip_range_min = 172.19.134.10

tests/ceod/api/test_cloud.py

@@ -89,6 +89,7 @@ def test_purge_accounts(
 def test_cloud_vhosts(cfg, client, new_user, ldap_conn):
     members_domain = cfg.get('cloud vhosts_members_domain')
     max_vhosts = cfg.get('cloud vhosts_max_vhosts_per_account')
+    rate_limit_secs = cfg.get('cloud vhosts_rate_limit_seconds')
     uid = new_user.uid
 
     domain1 = uid + '.' + members_domain
@@ -101,55 +102,66 @@ def test_cloud_vhosts(cfg, client, new_user, ldap_conn):
     assert status == 200
     assert data == {'vhosts': [{'domain': domain1, 'ip_address': ip1}]}
 
-    # invalid domain name
+    # rate limit
-    domain2 = uid + 'cloud.' + cfg.get('base_domain')
-    ip2 = ip1
-    status, _ = client.put(
-        f'/api/cloud/vhosts/{domain2}', json={'ip_address': ip2},
-        principal=uid)
-    assert status == 400
-
-    # invalid IP address
-    domain3 = domain1
-    ip3 = '129.97.134.10'
-    status, _ = client.put(
-        f'/api/cloud/vhosts/{domain3}', json={'ip_address': ip3},
-        principal=uid)
-    assert status == 400
-
-    # new vhost with same domain should replace old one
-    domain4 = domain1
-    ip4 = '172.19.134.14'
-    status, _ = client.put(
-        f'/api/cloud/vhosts/{domain4}', json={'ip_address': ip4},
-        principal=uid)
-    assert status == 200
-    status, data = client.get('/api/cloud/vhosts', principal=uid)
-    assert status == 200
-    assert data == {'vhosts': [{'domain': domain4, 'ip_address': ip4}]}
-
-    # maximum number of vhosts
-    for i in range(max_vhosts):
-        domain = 'app' + str(i + 1) + '.' + uid + '.' + members_domain
-        status, _ = client.put(
-            f'/api/cloud/vhosts/{domain}', json={'ip_address': ip1},
-            principal=uid)
-        if i < max_vhosts - 1:
-            assert status == 200
-        else:
-            assert status != 200
-
-    # delete a vhost
-    status, _ = client.delete(f'/api/cloud/vhosts/{domain1}', principal=uid)
-    assert status == 200
-
-    # expired members may not create vhosts
-    expire_member(new_user, ldap_conn)
     status, _ = client.put(
         f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1},
         principal=uid)
     assert status == 403
 
+    now = ceo_common_utils.get_current_datetime()
+    with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock:
+        now_mock.return_value = now + datetime.timedelta(seconds=rate_limit_secs)
+
+        # invalid domain name
+        domain2 = uid + 'cloud.' + cfg.get('base_domain')
+        ip2 = ip1
+        status, _ = client.put(
+            f'/api/cloud/vhosts/{domain2}', json={'ip_address': ip2},
+            principal=uid)
+        assert status == 400
+
+        # invalid IP address
+        domain3 = domain1
+        ip3 = '129.97.134.10'
+        status, _ = client.put(
+            f'/api/cloud/vhosts/{domain3}', json={'ip_address': ip3},
+            principal=uid)
+        assert status == 400
+
+        # new vhost with same domain should replace old one
+        domain4 = domain1
+        ip4 = '172.19.134.14'
+        status, _ = client.put(
+            f'/api/cloud/vhosts/{domain4}', json={'ip_address': ip4},
+            principal=uid)
+        assert status == 200
+        status, data = client.get('/api/cloud/vhosts', principal=uid)
+        assert status == 200
+        assert data == {'vhosts': [{'domain': domain4, 'ip_address': ip4}]}
+
+        # maximum number of vhosts
+        for i in range(max_vhosts):
+            now_mock.return_value += datetime.timedelta(seconds=rate_limit_secs)
+            domain = 'app' + str(i + 1) + '.' + uid + '.' + members_domain
+            status, _ = client.put(
+                f'/api/cloud/vhosts/{domain}', json={'ip_address': ip1},
+                principal=uid)
+            if i < max_vhosts - 1:
+                assert status == 200
+            else:
+                assert status != 200
+
+        # delete a vhost
+        status, _ = client.delete(f'/api/cloud/vhosts/{domain1}', principal=uid)
+        assert status == 200
+
+        # expired members may not create vhosts
+        expire_member(new_user, ldap_conn)
+        status, _ = client.put(
+            f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1},
+            principal=uid)
+        assert status == 403
+
 
 def test_cloud_vhosts_purged_account(
     cfg, client, mock_cloud_server, mock_mail_server, cloud_srv, new_user,

tests/ceod_dev.ini

@@ -75,7 +75,12 @@ secret_key = REPLACE_ME
 base_url = http://localhost:8080/client/api
 
 [cloud vhosts]
-config_dir = /run/ceod/member-vhosts
+acme_challenge_dir = /var/www
+vhost_dir = /run/ceod/member-vhosts
+ssl_dir = /run/ceod/member-ssl
+default_ssl_cert = /etc/ssl/private/csclub.cloud.chain
+default_ssl_key = /etc/ssl/private/csclub.cloud.key
+rate_limit_seconds = 10
 max_vhosts_per_account = 10
 members_domain = csclub.cloud
 ip_range_min = 172.19.134.10

tests/ceod_test_local.ini

@@ -74,7 +74,12 @@ secret_key = REPLACE_ME
 base_url = http://localhost:8080/client/api
 
 [cloud vhosts]
-config_dir = /run/ceod/member-vhosts
+acme_challenge_dir = /var/www
+vhost_dir = /run/ceod/member-vhosts
+ssl_dir = /run/ceod/member-ssl
+default_ssl_cert = /etc/ssl/private/csclub.cloud.chain
+default_ssl_key = /etc/ssl/private/csclub.cloud.key
+rate_limit_seconds = 10
 max_vhosts_per_account = 10
 members_domain = csclub.cloud
 ip_range_min = 172.19.134.10

tests/conftest.py

@@ -269,12 +269,12 @@ def postgresql_srv(cfg):
 
 @pytest.fixture(scope='session')
 def vhost_dir_setup(cfg):
-    vhost_dir = cfg.get('cloud vhosts_config_dir')
+    state_dir = '/run/ceod'
-    if os.path.isdir(vhost_dir):
+    if os.path.isdir(state_dir):
-        shutil.rmtree(vhost_dir)
+        shutil.rmtree(state_dir)
-    os.makedirs(vhost_dir)
+    os.makedirs(state_dir)
     yield
-    shutil.rmtree(vhost_dir)
+    shutil.rmtree(state_dir)
 
 
 @pytest.fixture(scope='session')