add shadowExpire to api test member assert thing

make create_user fixtures function scope
add shadowExpre atttribute to IUser
fix Term start month calulation

.drone/common.sh

@@ -6,6 +6,9 @@ sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /
 cp /tmp/resolv.conf /etc/resolv.conf
 rm /tmp/resolv.conf
 
+# mock out systemctl
+ln -s /bin/true /usr/local/bin/systemctl
+
 get_ip_addr() {
     getent hosts $1 | cut -d' ' -f1
 }

.drone/data.ldif

@@ -94,7 +94,7 @@ objectClass: posixAccount
 objectClass: shadowAccount
 objectClass: member
 program: MAT/Mathematics Computer Science
-term: s2021
+term: f2021
 
 dn: cn=ctdalek,ou=Group,dc=csclub,dc=internal
 objectClass: top
@@ -119,7 +119,7 @@ objectClass: posixAccount
 objectClass: shadowAccount
 objectClass: member
 program: MAT/Mathematics Computer Science
-term: s2021
+term: f2021
 
 dn: cn=regular1,ou=Group,dc=csclub,dc=internal
 objectClass: top
@@ -144,7 +144,7 @@ objectClass: posixAccount
 objectClass: shadowAccount
 objectClass: member
 program: MAT/Mathematics Computer Science
-term: s2021
+term: f2021
 
 dn: cn=exec1,ou=Group,dc=csclub,dc=internal
 objectClass: top

.drone/mail-setup.sh

@@ -8,14 +8,18 @@ set -ex
 add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail
 add_fqdn_to_hosts $(get_ip_addr auth1) auth1
 
-[ -f venv/bin/activate ] && . venv/bin/activate
-python tests/MockMailmanServer.py &
-python tests/MockSMTPServer.py &
+. venv/bin/activate
+python -m tests.MockMailmanServer &
+python -m tests.MockSMTPServer &
+python -m tests.MockCloudStackServer &
 
 export DEBIAN_FRONTEND=noninteractive
 apt update
 apt install -y netcat-openbsd
 auth_setup mail
 
+# for the VHostManager
+mkdir -p /run/ceod/member-vhosts
+
 # sync with phosphoric-acid
 nc -l 0.0.0.0 9000 &

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 .vscode/
 *.o
 *.so
+*.swp
 .idea/
 /docs/*.1
 /docs/*.5

README.md

@@ -1,5 +1,5 @@
 # pyceo
-[![Build Status](https://ci.csclub.uwaterloo.ca/api/badges/public/pyceo/status.svg?ref=refs/heads/v1)](https://ci.csclub.uwaterloo.ca/public/pyceo)
+[![Build Status](https://ci.csclub.uwaterloo.ca/api/badges/public/pyceo/status.svg)](https://ci.csclub.uwaterloo.ca/public/pyceo)
 
 CEO (**C**SC **E**lectronic **O**ffice) is the tool used by CSC to manage
 club accounts and memberships. See [docs/architecture.md](docs/architecture.md) for an
@@ -14,7 +14,7 @@ Docker containers instead, which are much easier to work with than the VM.
 
 First, make sure you create the virtualenv:
 ```sh
-docker run --rm -v "$PWD:$PWD" -w "$PWD" -u $(id -u):$(id -g) python:3.7-buster \
+docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \
     sh -c 'python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
 ```
 Then bring up the containers:
@@ -239,6 +239,9 @@ apt install devscripts debhelper git-buildpackage
 ```
 Make sure to also install all of the packages in the 'Build-Depends' section in debian/control.
 
+There are two important files to change before creating a new package: debian/changelog
+(which can be edited by running `dch -i`), and VERSION.txt.
+
 Make sure you git commit your changes *before* building the packages.
 
 To build unsigned packages:

VERSION.txt

@@ -1 +1 @@
-1.0.9
+1.0.12

ceo/cli/cloud.py

@@ -0,0 +1,85 @@
+import click
+from zope import component
+
+from ceo_common.interfaces import IConfig
+
+from ..utils import http_post, http_put, http_get, http_delete
+from .utils import Abort, handle_sync_response, print_colon_kv
+
+
+@click.group(short_help='Perform operations on the CSC cloud')
+def cloud():
+    pass
+
+
+@cloud.group(short_help='Manage your cloud account')
+def account():
+    pass
+
+
+@account.command(short_help='Activate your cloud account')
+def activate():
+    cfg = component.getUtility(IConfig)
+    base_domain = cfg.get('base_domain')
+
+    resp = http_post('/api/cloud/accounts/create')
+    handle_sync_response(resp)
+    lines = [
+        'Congratulations! Your cloud account has been activated.',
+        f'You may now login into https://cloud.{base_domain} with your CSC credentials.',
+        "Make sure to enter 'Members' for the domain (no quotes).",
+    ]
+    for line in lines:
+        click.echo(line)
+
+
+@cloud.group(short_help='Manage cloud accounts')
+def accounts():
+    pass
+
+
+@accounts.command(short_help='Purge expired cloud accounts')
+def purge():
+    resp = http_post('/api/cloud/accounts/purge')
+    result = handle_sync_response(resp)
+    click.echo('Accounts to be deleted: ' + ','.join(result['accounts_to_be_deleted']))
+    click.echo('Accounts which were deleted: ' + ','.join(result['accounts_deleted']))
+
+
+@cloud.group(short_help='Manage your virtual hosts')
+def vhosts():
+    pass
+
+
+@vhosts.command(name='add', short_help='Add a virtual host')
+@click.argument('domain')
+@click.argument('ip_address')
+def add_vhost(domain, ip_address):
+    body = {'ip_address': ip_address}
+    if '/' in domain:
+        raise Abort('invalid domain name')
+    resp = http_put('/api/cloud/vhosts/' + domain, json=body)
+    handle_sync_response(resp)
+    click.echo('Done.')
+
+
+@vhosts.command(name='delete', short_help='Delete a virtual host')
+@click.argument('domain')
+def delete_vhost(domain):
+    if '/' in domain:
+        raise Abort('invalid domain name')
+    resp = http_delete('/api/cloud/vhosts/' + domain)
+    handle_sync_response(resp)
+    click.echo('Done.')
+
+
+@vhosts.command(name='list', short_help='List virtual hosts')
+def list_vhosts():
+    resp = http_get('/api/cloud/vhosts')
+    result = handle_sync_response(resp)
+    vhosts = result['vhosts']
+    if not vhosts:
+        click.echo('No vhosts found.')
+        return
+    pairs = [(d['domain'], d['ip_address']) for d in vhosts]
+    print_colon_kv(pairs)

ceo/cli/entrypoint.py

@@ -7,6 +7,7 @@ from .updateprograms import updateprograms
 from .mysql import mysql
 from .postgresql import postgresql
 from .mailman import mailman
+from .cloud import cloud
 
 
 @click.group()
@@ -21,3 +22,4 @@ cli.add_command(updateprograms)
 cli.add_command(mysql)
 cli.add_command(postgresql)
 cli.add_command(mailman)
+cli.add_command(cloud)

ceo/utils.py

@@ -18,6 +18,8 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response:
         host = cfg.get('ceod_database_host')
     elif path.startswith('/api/mailman'):
         host = cfg.get('ceod_mailman_host')
+    elif path.startswith('/api/cloud'):
+        host = cfg.get('ceod_cloud_host')
     else:
         host = cfg.get('ceod_admin_host')
     return client.request(
@@ -40,6 +42,10 @@ def http_delete(path: str, **kwargs) -> requests.Response:
     return http_request('DELETE', path, **kwargs)
 
 
+def http_put(path: str, **kwargs) -> requests.Response:
+    return http_request('PUT', path, **kwargs)
+
+
 def get_failed_operations(data: List[Dict]) -> List[str]:
     """
     Get a list of the failed operations using the JSON objects

ceo_common/errors.py

@@ -64,3 +64,22 @@ class DatabaseConnectionError(Exception):
 class DatabasePermissionError(Exception):
     def __init__(self):
         super().__init__('unable to perform action due to lack of permissions')
+
+
+class InvalidMembershipError(Exception):
+    def __init__(self):
+        super().__init__('membership is invalid or expired')
+
+
+class CloudStackAPIError(Exception):
+    pass
+
+
+class InvalidDomainError(Exception):
+    def __init__(self):
+        super().__init__('domain is invalid')
+
+
+class InvalidIPError(Exception):
+    def __init__(self):
+        super().__init__('IP address is invalid')

ceo_common/interfaces/ICloudService.py

@@ -0,0 +1,42 @@
+from typing import Dict, List
+
+from zope.interface import Interface
+
+from .IUser import IUser
+
+
+class ICloudService(Interface):
+    """Performs operations on the CSC Cloud."""
+
+    def create_account(user: IUser):
+        """
+        Activate an LDAP account in CloudStack for the given user.
+        """
+
+    def purge_accounts() -> Dict:
+        """
+        Delete CloudStack accounts which correspond to expired CSC accounts.
+        A warning message will be emailed to users one week before their
+        cloud account is deleted.
+        Another message will be emailed to the users after their cloud account
+        has been deleted.
+        """
+
+    def create_vhost(username: str, domain: str, ip_address: str):
+        """
+        Create a new vhost record for the given domain and IP address.
+        """
+
+    def delete_vhost(username: str, domain: str):
+        """
+        Delete the vhost record for the given user and domain.
+        """
+
+    def get_vhosts(username: str) -> List[Dict]:
+        """
+        Get the vhost records for the given user. Each record has the form
+        {
+            "domain": "app.username.m.csclub.cloud",
+            "ip_address": "172.19.134.12"
+        }
+        """

ceo_common/interfaces/IUser.py

@@ -84,3 +84,6 @@ class IUser(Interface):
         If get_forwarding_addresses is True, the forwarding addresses
         for the user will also be returned, if present.
         """
+
+    def membership_is_valid() -> bool:
+        """Returns True iff the user's has a non-expired membership."""

ceo_common/interfaces/IVHostManager.py

@@ -0,0 +1,36 @@
+from typing import List, Dict
+
+from zope.interface import Interface
+
+
+class IVHostManager(Interface):
+    """Performs operations on the CSC Cloud."""
+
+    def create_vhost(username: str, domain: str, ip_address: str):
+        """
+        Create a new vhost record for the given domain and IP address.
+        """
+
+    def delete_vhost(username: str, domain: str):
+        """
+        Delete the vhost record for the given user and domain.
+        """
+
+    def delete_all_vhosts_for_user(username: str):
+        """
+        Delete all vhost records for the given user.
+        """
+
+    def get_num_vhosts(username: str) -> int:
+        """
+        Get the number of vhost records for the given user.
+        """
+
+    def get_vhosts(username: str) -> List[Dict]:
+        """
+        Get the vhost records for the given user. Each record has the form
+        {
+            "domain": "app.username.m.csclub.cloud",
+            "ip_address": "172.19.134.12"
+        }
+        """

ceo_common/interfaces/__init__.py

@@ -1,3 +1,4 @@
+from .ICloudService import ICloudService
 from .IKerberosService import IKerberosService
 from .IConfig import IConfig
 from .IUser import IUser
@@ -9,3 +10,4 @@ from .IMailService import IMailService
 from .IMailmanService import IMailmanService
 from .IHTTPClient import IHTTPClient
 from .IDatabaseService import IDatabaseService
+from .IVHostManager import IVHostManager

ceo_common/model/HTTPClient.py

@@ -27,8 +27,8 @@ class HTTPClient:
             host = host + '.' + self.base_domain
 
         if method == 'GET':
-            # This is the only GET endpoint which requires auth
-            need_auth = path.startswith('/api/members')
+            need_auth = path.startswith('/api/members') or \
+                path.startswith('/api/cloud')
             delegate = False
         else:
             need_auth = True

ceo_common/model/Term.py

@@ -1,5 +1,7 @@
 import datetime
 
+import ceo_common.utils as utils
+
 
 class Term:
     """A representation of a term in the CSC LDAP, e.g. 's2021'."""
@@ -17,7 +19,7 @@ class Term:
     @staticmethod
     def current():
         """Get a Term object for the current date."""
-        dt = datetime.datetime.now()
+        dt = utils.get_current_datetime()
         c = 'w'
         if 5 <= dt.month <= 8:
             c = 's'
@@ -70,3 +72,10 @@ class Term:
 
     def __le__(self, other):
         return self < other or self == other
+
+    def to_datetime(self) -> datetime.datetime:
+        c = self.s_term[0]
+        year = int(self.s_term[1:])
+        month = self.seasons.index(c) * 4 + 1
+        day = 1
+        return datetime.datetime(year, month, day)

ceo_common/utils.py

@@ -0,0 +1,7 @@
+import datetime
+
+
+def get_current_datetime() -> datetime.datetime:
+    # We place this in a separate function so that we can mock it out
+    # in our unit tests.
+    return datetime.datetime.now()

ceod/api/app_factory.py

@@ -7,11 +7,12 @@ from zope import component
 
 from .error_handlers import register_error_handlers
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
-    IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService
+    IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
+    ICloudService
 from ceo_common.model import Config, HTTPClient, RemoteMailmanService
 from ceod.api.spnego import init_spnego
 from ceod.model import KerberosService, LDAPService, FileService, \
-    MailmanService, MailService, UWLDAPService
+    MailmanService, MailService, UWLDAPService, CloudService
 from ceod.db import MySQLService, PostgreSQLService
 
 
@@ -41,6 +42,10 @@ def create_app(flask_config={}):
         from ceod.api import database
         app.register_blueprint(database.bp, url_prefix='/api/db')
 
+    if hostname == cfg.get('ceod_cloud_host'):
+        from ceod.api import cloud
+        app.register_blueprint(cloud.bp, url_prefix='/api/cloud')
+
     from ceod.api import groups
     app.register_blueprint(groups.bp, url_prefix='/api/groups')
 
@@ -118,3 +123,8 @@ def register_services(app):
     if hostname == cfg.get('ceod_database_host'):
         psql_srv = PostgreSQLService()
         component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
+
+    # CloudService
+    if hostname == cfg.get('ceod_cloud_host'):
+        cloud_srv = CloudService()
+        component.provideUtility(cloud_srv, ICloudService)

ceod/api/cloud.py

@@ -0,0 +1,51 @@
+from flask import Blueprint, request
+from zope import component
+
+from .utils import requires_authentication_no_realm, authz_restrict_to_syscom, \
+    get_valid_member_or_throw
+from ceo_common.interfaces import ICloudService
+
+bp = Blueprint('cloud', __name__)
+
+
+@bp.route('/accounts/create', methods=['POST'])
+@requires_authentication_no_realm
+def create_account(auth_user: str):
+    user = get_valid_member_or_throw(auth_user)
+    cloud_srv = component.getUtility(ICloudService)
+    cloud_srv.create_account(user)
+    return {'status': 'OK'}
+
+
+@bp.route('/accounts/purge', methods=['POST'])
+@authz_restrict_to_syscom
+def purge_accounts():
+    cloud_srv = component.getUtility(ICloudService)
+    return cloud_srv.purge_accounts()
+
+
+@bp.route('/vhosts/<domain>', methods=['PUT'])
+@requires_authentication_no_realm
+def create_vhost(auth_user: str, domain: str):
+    get_valid_member_or_throw(auth_user)
+    cloud_srv = component.getUtility(ICloudService)
+    body = request.get_json(force=True)
+    ip_address = body['ip_address']
+    cloud_srv.create_vhost(auth_user, domain, ip_address)
+    return {'status': 'OK'}
+
+
+@bp.route('/vhosts/<domain>', methods=['DELETE'])
+@requires_authentication_no_realm
+def delete_vhost(auth_user: str, domain: str):
+    cloud_srv = component.getUtility(ICloudService)
+    cloud_srv.delete_vhost(auth_user, domain)
+    return {'status': 'OK'}
+
+
+@bp.route('/vhosts', methods=['GET'])
+@requires_authentication_no_realm
+def get_vhosts(auth_user: str):
+    cloud_srv = component.getUtility(ICloudService)
+    vhosts = cloud_srv.get_vhosts(auth_user)
+    return {'vhosts': vhosts}

ceod/api/error_handlers.py

@@ -1,10 +1,14 @@
 import traceback
 
+from flask import request
 from flask.app import Flask
 import ldap3
 from werkzeug.exceptions import HTTPException
 
-from ceo_common.errors import UserNotFoundError, GroupNotFoundError
+from ceo_common.errors import UserNotFoundError, GroupNotFoundError, \
+    UserAlreadyExistsError, GroupAlreadyExistsError, BadRequest, \
+    UserAlreadySubscribedError, InvalidMembershipError, \
+    CloudStackAPIError, InvalidDomainError, InvalidIPError
 from ceo_common.logger_factory import logger_factory
 
 __all__ = ['register_error_handlers']
@@ -20,11 +24,28 @@ def generic_error_handler(err: Exception):
     """Return JSON for all errors."""
     if isinstance(err, HTTPException):
         status_code = err.code
+    elif any(isinstance(err, cls) for cls in [
+        BadRequest, InvalidDomainError, InvalidIPError
+    ]):
+        status_code = 400
+    elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult) \
+            or isinstance(err, InvalidMembershipError):
+        status_code = 403
     elif isinstance(err, UserNotFoundError) or isinstance(err, GroupNotFoundError):
         status_code = 404
-    elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult):
-        status_code = 403
+    elif any(isinstance(err, cls) for cls in [
+        UserAlreadyExistsError, GroupAlreadyExistsError, UserAlreadySubscribedError
+    ]):
+        status_code = 409
+    elif isinstance(err, CloudStackAPIError):
+        status_code = 500
     else:
         status_code = 500
         logger.error(traceback.format_exc())
+        if request.path.startswith('/api/cloud'):
+            # I've noticed that the requests library spits out the
+            # full URL when an Exception is raised, which will cause
+            # our CloudStack API key to be leaked. So we're going to mask
+            # it here instead.
+            err = Exception('Please contact the Systems Committee')
     return {'error': type(err).__name__ + ': ' + str(err)}, status_code

ceod/api/utils.py

@@ -7,14 +7,25 @@ import traceback
 from typing import Callable, List
 
 from flask import current_app, stream_with_context
+from zope import component
 
 from .spnego import requires_authentication
+from ceo_common.errors import InvalidMembershipError
+from ceo_common.interfaces import IUser, ILDAPService
 from ceo_common.logger_factory import logger_factory
 from ceod.transactions import AbstractTransaction
 
 logger = logger_factory(__name__)
 
 
+def get_valid_member_or_throw(username: str) -> IUser:
+    ldap_srv = component.getUtility(ILDAPService)
+    user = ldap_srv.get_user(username)
+    if not user.membership_is_valid():
+        raise InvalidMembershipError()
+    return user
+
+
 def requires_authentication_no_realm(f: Callable) -> Callable:
     """
     Like requires_authentication, but strips the realm out of the principal string.

ceod/model/CloudService.py

@@ -0,0 +1,222 @@
+from base64 import b64encode
+import datetime
+import hashlib
+import hmac
+import ipaddress
+import json
+import os
+import re
+from typing import Dict, List
+from urllib.parse import quote
+
+import requests
+from zope import component
+from zope.interface import implementer
+
+from .VHostManager import VHostManager
+from ceo_common.errors import CloudStackAPIError, InvalidDomainError, InvalidIPError
+from ceo_common.logger_factory import logger_factory
+from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \
+    IMailService
+from ceo_common.model import Term
+import ceo_common.utils as utils
+
+logger = logger_factory(__name__)
+
+
+@implementer(ICloudService)
+class CloudService:
+    VALID_DOMAIN_RE = re.compile(r'^(?:[0-9a-z-]+\.)+[a-z]+$')
+
+    def __init__(self):
+        cfg = component.getUtility(IConfig)
+        self.api_key = cfg.get('cloudstack_api_key')
+        self.secret_key = cfg.get('cloudstack_secret_key')
+        self.base_url = cfg.get('cloudstack_base_url')
+        self.members_domain = 'Members'
+        self.vhost_mgr = VHostManager(
+            vhost_dir=cfg.get('cloud vhosts_config_dir'),
+        )
+        self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account')
+        self.vhost_domain = cfg.get('cloud vhosts_members_domain')
+        self.vhost_ip_min = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_min'))
+        self.vhost_ip_max = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_max'))
+
+        state_dir = '/run/ceod'
+        if not os.path.isdir(state_dir):
+            os.mkdir(state_dir)
+        self.pending_deletions_file = os.path.join(state_dir, 'cloudstack_pending_account_deletions.json')
+
+    def _create_url(self, params: Dict[str, str]) -> str:
+        # See https://docs.cloudstack.apache.org/en/latest/developersguide/dev.html#the-cloudstack-api
+        if 'apiKey' not in params and 'apikey' not in params:
+            params['apiKey'] = self.api_key
+        params['response'] = 'json'
+        request_str = '&'.join(
+            key + '=' + quote(val)
+            for key, val in params.items()
+        )
+        sig_str = '&'.join(
+            key.lower() + '=' + quote(val).lower()
+            for key, val in sorted(params.items())
+        )
+        sig = hmac.new(self.secret_key.encode(), sig_str.encode(), hashlib.sha1).digest()
+        encoded_sig = b64encode(sig).decode()
+        url = self.base_url + '?' + request_str + '&signature=' + quote(encoded_sig)
+        return url
+
+    def _get_domain_id(self, domain_name: str) -> str:
+        url = self._create_url({
+            'command': 'listDomains',
+            'details': 'min',
+            'name': domain_name,
+        })
+        resp = requests.get(url)
+        resp.raise_for_status()
+        d = resp.json()['listdomainsresponse']
+        assert d['count'] == 1, 'there should be one domain found'
+        return d['domain'][0]['id']
+
+    def _get_all_accounts(self, domain_id: str) -> List[Dict]:
+        url = self._create_url({
+            'command': 'listAccounts',
+            'domainid': domain_id,
+            'details': 'min',
+        })
+        resp = requests.get(url)
+        resp.raise_for_status()
+        d = resp.json()['listaccountsresponse']
+        if 'account' not in d:
+            # The API returns an empty dict if there are no accounts
+            return []
+        return d['account']
+
+    def _delete_account(self, account_id: str):
+        url = self._create_url({
+            'command': 'deleteAccount',
+            'id': account_id,
+        })
+        resp = requests.post(url)
+        resp.raise_for_status()
+
+    def create_account(self, user: IUser):
+        domain_id = self._get_domain_id(self.members_domain)
+
+        url = self._create_url({
+            'command': 'ldapCreateAccount',
+            'accounttype': '0',
+            'domainid': domain_id,
+            'username': user.uid,
+        })
+        resp = requests.post(url)
+        d = resp.json()['createaccountresponse']
+        if not resp.ok:
+            raise CloudStackAPIError(d['errortext'])
+
+    def purge_accounts(self) -> Dict:
+        accounts_deleted = []
+        accounts_to_be_deleted = []
+        result = {
+            'accounts_deleted': accounts_deleted,
+            'accounts_to_be_deleted': accounts_to_be_deleted,
+        }
+
+        current_term = Term.current()
+        beginning_of_term = current_term.to_datetime()
+        now = utils.get_current_datetime()
+        delta = now - beginning_of_term
+        if delta.days < 30:
+            # one-month grace period
+            return result
+
+        ldap_srv = component.getUtility(ILDAPService)
+        mail_srv = component.getUtility(IMailService)
+        domain_id = self._get_domain_id(self.members_domain)
+        accounts = self._get_all_accounts(domain_id)
+
+        if os.path.isfile(self.pending_deletions_file):
+            state = json.load(open(self.pending_deletions_file))
+            last_check = datetime.datetime.fromtimestamp(state['timestamp'])
+            delta = now - last_check
+            if delta.days < 7:
+                logger.debug(
+                    'Skipping account purge because less than one week has '
+                    'passed since the warning emails were sent out'
+                )
+                accounts_to_be_deleted.extend(state['accounts_to_be_deleted'])
+                return result
+            username_to_account_id = {
+                account['name']: account['id']
+                for account in accounts
+            }
+            for username in state['accounts_to_be_deleted']:
+                if username not in username_to_account_id:
+                    continue
+                user = ldap_srv.get_user(username)
+                if user.membership_is_valid():
+                    continue
+                account_id = username_to_account_id[username]
+
+                self._delete_account(account_id)
+                self.vhost_mgr.delete_all_vhosts_for_user(username)
+
+                accounts_deleted.append(username)
+                mail_srv.send_cloud_account_has_been_deleted_message(user)
+                logger.info(f'Deleted cloud account for {username}')
+            os.unlink(self.pending_deletions_file)
+            return result
+
+        state = {
+            'timestamp': int(now.timestamp()),
+            'accounts_to_be_deleted': accounts_to_be_deleted,
+        }
+        for account in accounts:
+            username = account['name']
+            account_id = account['id']
+            user = ldap_srv.get_user(username)
+            if user.membership_is_valid():
+                continue
+            accounts_to_be_deleted.append(username)
+            mail_srv.send_cloud_account_will_be_deleted_message(user)
+            logger.info(
+                f'A warning email was sent to {username} because their '
+                'cloud account will be deleted'
+            )
+        if accounts_to_be_deleted:
+            json.dump(state, open(self.pending_deletions_file, 'w'))
+        return result
+
+    def _is_valid_domain(self, username: str, domain: str) -> bool:
+        subdomain = username + '.' + self.vhost_domain
+        if not (domain == subdomain or domain.endswith('.' + subdomain)):
+            return False
+        if self.VALID_DOMAIN_RE.match(domain) is None:
+            return False
+        if len(domain) > 80:
+            return False
+        return True
+
+    def _is_valid_ip_address(self, ip_address: str) -> bool:
+        try:
+            addr = ipaddress.ip_address(ip_address)
+        except ValueError:
+            return False
+        return self.vhost_ip_min <= addr <= self.vhost_ip_max
+
+    def create_vhost(self, username: str, domain: str, ip_address: str):
+        if self.vhost_mgr.get_num_vhosts(username) >= self.max_vhosts_per_account:
+            raise Exception(f'Only {self.max_vhosts_per_account} vhosts '
+                            'allowed per account')
+        if not self._is_valid_domain(username, domain):
+            raise InvalidDomainError()
+        if not self._is_valid_ip_address(ip_address):
+            raise InvalidIPError()
+        self.vhost_mgr.create_vhost(username, domain, ip_address)
+
+    def delete_vhost(self, username: str, domain: str):
+        if not self._is_valid_domain(username, domain):
+            raise InvalidDomainError()
+        self.vhost_mgr.delete_vhost(username, domain)
+
+    def get_vhosts(self, username: str) -> List[Dict]:
+        return self.vhost_mgr.get_vhosts(username)

ceod/model/MailService.py

@@ -58,8 +58,9 @@ class MailService:
 
     def send_welcome_message_to(self, user: IUser, password: str):
         template = self.jinja_env.get_template('welcome_message.j2')
-        # TODO: store surname and givenName in LDAP
-        first_name = user.cn.split(' ', 1)[0]
+        first_name = user.given_name
+        if not first_name:
+            first_name = user.cn.split(' ', 1)[0]
         body = template.render(name=first_name, user=user.uid, password=password)
         self.send(
             f'Computer Science Club <exec@{self.base_domain}>',
@@ -94,3 +95,29 @@ class MailService:
             },
             body,
         )
+
+    def send_cloud_account_will_be_deleted_message(self, user: IUser):
+        template = self.jinja_env.get_template('cloud_account_will_be_deleted.j2')
+        body = template.render(user=user)
+        self.send(
+            f'cloudaccounts <ceo+cloudaccounts@{self.base_domain}>',
+            f'{user.cn} <{user.uid}@{self.base_domain}>',
+            {
+                'Subject': 'Your CSC Cloud account will be deleted',
+                'Cc': f'ceo+cloudaccounts@{self.base_domain}',
+            },
+            body,
+        )
+
+    def send_cloud_account_has_been_deleted_message(self, user: IUser):
+        template = self.jinja_env.get_template('cloud_account_has_been_deleted.j2')
+        body = template.render(user=user)
+        self.send(
+            f'cloudaccounts <ceo+cloudaccounts@{self.base_domain}>',
+            f'{user.cn} <{user.uid}@{self.base_domain}>',
+            {
+                'Subject': 'Your CSC Cloud account has been deleted',
+                'Cc': f'ceo+cloudaccounts@{self.base_domain}',
+            },
+            body,
+        )

ceod/model/User.py

@@ -10,6 +10,7 @@ from .utils import should_be_club_rep
 from .validators import is_valid_shell, is_valid_term
 from ceo_common.interfaces import ILDAPService, IKerberosService, IFileService, \
     IUser, IConfig, IMailmanService
+from ceo_common.model import Term
 
 
 @implementer(IUser)
@@ -202,6 +203,13 @@ class User:
         file_srv = component.getUtility(IFileService)
         file_srv.set_forwarding_addresses(self, addresses)
 
+    def membership_is_valid(self) -> bool:
+        if not self.terms:
+            return False
+        current_term = Term.current()
+        most_recent_term = max(map(Term, self.terms))
+        return most_recent_term >= current_term
+
     def set_expired(self, expired: bool):
         with self.ldap_srv.entry_ctx_for_user(self) as entry:
             if expired:

ceod/model/VHostManager.py

@@ -0,0 +1,89 @@
+import glob
+import os
+import re
+import subprocess
+from typing import List, Dict
+
+import jinja2
+from zope.interface import implementer
+
+from ceo_common.logger_factory import logger_factory
+from ceo_common.interfaces import IVHostManager
+
+REVERSE_PROXY_IP_RE = re.compile(r'^\s+reverse_proxy\s+http://(?P<ip_address>[\d.]+)$')
+VHOST_FILENAME_RE = re.compile(r'^(?P<username>[0-9a-z-]+)_(?P<domain>[0-9a-z.-]+)$')
+logger = logger_factory(__name__)
+
+
+@implementer(IVHostManager)
+class VHostManager:
+    def __init__(self, vhost_dir: str):
+        self.vhost_dir = vhost_dir
+        self.jinja_env = jinja2.Environment(
+            loader=jinja2.PackageLoader('ceod.model'),
+        )
+
+    @staticmethod
+    def _vhost_filename(username: str, domain: str) -> str:
+        """Generate a filename for the vhost record"""
+        # sanity check...
+        assert '..' not in domain and '/' not in domain
+        return username + '_' + domain
+
+    def _vhost_filepath(self, username: str, domain: str) -> str:
+        """Generate an absolute path for the vhost record"""
+        return os.path.join(self.vhost_dir, self._vhost_filename(username, domain))
+
+    def _vhost_files(self, username: str) -> List[str]:
+        """Return a list of all vhost files for this user."""
+        return glob.glob(os.path.join(self.vhost_dir, username + '_*'))
+
+    def _reload_web_server(self):
+        logger.debug('Reloading Caddy')
+        subprocess.run(['systemctl', 'reload', 'caddy'], check=True)
+
+    def create_vhost(self, username: str, domain: str, ip_address: str):
+        template = self.jinja_env.get_template('caddy_cloud_vhost_config.j2')
+        body = template.render(
+            username=username, domain=domain, ip_address=ip_address)
+        filepath = self._vhost_filepath(username, domain)
+        logger.info(f'Writing a new vhost ({domain} -> {ip_address}) to {filepath}')
+        with open(filepath, 'w') as fo:
+            fo.write(body)
+        self._reload_web_server()
+
+    def delete_vhost(self, username: str, domain: str):
+        filepath = self._vhost_filepath(username, domain)
+        logger.info(f'Deleting {filepath}')
+        os.unlink(filepath)
+        self._reload_web_server()
+
+    def get_num_vhosts(self, username: str) -> int:
+        return len(self._vhost_files(username))
+
+    def get_vhosts(self, username: str) -> List[Dict]:
+        vhosts = []
+        for filepath in self._vhost_files(username):
+            filename = os.path.basename(filepath)
+            match = VHOST_FILENAME_RE.match(filename)
+            assert match is not None, f"'{filename}' does not match expected pattern"
+            domain = match.group('domain')
+            ip_address = None
+            for line in open(filepath):
+                match = REVERSE_PROXY_IP_RE.match(line)
+                if match is None:
+                    continue
+                ip_address = match.group('ip_address')
+                break
+            assert ip_address is not None, f"Could not find IP address in {filename}"
+            vhosts.append({'domain': domain, 'ip_address': ip_address})
+        return vhosts
+
+    def delete_all_vhosts_for_user(self, username: str):
+        filepaths = self._vhost_files(username)
+        if not filepaths:
+            return
+        for filepath in filepaths:
+            logger.info(f'Deleting {filepath}')
+            os.unlink(filepath)
+        self._reload_web_server()

ceod/model/__init__.py

@@ -1,3 +1,4 @@
+from .CloudService import CloudService
 from .KerberosService import KerberosService
 from .LDAPService import LDAPService, UserNotFoundError, GroupNotFoundError
 from .User import User
@@ -8,3 +9,4 @@ from .FileService import FileService
 from .SudoRole import SudoRole
 from .MailService import MailService
 from .MailmanService import MailmanService
+from .VHostManager import VHostManager

ceod/model/templates/caddy_cloud_vhost_config.j2

@@ -0,0 +1,22 @@
+# This file is automatically managed by ceod.
+# DO NOT EDIT THIS FILE MANUALLY UNLESS YOU KNOW WHAT YOU ARE DOING.
+
+{{ domain }} {
+  reverse_proxy http://{{ ip_address }}
+  log {
+    output file /var/log/caddy/member_{{ username }}.log {
+      roll_size 5MiB
+      roll_keep 2
+    }
+    format filter {
+      wrap json
+      fields {
+        request>headers delete
+        request>tls delete
+        resp_headers delete
+        user_id delete
+        common_log delete
+      }
+    }
+  }
+}

ceod/model/templates/cloud_account_has_been_deleted.j2

@@ -0,0 +1,14 @@
+Hello {{ user.given_name }},
+
+This is an automated message from ceo, the CSC Electronic Office.
+
+Your club membership has expired, so your CSC Cloud account
+has been deleted. If you decide to renew your membership, you
+may create a new cloud account, but it will not have any of the
+resources from your old cloud account.
+
+If you have any questions or concerns, please contact the Systems
+Committee: syscom@csclub.uwaterloo.ca
+
+Best regards,
+ceo

ceod/model/templates/cloud_account_will_be_deleted.j2

@@ -0,0 +1,18 @@
+Hello {{ user.given_name }},
+
+This is an automated message from ceo, the CSC Electronic Office.
+
+Your club membership has expired, and you have an active account in
+the CSC Cloud (https://cloud.csclub.uwaterloo.ca). All of your cloud
+resources (VMs, templates, DNS records, etc.) will be permanently
+deleted if your membership is not renewed in one week's time.
+
+If you wish to keep your cloud resources, please renew your club
+membership before next week. If you do not wish to keep your cloud
+resources, then you may safely ignore this message.
+
+If you have any questions or concerns, please contact the Systems
+Committee: syscom@csclub.uwaterloo.ca
+
+Best regards,
+ceo

debian/changelog

@@ -1,3 +1,39 @@
+ceo (1.0.12-buster1) buster; urgency=medium
+
+  * Use Caddy instead of NGINX for vhosts.
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Sun, 28 Nov 2021 20:30:31 +0000
+
+ceo (1.0.12-bullseye1) bullseye; urgency=medium
+
+  * Use Caddy instead of NGINX for vhosts.
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Sun, 28 Nov 2021 20:23:04 +0000
+
+ceo (1.0.11-buster1) buster; urgency=medium
+
+  * Add cloud vhosts API.
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Sat, 27 Nov 2021 23:18:03 +0000
+
+ceo (1.0.11-bullseye1) bullseye; urgency=medium
+
+  * Add cloud vhosts API.
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Sat, 27 Nov 2021 23:08:28 +0000
+
+ceo (1.0.10-buster1) buster; urgency=medium
+
+  * Add cloud accounts API.
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Sun, 21 Nov 2021 17:15:17 +0000
+
+ceo (1.0.10-bullseye1) bullseye; urgency=medium
+
+  * Add cloud accounts API.
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Sun, 21 Nov 2021 16:55:07 +0000
+
 ceo (1.0.9-bullseye1) bullseye; urgency=medium
 
   * Go back to symlinks.

docs/openapi.yaml

@@ -32,6 +32,8 @@ tags:
   description: Operations related to the UW LDAP directory
 - name: database
   description: Operations related to databases
+- name: cloud
+  description: Operations related to the CSC Cloud
 security:
 - GSSAPIAuth: []
 paths:
@@ -702,6 +704,123 @@ paths:
           $ref: "#/components/responses/UserNotFoundErrorResponse"
         "500":
           $ref: "#/components/responses/DBConnectionOrPermissionErrorResponse"
+  /cloud/accounts/create:
+    post:
+      tags: ['cloud']
+      servers:
+      - url: https://biloba.csclub.uwaterloo.ca:9987/api
+      summary: Activate a cloud account
+      description: Activate a cloud account for the calling user
+      responses:
+        "200":
+          "$ref": "#/components/responses/SimpleSuccessResponse"
+        "403":
+          "$ref": "#/components/responses/InvalidMembershipErrorResponse"
+  /cloud/accounts/purge:
+    post:
+      tags: ['cloud']
+      servers:
+      - url: https://biloba.csclub.uwaterloo.ca:9987/api
+      summary: Purge expired accounts
+      description: |
+        Delete the cloud accounts of expired members.
+
+        There is a one-month grace period after the expiration.
+        After one month, expired members will be sent an email warning them
+        that their account will be deleted.
+        One week after that, if an expired member has still not renewed their
+        membership, their account will be deleted.
+      responses:
+        "200":
+          description: Success
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  accounts_to_be_deleted:
+                    type: array
+                    description: usernames of accounts which will be deleted
+                    items:
+                      type: string
+                  accounts_deleted:
+                    type: array
+                    description: usernames of accounts which were deleted
+                    items:
+                      type: string
+  /cloud/vhosts/{domain}:
+    put:
+      tags: ['cloud']
+      servers:
+      - url: https://biloba.csclub.uwaterloo.ca:9987/api
+      summary: Create a vhost
+      description: Add a new virtual host configuration.
+      parameters:
+        - name: domain
+          in: path
+          description: domain name of the virtual host
+          required: true
+          schema:
+            type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                ip_address:
+                  type: string
+                  description: IP address of the virtual host
+            example: {"ip_address": "172.19.134.11"}
+      responses:
+        "200":
+          "$ref": "#/components/responses/SimpleSuccessResponse"
+        "403":
+          "$ref": "#/components/responses/InvalidMembershipErrorResponse"
+    delete:
+      tags: ['cloud']
+      servers:
+      - url: https://biloba.csclub.uwaterloo.ca:9987/api
+      summary: Delete a vhost
+      description: Delete a virtual host configuration.
+      parameters:
+        - name: domain
+          in: path
+          description: domain name of the virtual host
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          "$ref": "#/components/responses/SimpleSuccessResponse"
+  /cloud/vhosts:
+    get:
+      tags: ['cloud']
+      servers:
+      - url: https://biloba.csclub.uwaterloo.ca:9987/api
+      summary: List all vhosts
+      description: List all virtual host configurations for the calling user.
+      responses:
+        "200":
+          description: Success
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  vhosts:
+                    type: array
+                    description: virtual hosts
+                    items:
+                      type: object
+                      properties:
+                        domain:
+                          type: string
+                          description: domain name of the virtual host
+                        ip_address:
+                          type: string
+                          description: IP address of the virtual host
+              example: {"vhosts": [{"domain": "ctdalek.m.csclub.cloud", "ip_address": "172.19.134.11"}]}
 components:
   securitySchemes:
     GSSAPIAuth:
@@ -874,3 +993,17 @@ components:
     DBConnectionOrPermissionErrorResponse:
       <<: *ErrorResponse
       description: Unable to connect to database or action failed due to permissions
+    InvalidMembershipErrorResponse:
+      <<: *ErrorResponse
+      description: Membership is invalid or expired
+    SimpleSuccessResponse:
+      description: Success
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              status:
+                type: string
+                description: '"OK"'
+          example: {"status": "OK"}

etc/ceo.ini

@@ -9,6 +9,8 @@ admin_host = phosphoric-acid
 database_host = caffeine
 # this is the host which can make API requests to Mailman
 mailman_host = mail
+# this is the host running a CloudStack management server
+cloud_host = biloba
 use_https = true
 port = 9987
 

etc/ceod.ini

@@ -10,6 +10,8 @@ fs_root_host = phosphoric-acid
 database_host = caffeine
 # this is the host which can make API requests to Mailman
 mailman_host = mail
+# this is the host which is running a CloudStack management server
+cloud_host = biloba
 use_https = true
 port = 9987
 
@@ -72,3 +74,15 @@ host = localhost
 username = REPLACE_ME
 password = REPLACE_ME
 host = localhost
+
+[cloudstack]
+api_key = REPLACE_ME
+secret_key = REPLACE_ME
+base_url = http://localhost:8080/client/api
+
+[cloud vhosts]
+config_dir = /etc/caddy/ceod-member-vhosts
+max_vhosts_per_account = 10
+members_domain = csclub.cloud
+ip_range_min = 172.19.134.10
+ip_range_max = 172.19.134.160

one_time_scripts/first_and_last_names.py

@@ -30,15 +30,12 @@ for csc_entry in csc_conn.entries:
     cn = csc_entry.cn.value
     sn = None
     given_name = None
-    try:
-        uw_conn.search(
-            f'uid={uid},{UWLDAP_MEMBERS_BASE}', '(objectClass=*)',
-            attributes=['sn', 'givenName'], search_scope=ldap3.BASE)
+    uw_conn.search(
+        UWLDAP_MEMBERS_BASE, f'(uid={uid})', attributes=['sn', 'givenName'])
+    if uw_conn.entries:
         uw_entry = uw_conn.entries[0]
         sn = uw_entry.sn.value
         given_name = uw_entry.givenName.value
-    except ldap3.core.exceptions.LDAPNoSuchObjectResult:
-        pass
     if given_name is None or sn is None:
         print(f'WARNING: could not retrieve first and last names for {uid}; inferring from whitespace instead')
         words = cn.split()

tests/MockCloudStackServer.py

@@ -0,0 +1,160 @@
+from uuid import uuid4
+
+from aiohttp import web
+
+from .MockHTTPServerBase import MockHTTPServerBase
+
+
+def gen_uuid():
+    return str(uuid4())
+
+
+class MockCloudStackServer(MockHTTPServerBase):
+    def __init__(self, port=8080):
+        routes = [
+            web.get('/client/api', self.generic_handler),
+            web.post('/client/api', self.generic_handler),
+            # for debugging purposes
+            web.get('/reset', self.reset_handler),
+            web.post('/reset', self.reset_handler),
+        ]
+        super().__init__(port, routes)
+
+        self.users_by_accountid = {}
+        self.users_by_username = {}
+
+    def clear(self):
+        self.users_by_accountid.clear()
+        self.users_by_username.clear()
+
+    async def reset_handler(self, request):
+        self.clear()
+        return web.Response(text='OK\n')
+
+    def _add_user(self, username: str):
+        account_id = gen_uuid()
+        user_id = gen_uuid()
+        user = {
+            "id": user_id,
+            "username": username,
+            "firstname": "Calum",
+            "lastname": "Dalek",
+            "email": username + "@csclub.internal",
+            "created": "2021-11-20T11:08:24-0500",
+            "state": "enabled",
+            "account": username,
+            "accounttype": 0,
+            "usersource": "ldap",
+            "roleid": "24422759-45de-11ec-b585-32ee6075b19b",
+            "roletype": "User",
+            "rolename": "User",
+            "domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
+            "domain": "Members",
+            "accountid": account_id,
+            "iscallerchilddomain": False,
+            "isdefault": False
+        }
+        self.users_by_accountid[account_id] = user
+        self.users_by_username[username] = user
+        return user
+
+    def _delete_user(self, account_id: str):
+        user = self.users_by_accountid[account_id]
+        username = user['username']
+        del self.users_by_accountid[account_id]
+        del self.users_by_username[username]
+
+    def _account_from_username(self, username: str):
+        user = self.users_by_username[username]
+        return {
+            "id": user['accountid'],
+            "name": username,
+            "accounttype": 0,
+            "roleid": "24422759-45de-11ec-b585-32ee6075b19b",
+            "roletype": "User",
+            "rolename": "User",
+            "domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
+            "domain": "Members",
+            "domainpath": "ROOT/Members",
+            "state": "enabled",
+            "user": [user],
+            "isdefault": False,
+            "groups": []
+        }
+
+    async def generic_handler(self, request):
+        command = request.query['command']
+        if command == 'listDomains':
+            return web.json_response({
+                "listdomainsresponse": {
+                    "count": 1,
+                    "domain": [{
+                        "id": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
+                        "name": "Members",
+                        "level": 1,
+                        "parentdomainid": "f0f8263c-45dd-11ec-b585-32ee6075b19b",
+                        "parentdomainname": "ROOT",
+                        "haschild": False,
+                        "path": "ROOT/Members",
+                        "state": "Active",
+                        "secondarystoragetotal": 0.0
+                    }]
+                }
+            })
+        elif command == 'ldapCreateAccount':
+            username = request.query['username']
+            if username in self.users_by_username:
+                return web.json_response({
+                    "createaccountresponse": {
+                        "uuidList": [],
+                        "errorcode": 530,
+                        "cserrorcode": 4250,
+                        "errortext": f"The user {username} already exists in domain 2"
+                    }
+                }, status=530)
+            self._add_user(username)
+            return web.json_response({
+                "createaccountresponse": {
+                    "account": self._account_from_username(username),
+                }
+            })
+        elif command == 'listUsers':
+            users = list(self.users_by_username.values())
+            return web.json_response({
+                'listusersresponse': {
+                    'count': len(users),
+                    'user': users,
+                }
+            })
+        elif command == 'listAccounts':
+            usernames = list(self.users_by_username.keys())
+            return web.json_response({
+                'listaccountsresponse': {
+                    'count': len(usernames),
+                    'account': [
+                        self._account_from_username(username)
+                        for username in usernames
+                    ]
+                }
+            })
+        elif command == 'deleteAccount':
+            account_id = request.query['id']
+            self._delete_user(account_id)
+            return web.json_response({
+                'deleteaccountresponse': {
+                    'jobid': gen_uuid()
+                }
+            })
+        else:
+            return web.json_response({
+                "errorresponse": {
+                    "uuidList": [],
+                    "errorcode": 401,
+                    "errortext": "unable to verify user credentials and/or request signature"
+                }
+            }, status=401)
+
+
+if __name__ == '__main__':
+    server = MockCloudStackServer()
+    server.start()

tests/MockHTTPServerBase.py

@@ -0,0 +1,29 @@
+from abc import ABC
+import asyncio
+from threading import Thread
+from typing import List
+
+from aiohttp import web
+
+
+class MockHTTPServerBase(ABC):
+    def __init__(self, port: int, routes: List):
+        self.port = port
+        self.app = web.Application()
+        self.app.add_routes(routes)
+        self.runner = web.AppRunner(self.app)
+        self.loop = asyncio.new_event_loop()
+
+    def _start_loop(self):
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_until_complete(self.runner.setup())
+        site = web.TCPSite(self.runner, '127.0.0.1', self.port)
+        self.loop.run_until_complete(site.start())
+        self.loop.run_forever()
+
+    def start(self):
+        t = Thread(target=self._start_loop)
+        t.start()
+
+    def stop(self):
+        self.loop.call_soon_threadsafe(self.loop.stop)

tests/MockMailmanServer.py

@@ -1,18 +1,15 @@
-import asyncio
-from threading import Thread
 from aiohttp import web
 
+from .MockHTTPServerBase import MockHTTPServerBase
 
-class MockMailmanServer:
+
+class MockMailmanServer(MockHTTPServerBase):
     def __init__(self, port=8001, prefix='/3.1'):
-        self.port = port
-        self.app = web.Application()
-        self.app.add_routes([
+        routes = [
             web.post(prefix + '/members', self.subscribe),
             web.delete(prefix + '/lists/{mailing_list}/member/{address}', self.unsubscribe),
-        ])
-        self.runner = web.AppRunner(self.app)
-        self.loop = asyncio.new_event_loop()
+        ]
+        super().__init__(port, routes)
 
         # add more as necessary
         self.subscriptions = {
@@ -22,20 +19,6 @@ class MockMailmanServer:
             'syscom-alerts': [],
         }
 
-    def _start_loop(self):
-        asyncio.set_event_loop(self.loop)
-        self.loop.run_until_complete(self.runner.setup())
-        site = web.TCPSite(self.runner, '127.0.0.1', self.port)
-        self.loop.run_until_complete(site.start())
-        self.loop.run_forever()
-
-    def start(self):
-        t = Thread(target=self._start_loop)
-        t.start()
-
-    def stop(self):
-        self.loop.call_soon_threadsafe(self.loop.stop)
-
     def clear(self):
         for key in self.subscriptions:
             self.subscriptions[key].clear()

tests/__init__.py

@@ -1,2 +0,0 @@
-from .MockSMTPServer import MockSMTPServer
-from .MockMailmanServer import MockMailmanServer

tests/ceo/cli/test_cloud.py

@@ -0,0 +1,54 @@
+from click.testing import CliRunner
+
+from ...utils import gssapi_token_ctx
+from ceo.cli import cli
+
+
+def test_cloud_account_activate(cli_setup, mock_cloud_server, new_user, cfg):
+    base_domain = cfg.get('base_domain')
+    mock_cloud_server.clear()
+
+    runner = CliRunner()
+    with gssapi_token_ctx(new_user.uid):
+        result = runner.invoke(cli, ['cloud', 'account', 'activate'])
+    expected = (
+        'Congratulations! Your cloud account has been activated.\n'
+        f'You may now login into https://cloud.{base_domain} with your CSC credentials.\n'
+        "Make sure to enter 'Members' for the domain (no quotes).\n"
+    )
+    assert result.exit_code == 0
+    assert result.output == expected
+
+
+def test_cloud_accounts_purge(cli_setup, mock_cloud_server):
+    mock_cloud_server.clear()
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ['cloud', 'accounts', 'purge'])
+    assert result.exit_code == 0
+
+
+def test_cloud_vhosts(cli_setup, new_user, cfg):
+    members_domain = cfg.get('cloud vhosts_members_domain')
+    uid = new_user.uid
+    domain1 = uid + '.' + members_domain
+    ip1 = '172.19.134.11'
+
+    runner = CliRunner()
+    with gssapi_token_ctx(uid):
+        result = runner.invoke(cli, ['cloud', 'vhosts', 'add', domain1, ip1])
+    expected = 'Done.\n'
+    assert result.exit_code == 0
+    assert result.output == expected
+
+    with gssapi_token_ctx(uid):
+        result = runner.invoke(cli, ['cloud', 'vhosts', 'list'])
+    expected = domain1 + ': ' + ip1 + '\n'
+    assert result.exit_code == 0
+    assert result.output == expected
+
+    with gssapi_token_ctx(uid):
+        result = runner.invoke(cli, ['cloud', 'vhosts', 'delete', domain1])
+    expected = 'Done.\n'
+    assert result.exit_code == 0
+    assert result.output == expected

tests/ceo_dev.ini

@@ -7,6 +7,7 @@ uw_domain = uwaterloo.internal
 admin_host = phosphoric-acid
 database_host = coffee
 mailman_host = mail
+cloud_host = mail
 use_https = false
 port = 9987
 

tests/ceod/api/test_cloud.py

@@ -0,0 +1,186 @@
+import datetime
+import os
+from unittest.mock import patch
+
+import ldap3
+
+from ceo_common.model import Term
+import ceo_common.utils as ceo_common_utils
+
+
+def expire_member(user, ldap_conn):
+    most_recent_term = max(map(Term, user.terms))
+    new_term = most_recent_term - 1
+    changes = {
+        'term': [(ldap3.MODIFY_REPLACE, [str(new_term)])]
+    }
+    dn = user.ldap_srv.uid_to_dn(user.uid)
+    ldap_conn.modify(dn, changes)
+
+
+def test_create_account(client, mock_cloud_server, new_user, ldap_conn):
+    uid = new_user.uid
+    mock_cloud_server.clear()
+    status, _ = client.post('/api/cloud/accounts/create', principal=uid)
+    assert status == 200
+    assert uid in mock_cloud_server.users_by_username
+
+    status, _ = client.post('/api/cloud/accounts/create', principal=uid)
+    assert status != 200
+
+    mock_cloud_server.clear()
+    expire_member(new_user, ldap_conn)
+    status, _ = client.post('/api/cloud/accounts/create', principal=uid)
+    assert status == 403
+
+
+def test_purge_accounts(
+    client, mock_cloud_server, cloud_srv, mock_mail_server, new_user,
+    ldap_conn,
+):
+    uid = new_user.uid
+    mock_cloud_server.clear()
+    mock_mail_server.messages.clear()
+    accounts_deleted = []
+    accounts_to_be_deleted = []
+    if os.path.isfile(cloud_srv.pending_deletions_file):
+        os.unlink(cloud_srv.pending_deletions_file)
+    expected = {
+        'accounts_deleted': accounts_deleted,
+        'accounts_to_be_deleted': accounts_to_be_deleted,
+    }
+    current_term = Term.current()
+    beginning_of_term = current_term.to_datetime()
+    client.post('/api/cloud/accounts/create', principal=uid)
+    expire_member(new_user, ldap_conn)
+    with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock:
+        # one-month grace period - account should not be deleted
+        now_mock.return_value = beginning_of_term + datetime.timedelta(days=1)
+        status, data = client.post('/api/cloud/accounts/purge')
+        assert status == 200
+        assert data == expected
+
+        # grace period has passed - user should be sent a warning
+        now_mock.return_value += datetime.timedelta(days=32)
+        accounts_to_be_deleted.append(new_user.uid)
+        status, data = client.post('/api/cloud/accounts/purge')
+        assert status == 200
+        assert data == expected
+        assert os.path.isfile(cloud_srv.pending_deletions_file)
+        assert len(mock_mail_server.messages) == 1
+
+        # user still has one week left to renew their membership
+        status, data = client.post('/api/cloud/accounts/purge')
+        assert status == 200
+        assert data == expected
+
+        # one week has passed - the account can now be deleted
+        now_mock.return_value += datetime.timedelta(days=8)
+        accounts_to_be_deleted.clear()
+        accounts_deleted.append(new_user.uid)
+        status, data = client.post('/api/cloud/accounts/purge')
+        assert status == 200
+        assert data == expected
+        assert new_user.uid not in mock_cloud_server.users_by_username
+        assert len(mock_mail_server.messages) == 2
+    mock_mail_server.messages.clear()
+
+
+def test_cloud_vhosts(cfg, client, new_user, ldap_conn):
+    members_domain = cfg.get('cloud vhosts_members_domain')
+    max_vhosts = cfg.get('cloud vhosts_max_vhosts_per_account')
+    uid = new_user.uid
+
+    domain1 = uid + '.' + members_domain
+    ip1 = '172.19.134.11'
+    status, _ = client.put(
+        f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1},
+        principal=uid)
+    assert status == 200
+    status, data = client.get('/api/cloud/vhosts', principal=uid)
+    assert status == 200
+    assert data == {'vhosts': [{'domain': domain1, 'ip_address': ip1}]}
+
+    # invalid domain name
+    domain2 = uid + 'cloud.' + cfg.get('base_domain')
+    ip2 = ip1
+    status, _ = client.put(
+        f'/api/cloud/vhosts/{domain2}', json={'ip_address': ip2},
+        principal=uid)
+    assert status == 400
+
+    # invalid IP address
+    domain3 = domain1
+    ip3 = '129.97.134.10'
+    status, _ = client.put(
+        f'/api/cloud/vhosts/{domain3}', json={'ip_address': ip3},
+        principal=uid)
+    assert status == 400
+
+    # new vhost with same domain should replace old one
+    domain4 = domain1
+    ip4 = '172.19.134.14'
+    status, _ = client.put(
+        f'/api/cloud/vhosts/{domain4}', json={'ip_address': ip4},
+        principal=uid)
+    assert status == 200
+    status, data = client.get('/api/cloud/vhosts', principal=uid)
+    assert status == 200
+    assert data == {'vhosts': [{'domain': domain4, 'ip_address': ip4}]}
+
+    # maximum number of vhosts
+    for i in range(max_vhosts):
+        domain = 'app' + str(i + 1) + '.' + uid + '.' + members_domain
+        status, _ = client.put(
+            f'/api/cloud/vhosts/{domain}', json={'ip_address': ip1},
+            principal=uid)
+        if i < max_vhosts - 1:
+            assert status == 200
+        else:
+            assert status != 200
+
+    # delete a vhost
+    status, _ = client.delete(f'/api/cloud/vhosts/{domain1}', principal=uid)
+    assert status == 200
+
+    # expired members may not create vhosts
+    expire_member(new_user, ldap_conn)
+    status, _ = client.put(
+        f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1},
+        principal=uid)
+    assert status == 403
+
+
+def test_cloud_vhosts_purged_account(
+    cfg, client, mock_cloud_server, mock_mail_server, cloud_srv, new_user,
+    ldap_conn,
+):
+    uid = new_user.uid
+    members_domain = cfg.get('cloud vhosts_members_domain')
+    mock_cloud_server.clear()
+    current_term = Term.current()
+    beginning_of_term = current_term.to_datetime()
+    domain1 = uid + '.' + members_domain
+    ip1 = '172.19.134.11'
+
+    client.post('/api/cloud/accounts/create', principal=uid)
+    client.put(
+        f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1},
+        principal=uid)
+
+    expire_member(new_user, ldap_conn)
+    with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock:
+        # grace period has passed - user should be sent a warning
+        now_mock.return_value = beginning_of_term + datetime.timedelta(days=31)
+        client.post('/api/cloud/accounts/purge')
+
+        # one week has passed - the account can now be deleted
+        now_mock.return_value += datetime.timedelta(days=8)
+        client.post('/api/cloud/accounts/purge')
+
+    # vhosts should have been deleted
+    status, data = client.get('/api/cloud/vhosts', principal=uid)
+    assert status == 200
+    assert data == {'vhosts': []}
+
+    mock_mail_server.messages.clear()

tests/ceod/model/test_vhosts.py

@@ -0,0 +1,28 @@
+import os
+
+
+def test_vhost_mgr(cloud_srv):
+    vhost_mgr = cloud_srv.vhost_mgr
+    username = 'test1'
+    domain = username + '.csclub.cloud'
+    filename = f'{username}_{domain}'
+    ip_address = '172.19.134.11'
+    vhost_mgr.create_vhost(username, domain, ip_address)
+    path = os.path.join(vhost_mgr.vhost_dir, filename)
+    assert os.path.isfile(path)
+
+    assert vhost_mgr.get_num_vhosts(username) == 1
+
+    assert vhost_mgr.get_vhosts(username) == [{
+        'domain': domain, 'ip_address': ip_address,
+    }]
+
+    domain2 = 'app.' + domain
+    vhost_mgr.create_vhost(username, domain2, ip_address)
+    assert vhost_mgr.get_num_vhosts(username) == 2
+
+    vhost_mgr.delete_vhost(username, domain)
+    assert vhost_mgr.get_num_vhosts(username) == 1
+
+    vhost_mgr.delete_all_vhosts_for_user(username)
+    assert vhost_mgr.get_num_vhosts(username) == 0

tests/ceod_dev.ini

@@ -8,6 +8,7 @@ admin_host = phosphoric-acid
 fs_root_host = phosphoric-acid
 mailman_host = mail
 database_host = coffee
+cloud_host = mail
 use_https = false
 port = 9987
 
@@ -67,3 +68,15 @@ host = localhost
 username = postgres
 password = postgres
 host = localhost
+
+[cloudstack]
+api_key = REPLACE_ME
+secret_key = REPLACE_ME
+base_url = http://localhost:8080/client/api
+
+[cloud vhosts]
+config_dir = /run/ceod/member-vhosts
+max_vhosts_per_account = 10
+members_domain = csclub.cloud
+ip_range_min = 172.19.134.10
+ip_range_max = 172.19.134.160

tests/ceod_test_local.ini

@@ -8,6 +8,7 @@ admin_host = phosphoric-acid
 fs_root_host = phosphoric-acid
 mailman_host = phosphoric-acid
 database_host = phosphoric-acid
+cloud_host = phosphoric-acid
 use_https = false
 port = 9988
 
@@ -66,3 +67,15 @@ host = coffee
 username = postgres
 password = postgres
 host = coffee
+
+[cloudstack]
+api_key = REPLACE_ME
+secret_key = REPLACE_ME
+base_url = http://localhost:8080/client/api
+
+[cloud vhosts]
+config_dir = /run/ceod/member-vhosts
+max_vhosts_per_account = 10
+members_domain = csclub.cloud
+ip_range_min = 172.19.134.10
+ip_range_max = 172.19.134.160

tests/conftest.py

@@ -22,15 +22,17 @@ from zope import component
 from .utils import gssapi_token_ctx, ccache_cleanup  # noqa: F401
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
     IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
-    IDatabaseService
-from ceo_common.model import Config, HTTPClient
+    IDatabaseService, ICloudService
+from ceo_common.model import Config, HTTPClient, Term
 from ceod.api import create_app
 from ceod.db import MySQLService, PostgreSQLService
 from ceod.model import KerberosService, LDAPService, FileService, User, \
-    MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService
+    MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
+    CloudService
 import ceod.utils as utils
 from .MockSMTPServer import MockSMTPServer
 from .MockMailmanServer import MockMailmanServer
+from .MockCloudStackServer import MockCloudStackServer
 from .conftest_ceod_api import client  # noqa: F401
 from .conftest_ceo import cli_setup  # noqa: F401
 
@@ -57,7 +59,7 @@ def cfg(_drone_hostname_mock):
 
 def delete_test_princs(krb_srv):
     proc = subprocess.run([
-        'kadmin', '-k', '-p', krb_srv.admin_principal, 'listprincs', 'test_*',
+        'kadmin', '-k', '-p', krb_srv.admin_principal, 'listprincs', 'test*',
     ], text=True, capture_output=True, check=True)
     princs = [line.strip() for line in proc.stdout.splitlines()]
     for princ in princs:
@@ -243,6 +245,14 @@ def mail_srv(cfg, mock_mail_server):
     return _mail_srv
 
 
+@pytest.fixture(scope='session')
+def mock_cloud_server():
+    mock_server = MockCloudStackServer()
+    mock_server.start()
+    yield mock_server
+    mock_server.stop()
+
+
 @pytest.fixture(scope='session')
 def mysql_srv(cfg):
     mysql_srv = MySQLService()
@@ -257,6 +267,23 @@ def postgresql_srv(cfg):
     return psql_srv
 
 
+@pytest.fixture(scope='session')
+def vhost_dir_setup(cfg):
+    vhost_dir = cfg.get('cloud vhosts_config_dir')
+    if os.path.isdir(vhost_dir):
+        shutil.rmtree(vhost_dir)
+    os.makedirs(vhost_dir)
+    yield
+    shutil.rmtree(vhost_dir)
+
+
+@pytest.fixture(scope='session')
+def cloud_srv(cfg, vhost_dir_setup):
+    _cloud_srv = CloudService()
+    component.getGlobalSiteManager().registerUtility(_cloud_srv, ICloudService)
+    return _cloud_srv
+
+
 @pytest.fixture(autouse=True, scope='session')
 def app(
     cfg,
@@ -268,6 +295,7 @@ def app(
     mail_srv,
     mysql_srv,
     postgresql_srv,
+    cloud_srv,
 ):
     app = create_app({'TESTING': True})
     return app
@@ -328,6 +356,34 @@ def krb_user(simple_user):
     simple_user.remove_from_kerberos()
 
 
+_new_user_id_counter = 10001
+@pytest.fixture  # noqa: E302
+def new_user(client, g_admin_ctx, ldap_srv_session):  # noqa: F811
+    global _new_user_id_counter
+    uid = 'test' + str(_new_user_id_counter)
+    _new_user_id_counter += 1
+    status, data = client.post('/api/members', json={
+        'uid': uid,
+        'cn': 'John Doe',
+        'given_name': 'John',
+        'sn': 'Doe',
+        'program': 'Math',
+        'terms': [str(Term.current())],
+    })
+    assert status == 200
+    assert data[-1]['status'] == 'completed'
+    with g_admin_ctx():
+        user = ldap_srv_session.get_user(uid)
+    subprocess.run([
+        'kadmin', '-k', '-p', 'ceod/admin', 'cpw',
+        '-pw', 'krb5', uid,
+    ], check=True)
+    yield user
+    status, data = client.delete(f'/api/members/{uid}')
+    assert status == 200
+    assert data[-1]['status'] == 'completed'
+
+
 @pytest.fixture
 def simple_group():
     return Group(

tests/conftest_ceo.py

@@ -1,5 +1,3 @@
-import os
-
 import pytest
 
 from .utils import gssapi_token_ctx
@@ -7,9 +5,6 @@ from .utils import gssapi_token_ctx
 
 @pytest.fixture(scope='module')
 def cli_setup(app_process):
-    # This tells the CLI entrypoint not to register additional zope services.
-    os.environ['PYTEST'] = '1'
-
     # Running the client and the server in the same process would be very
     # messy because they would be sharing the same environment variables,
     # Kerberos cache, and registered utilities (via zope). So we're just

tests/conftest_ceod_api.py

@@ -73,3 +73,6 @@ class CeodTestClient:
 
     def delete(self, path, principal=None, need_auth=True, delegate=True, **kwargs):
         return self.request('DELETE', path, principal, need_auth, delegate, **kwargs)
+
+    def put(self, path, principal=None, need_auth=True, delegate=True, **kwargs):
+        return self.request('PUT', path, principal, need_auth, delegate, **kwargs)