add cloud account API endpoints

.drone/data.ldif

@@ -94,7 +94,7 @@ objectClass: posixAccount
 objectClass: shadowAccount
 objectClass: member
 program: MAT/Mathematics Computer Science
-term: s2021
+term: f2021
 
 dn: cn=ctdalek,ou=Group,dc=csclub,dc=internal
 objectClass: top
@@ -119,7 +119,7 @@ objectClass: posixAccount
 objectClass: shadowAccount
 objectClass: member
 program: MAT/Mathematics Computer Science
-term: s2021
+term: f2021
 
 dn: cn=regular1,ou=Group,dc=csclub,dc=internal
 objectClass: top
@@ -144,7 +144,7 @@ objectClass: posixAccount
 objectClass: shadowAccount
 objectClass: member
 program: MAT/Mathematics Computer Science
-term: s2021
+term: f2021
 
 dn: cn=exec1,ou=Group,dc=csclub,dc=internal
 objectClass: top

.drone/mail-setup.sh

@@ -8,9 +8,10 @@ set -ex
 add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail
 add_fqdn_to_hosts $(get_ip_addr auth1) auth1
 
-[ -f venv/bin/activate ] && . venv/bin/activate
+. venv/bin/activate
-python tests/MockMailmanServer.py &
+python -m tests.MockMailmanServer &
-python tests/MockSMTPServer.py &
+python -m tests.MockSMTPServer &
+python -m tests.MockCloudStackServer &
 
 export DEBIAN_FRONTEND=noninteractive
 apt update

ceo_common/errors.py

@@ -64,3 +64,12 @@ class DatabaseConnectionError(Exception):
 class DatabasePermissionError(Exception):
     def __init__(self):
         super().__init__('unable to perform action due to lack of permissions')
+
+
+class InvalidMembershipError(Exception):
+    def __init__(self):
+        super().__init__('membership is invalid or expired')
+
+
+class CloudStackAPIError(Exception):
+    pass

ceo_common/interfaces/ICloudService.py

@@ -1,3 +1,5 @@
+from typing import Dict
+
 from zope.interface import Interface
 
 from .IUser import IUser
@@ -10,3 +12,12 @@ class ICloudService(Interface):
         """
         Activate an LDAP account in CloudStack for the given user.
         """
+
+    def purge_accounts() -> Dict:
+        """
+        Delete CloudStack accounts which correspond to expired CSC accounts.
+        A warning message will be emailed to users one week before their
+        cloud account is deleted.
+        Another message will be emailed to the users after their cloud account
+        has been deleted.
+        """

ceo_common/interfaces/IUser.py

@@ -83,3 +83,6 @@ class IUser(Interface):
         If get_forwarding_addresses is True, the forwarding addresses
         for the user will also be returned, if present.
         """
+
+    def membership_is_valid() -> bool:
+        """Returns True iff the user's has a non-expired membership."""

ceo_common/model/Term.py

@@ -65,3 +65,10 @@ class Term:
 
     def __le__(self, other):
         return self < other or self == other
+
+    def to_datetime(self) -> datetime.datetime:
+        c = self.s_term[0]
+        year = int(self.s_term[1:])
+        month = self.seasons.index(c) * 4 + 1
+        day = 1
+        return datetime.datetime(year, month, day)

ceod/api/app_factory.py

@@ -42,6 +42,10 @@ def create_app(flask_config={}):
         from ceod.api import database
         app.register_blueprint(database.bp, url_prefix='/api/db')
 
+    if hostname == cfg.get('ceod_cloud_host'):
+        from ceod.api import cloud
+        app.register_blueprint(cloud.bp, url_prefix='/api/cloud')
+
     from ceod.api import groups
     app.register_blueprint(groups.bp, url_prefix='/api/groups')
 

ceod/api/cloud.py

@@ -0,0 +1,24 @@
+from flask import Blueprint
+from zope import component
+
+from .utils import requires_authentication_no_realm, authz_restrict_to_syscom
+from ceo_common.interfaces import ICloudService, ILDAPService
+
+bp = Blueprint('cloud', __name__)
+
+
+@bp.route('/accounts/create', methods=['POST'])
+@requires_authentication_no_realm
+def create_account(auth_user: str):
+    cloud_srv = component.getUtility(ICloudService)
+    ldap_srv = component.getUtility(ILDAPService)
+    user = ldap_srv.get_user(auth_user)
+    cloud_srv.create_account(user)
+    return {'status': 'OK'}
+
+
+@bp.route('/accounts/purge', methods=['POST'])
+@authz_restrict_to_syscom
+def purge_accounts():
+    cloud_srv = component.getUtility(ICloudService)
+    return cloud_srv.purge_accounts()

ceod/api/error_handlers.py

@@ -1,10 +1,14 @@
 import traceback
 
+from flask import request
 from flask.app import Flask
 import ldap3
 from werkzeug.exceptions import HTTPException
 
-from ceo_common.errors import UserNotFoundError, GroupNotFoundError
+from ceo_common.errors import UserNotFoundError, GroupNotFoundError, \
+    UserAlreadyExistsError, GroupAlreadyExistsError, BadRequest, \
+    UserAlreadySubscribedError, InvalidMembershipError, \
+    CloudStackAPIError
 from ceo_common.logger_factory import logger_factory
 
 __all__ = ['register_error_handlers']
@@ -20,11 +24,26 @@ def generic_error_handler(err: Exception):
     """Return JSON for all errors."""
     if isinstance(err, HTTPException):
         status_code = err.code
+    elif isinstance(err, BadRequest):
+        status_code = 400
+    elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult) \
+            or isinstance(err, InvalidMembershipError):
+        status_code = 403
     elif isinstance(err, UserNotFoundError) or isinstance(err, GroupNotFoundError):
         status_code = 404
-    elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult):
+    elif any(isinstance(err, cls) for cls in [
-        status_code = 403
+        UserAlreadyExistsError, GroupAlreadyExistsError, UserAlreadySubscribedError
+    ]):
+        status_code = 409
+    elif isinstance(err, CloudStackAPIError):
+        status_code = 500
     else:
         status_code = 500
         logger.error(traceback.format_exc())
+        if request.path.startswith('/api/cloud'):
+            # I've noticed that the requests library spits out the
+            # full URL when an Exception is raised, which will cause
+            # our CloudStack API key to be leaked. So we're going to mask
+            # it here instead.
+            err = Exception('Please contact the Systems Committee')
     return {'error': type(err).__name__ + ': ' + str(err)}, status_code

ceod/model/CloudService.py

@@ -1,16 +1,24 @@
 from base64 import b64encode
+import datetime
 import hashlib
 import hmac
-from typing import Dict
+import json
+import os
+from typing import Dict, List
 from urllib.parse import quote
 
 import requests
 from zope import component
 from zope.interface import implementer
 
-from ceo_common.interfaces import ICloudService, IConfig, IUser
+from ceo_common.errors import InvalidMembershipError, CloudStackAPIError
+from ceo_common.logger_factory import logger_factory
+from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \
+    IMailService
 from ceo_common.model import Term
 
+logger = logger_factory(__name__)
+
 
 @implementer(ICloudService)
 class CloudService:
@@ -21,6 +29,11 @@ class CloudService:
         self.base_url = cfg.get('cloudstack_base_url')
         self.members_domain = cfg.get('cloudstack_members_domain')
 
+        state_dir = '/run/ceod'
+        if not os.path.isdir(state_dir):
+            os.mkdir(state_dir)
+        self.pending_deletions_file = os.path.join(state_dir, 'cloudstack_pending_account_deletions.json')
+
     def _create_url(self, params: Dict[str, str]) -> str:
         # See https://docs.cloudstack.apache.org/en/latest/developersguide/dev.html#the-cloudstack-api
         if 'apiKey' not in params and 'apikey' not in params:
@@ -51,12 +64,31 @@ class CloudService:
         assert d['count'] == 1, 'there should be one domain found'
         return d['domain'][0]['id']
 
+    def _get_all_accounts(self, domain_id: str) -> List[Dict]:
+        url = self._create_url({
+            'command': 'listAccounts',
+            'domainid': domain_id,
+            'details': 'min',
+        })
+        resp = requests.get(url)
+        resp.raise_for_status()
+        d = resp.json()['listaccountsresponse']
+        if 'account' not in d:
+            # The API returns an empty dict if there are no accounts
+            return []
+        return d['account']
+
+    def _delete_account(self, account_id: str):
+        url = self._create_url({
+            'command': 'deleteAccount',
+            'id': account_id,
+        })
+        resp = requests.post(url)
+        resp.raise_for_status()
+
     def create_account(self, user: IUser):
-        if not user.terms:
+        if not user.membership_is_valid():
-            raise Exception('Only members may create cloud accounts')
+            raise InvalidMembershipError()
-        most_recent_term = max(map(Term, user.terms))
-        if most_recent_term < Term.current():
-            raise Exception('Membership has expired for this user')
         domain_id = self._get_domain_id(self.members_domain)
 
         url = self._create_url({
@@ -68,4 +100,73 @@ class CloudService:
         resp = requests.post(url)
         d = resp.json()['createaccountresponse']
         if not resp.ok:
-            raise Exception(d['errortext'])
+            raise CloudStackAPIError(d['errortext'])
+
+    def purge_accounts(self) -> Dict:
+        accounts_deleted = []
+        accounts_to_be_deleted = []
+        result = {
+            'accounts_deleted': accounts_deleted,
+            'accounts_to_be_deleted': accounts_to_be_deleted,
+        }
+
+        current_term = Term.current()
+        beginning_of_term = current_term.to_datetime()
+        now = datetime.datetime.now()
+        delta = now - beginning_of_term
+        if delta.days < 30:
+            # one-month grace period
+            return result
+
+        ldap_srv = component.getUtility(ILDAPService)
+        mail_srv = component.getUtility(IMailService)
+        domain_id = self._get_domain_id(self.members_domain)
+        accounts = self._get_all_accounts(domain_id)
+
+        if os.path.isfile(self.pending_deletions_file):
+            state = json.load(open(self.pending_deletions_file))
+            last_check = datetime.datetime.fromtimestamp(state['timestamp'])
+            delta = now - last_check
+            if delta.days < 7:
+                logger.debug(
+                    'Skipping account purge because less than one week has '
+                    'passed since the warning emails were sent out'
+                )
+                return result
+            username_to_account_id = {
+                account['name']: account['id']
+                for account in accounts
+            }
+            for username in state['accounts_to_be_deleted']:
+                if username not in username_to_account_id:
+                    continue
+                user = ldap_srv.get_user(username)
+                if user.membership_is_valid():
+                    continue
+                account_id = username_to_account_id[username]
+                self._delete_account(account_id)
+                accounts_deleted.append(username)
+                mail_srv.send_cloud_account_has_been_deleted_message(user)
+                logger.info(f'Deleted cloud account for {username}')
+            os.unlink(self.pending_deletions_file)
+            return result
+
+        state = {
+            'timestamp': int(now.timestamp()),
+            'accounts_to_be_deleted': accounts_to_be_deleted,
+        }
+        for account in accounts:
+            username = account['name']
+            account_id = account['id']
+            user = ldap_srv.get_user(username)
+            if user.membership_is_valid():
+                continue
+            accounts_to_be_deleted.append(username)
+            mail_srv.send_cloud_account_will_be_deleted_message(user)
+            logger.info(
+                f'A warning email was sent to {username} because their '
+                'cloud account will be deleted'
+            )
+        if accounts_to_be_deleted:
+            json.dump(state, open(self.pending_deletions_file, 'w'))
+        return result

ceod/model/MailService.py

@@ -58,7 +58,8 @@ class MailService:
 
     def send_welcome_message_to(self, user: IUser, password: str):
         template = self.jinja_env.get_template('welcome_message.j2')
-        # TODO: store surname and givenName in LDAP
+        first_name = user.given_name
+        if not first_name:
             first_name = user.cn.split(' ', 1)[0]
         body = template.render(name=first_name, user=user.uid, password=password)
         self.send(
@@ -94,3 +95,29 @@ class MailService:
             },
             body,
         )
+
+    def send_cloud_account_will_be_deleted_message(self, user: IUser):
+        template = self.jinja_env.get_template('cloud_account_will_be_deleted.j2')
+        body = template.render(user=user)
+        self.send(
+            f'cloudaccounts <ceo+cloudaccounts@{self.base_domain}>',
+            f'{user.cn} <{user.uid}@{self.base_domain}>',
+            {
+                'Subject': 'Your CSC Cloud account will be deleted',
+                'Cc': f'ceo+cloudaccounts@{self.base_domain}',
+            },
+            body,
+        )
+
+    def send_cloud_account_has_been_deleted_message(self, user: IUser):
+        template = self.jinja_env.get_template('cloud_account_has_been_deleted.j2')
+        body = template.render(user=user)
+        self.send(
+            f'cloudaccounts <ceo+cloudaccounts@{self.base_domain}>',
+            f'{user.cn} <{user.uid}@{self.base_domain}>',
+            {
+                'Subject': 'Your CSC Cloud account has been deleted',
+                'Cc': f'ceo+cloudaccounts@{self.base_domain}',
+            },
+            body,
+        )

ceod/model/User.py

@@ -10,6 +10,7 @@ from .utils import should_be_club_rep
 from .validators import is_valid_shell, is_valid_term
 from ceo_common.interfaces import ILDAPService, IKerberosService, IFileService, \
     IUser, IConfig, IMailmanService
+from ceo_common.model import Term
 
 
 @implementer(IUser)
@@ -197,3 +198,10 @@ class User:
     def set_forwarding_addresses(self, addresses: List[str]):
         file_srv = component.getUtility(IFileService)
         file_srv.set_forwarding_addresses(self, addresses)
+
+    def membership_is_valid(self) -> bool:
+        if not self.terms:
+            return False
+        current_term = Term.current()
+        most_recent_term = max(map(Term, self.terms))
+        return most_recent_term >= current_term

ceod/model/templates/cloud_account_has_been_deleted.j2

@@ -0,0 +1,14 @@
+Hello {{ user.given_name }},
+
+This is an automated message from ceo, the CSC Electronic Office.
+
+Your club membership has expired, so your CSC Cloud account
+has been deleted. If you decide to renew your membership, you
+may create a new cloud account, but it will not have any of the
+resources from your old cloud account.
+
+If you have any questions or concerns, please contact the Systems
+Committee: syscom@csclub.uwaterloo.ca
+
+Best regards,
+ceo

ceod/model/templates/cloud_account_will_be_deleted.j2

@@ -0,0 +1,18 @@
+Hello {{ user.given_name }},
+
+This is an automated message from ceo, the CSC Electronic Office.
+
+Your club membership has expired, and you have an active account in
+the CSC Cloud (https://cloud.csclub.uwaterloo.ca). All of your cloud
+resources (VMs, templates, DNS records, etc.) will be permanently
+deleted if your membership is not renewed in one week's time.
+
+If you wish to keep your cloud resources, please renew your club
+membership before next week. If you do not wish to keep your cloud
+resources, then you may safely ignore this message.
+
+If you have any questions or concerns, please contact the Systems
+Committee: syscom@csclub.uwaterloo.ca
+
+Best regards,
+ceo

etc/ceod.ini

@@ -78,5 +78,5 @@ host = localhost
 [cloudstack]
 api_key = REPLACE_ME
 secret_key = REPLACE_ME
-base_url = http://localhost:8080/api/client
+base_url = http://localhost:8080/client/api
 members_domain = Members

tests/MockCloudStackServer.py

@@ -0,0 +1,160 @@
+from uuid import uuid4
+
+from aiohttp import web
+
+from .MockHTTPServerBase import MockHTTPServerBase
+
+
+def gen_uuid():
+    return str(uuid4())
+
+
+class MockCloudStackServer(MockHTTPServerBase):
+    def __init__(self, port=8080):
+        routes = [
+            web.get('/client/api', self.generic_handler),
+            web.post('/client/api', self.generic_handler),
+            # for debugging purposes
+            web.get('/reset', self.reset_handler),
+            web.post('/reset', self.reset_handler),
+        ]
+        super().__init__(port, routes)
+
+        self.users_by_accountid = {}
+        self.users_by_username = {}
+
+    def clear(self):
+        self.users_by_accountid.clear()
+        self.users_by_username.clear()
+
+    def reset_handler(self, request):
+        self.clear()
+        return web.Response(text='OK\n')
+
+    def _add_user(self, username: str):
+        account_id = gen_uuid()
+        user_id = gen_uuid()
+        user = {
+            "id": user_id,
+            "username": username,
+            "firstname": "Calum",
+            "lastname": "Dalek",
+            "email": username + "@csclub.internal",
+            "created": "2021-11-20T11:08:24-0500",
+            "state": "enabled",
+            "account": username,
+            "accounttype": 0,
+            "usersource": "ldap",
+            "roleid": "24422759-45de-11ec-b585-32ee6075b19b",
+            "roletype": "User",
+            "rolename": "User",
+            "domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
+            "domain": "Members",
+            "accountid": account_id,
+            "iscallerchilddomain": False,
+            "isdefault": False
+        }
+        self.users_by_accountid[account_id] = user
+        self.users_by_username[username] = user
+        return user
+
+    def _delete_user(self, account_id: str):
+        user = self.users_by_accountid[account_id]
+        username = user['username']
+        del self.users_by_accountid[account_id]
+        del self.users_by_username[username]
+
+    def _account_from_username(self, username: str):
+        user = self.users_by_username[username]
+        return {
+            "id": user['accountid'],
+            "name": username,
+            "accounttype": 0,
+            "roleid": "24422759-45de-11ec-b585-32ee6075b19b",
+            "roletype": "User",
+            "rolename": "User",
+            "domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
+            "domain": "Members",
+            "domainpath": "ROOT/Members",
+            "state": "enabled",
+            "user": [user],
+            "isdefault": False,
+            "groups": []
+        }
+
+    async def generic_handler(self, request):
+        command = request.query['command']
+        if command == 'listDomains':
+            return web.json_response({
+                "listdomainsresponse": {
+                    "count": 1,
+                    "domain": [{
+                        "id": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
+                        "name": "Members",
+                        "level": 1,
+                        "parentdomainid": "f0f8263c-45dd-11ec-b585-32ee6075b19b",
+                        "parentdomainname": "ROOT",
+                        "haschild": False,
+                        "path": "ROOT/Members",
+                        "state": "Active",
+                        "secondarystoragetotal": 0.0
+                    }]
+                }
+            })
+        elif command == 'ldapCreateAccount':
+            username = request.query['username']
+            if username in self.users_by_username:
+                return web.json_response({
+                    "createaccountresponse": {
+                        "uuidList": [],
+                        "errorcode": 530,
+                        "cserrorcode": 4250,
+                        "errortext": f"The user {username} already exists in domain 2"
+                    }
+                }, status=530)
+            self._add_user(username)
+            return web.json_response({
+                "createaccountresponse": {
+                    "account": self._account_from_username(username),
+                }
+            })
+        elif command == 'listUsers':
+            users = list(self.users_by_username.values())
+            return web.json_response({
+                'listusersresponse': {
+                    'count': len(users),
+                    'user': users,
+                }
+            })
+        elif command == 'listAccounts':
+            usernames = list(self.users_by_username.keys())
+            return web.json_response({
+                'listaccountsresponse': {
+                    'count': len(usernames),
+                    'account': [
+                        self._account_from_username(username)
+                        for username in usernames
+                    ]
+                }
+            })
+        elif command == 'deleteAccount':
+            account_id = request.query['id']
+            self._delete_user(account_id)
+            return web.json_response({
+                'deleteaccountresponse': {
+                    'jobid': gen_uuid()
+                }
+            })
+        else:
+            return web.json_response({
+                "errorresponse": {
+                    "uuidList": [],
+                    "errorcode": 401,
+                    "errortext": "unable to verify user credentials and/or request signature"
+                }
+            }, status=401)
+
+
+if __name__ == '__main__':
+    server = MockCloudStackServer()
+    server.start()

tests/MockHTTPServerBase.py

@@ -0,0 +1,29 @@
+from abc import ABC
+import asyncio
+from threading import Thread
+from typing import List
+
+from aiohttp import web
+
+
+class MockHTTPServerBase(ABC):
+    def __init__(self, port: int, routes: List):
+        self.port = port
+        self.app = web.Application()
+        self.app.add_routes(routes)
+        self.runner = web.AppRunner(self.app)
+        self.loop = asyncio.new_event_loop()
+
+    def _start_loop(self):
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_until_complete(self.runner.setup())
+        site = web.TCPSite(self.runner, '127.0.0.1', self.port)
+        self.loop.run_until_complete(site.start())
+        self.loop.run_forever()
+
+    def start(self):
+        t = Thread(target=self._start_loop)
+        t.start()
+
+    def stop(self):
+        self.loop.call_soon_threadsafe(self.loop.stop)

tests/MockMailmanServer.py

@@ -1,18 +1,15 @@
-import asyncio
-from threading import Thread
 from aiohttp import web
 
+from .MockHTTPServerBase import MockHTTPServerBase
 
-class MockMailmanServer:
+
+class MockMailmanServer(MockHTTPServerBase):
     def __init__(self, port=8001, prefix='/3.1'):
-        self.port = port
+        routes = [
-        self.app = web.Application()
-        self.app.add_routes([
             web.post(prefix + '/members', self.subscribe),
             web.delete(prefix + '/lists/{mailing_list}/member/{address}', self.unsubscribe),
-        ])
+        ]
-        self.runner = web.AppRunner(self.app)
+        super().__init__(port, routes)
-        self.loop = asyncio.new_event_loop()
 
         # add more as necessary
         self.subscriptions = {
@@ -22,20 +19,6 @@ class MockMailmanServer:
             'syscom-alerts': [],
         }
 
-    def _start_loop(self):
-        asyncio.set_event_loop(self.loop)
-        self.loop.run_until_complete(self.runner.setup())
-        site = web.TCPSite(self.runner, '127.0.0.1', self.port)
-        self.loop.run_until_complete(site.start())
-        self.loop.run_forever()
-
-    def start(self):
-        t = Thread(target=self._start_loop)
-        t.start()
-
-    def stop(self):
-        self.loop.call_soon_threadsafe(self.loop.stop)
-
     def clear(self):
         for key in self.subscriptions:
             self.subscriptions[key].clear()

tests/__init__.py

@@ -1,2 +0,0 @@
-from .MockSMTPServer import MockSMTPServer
-from .MockMailmanServer import MockMailmanServer

tests/ceod_dev.ini

@@ -8,7 +8,7 @@ admin_host = phosphoric-acid
 fs_root_host = phosphoric-acid
 mailman_host = mail
 database_host = coffee
-cloud_host = biloba
+cloud_host = mail
 use_https = false
 port = 9987
 
@@ -72,5 +72,5 @@ host = localhost
 [cloudstack]
 api_key = REPLACE_ME
 secret_key = REPLACE_ME
-base_url = http://localhost:8080/api/client
+base_url = http://localhost:8080/client/api
 members_domain = Members