diff --git a/.drone/data.ldif b/.drone/data.ldif index 383e051..2bab1d4 100644 --- a/.drone/data.ldif +++ b/.drone/data.ldif @@ -94,7 +94,7 @@ objectClass: posixAccount objectClass: shadowAccount objectClass: member program: MAT/Mathematics Computer Science -term: s2021 +term: f2021 dn: cn=ctdalek,ou=Group,dc=csclub,dc=internal objectClass: top @@ -119,7 +119,7 @@ objectClass: posixAccount objectClass: shadowAccount objectClass: member program: MAT/Mathematics Computer Science -term: s2021 +term: f2021 dn: cn=regular1,ou=Group,dc=csclub,dc=internal objectClass: top @@ -144,7 +144,7 @@ objectClass: posixAccount objectClass: shadowAccount objectClass: member program: MAT/Mathematics Computer Science -term: s2021 +term: f2021 dn: cn=exec1,ou=Group,dc=csclub,dc=internal objectClass: top diff --git a/.drone/mail-setup.sh b/.drone/mail-setup.sh index 7d8384a..0e83a78 100755 --- a/.drone/mail-setup.sh +++ b/.drone/mail-setup.sh @@ -8,9 +8,10 @@ set -ex add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail add_fqdn_to_hosts $(get_ip_addr auth1) auth1 -[ -f venv/bin/activate ] && . venv/bin/activate -python tests/MockMailmanServer.py & -python tests/MockSMTPServer.py & +. venv/bin/activate +python -m tests.MockMailmanServer & +python -m tests.MockSMTPServer & +python -m tests.MockCloudStackServer & export DEBIAN_FRONTEND=noninteractive apt update diff --git a/.gitignore b/.gitignore index 4b5b571..84b1aba 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ .vscode/ *.o *.so +*.swp .idea/ /docs/*.1 /docs/*.5 diff --git a/ceo/cli/cloud.py b/ceo/cli/cloud.py new file mode 100644 index 0000000..150549a --- /dev/null +++ b/ceo/cli/cloud.py @@ -0,0 +1,46 @@ +import click +from zope import component + +from ceo_common.interfaces import IConfig + +from ..utils import http_post +from .utils import handle_sync_response + + +@click.group(short_help='Perform operations on the CSC cloud') +def cloud(): + pass + + +@cloud.group(short_help='Manage your cloud account') +def account(): + pass + + +@account.command(short_help='Activate your cloud account') +def activate(): + cfg = component.getUtility(IConfig) + base_domain = cfg.get('base_domain') + + resp = http_post('/api/cloud/accounts/create') + handle_sync_response(resp) + lines = [ + 'Congratulations! Your cloud account has been activated.', + f'You may now login into https://cloud.{base_domain} with your CSC credentials.', + "Make sure to enter 'Members' for the domain (no quotes).", + ] + for line in lines: + click.echo(line) + + +@cloud.group(short_help='Manage cloud accounts') +def accounts(): + pass + + +@accounts.command(short_help='Purge expired cloud accounts') +def purge(): + resp = http_post('/api/cloud/accounts/purge') + result = handle_sync_response(resp) + click.echo('Accounts to be deleted: ' + ','.join(result['accounts_to_be_deleted'])) + click.echo('Accounts which were deleted: ' + ','.join(result['accounts_deleted'])) diff --git a/ceo/cli/entrypoint.py b/ceo/cli/entrypoint.py index c44cda5..782cc99 100644 --- a/ceo/cli/entrypoint.py +++ b/ceo/cli/entrypoint.py @@ -7,6 +7,7 @@ from .updateprograms import updateprograms from .mysql import mysql from .postgresql import postgresql from .mailman import mailman +from .cloud import cloud @click.group() @@ -21,3 +22,4 @@ cli.add_command(updateprograms) cli.add_command(mysql) cli.add_command(postgresql) cli.add_command(mailman) +cli.add_command(cloud) diff --git a/ceo/utils.py b/ceo/utils.py index b6490a2..f915a9a 100644 --- a/ceo/utils.py +++ b/ceo/utils.py @@ -18,6 +18,8 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response: host = cfg.get('ceod_database_host') elif path.startswith('/api/mailman'): host = cfg.get('ceod_mailman_host') + elif path.startswith('/api/cloud'): + host = cfg.get('ceod_cloud_host') else: host = cfg.get('ceod_admin_host') return client.request( diff --git a/ceo_common/errors.py b/ceo_common/errors.py index c640419..2550fc2 100644 --- a/ceo_common/errors.py +++ b/ceo_common/errors.py @@ -64,3 +64,12 @@ class DatabaseConnectionError(Exception): class DatabasePermissionError(Exception): def __init__(self): super().__init__('unable to perform action due to lack of permissions') + + +class InvalidMembershipError(Exception): + def __init__(self): + super().__init__('membership is invalid or expired') + + +class CloudStackAPIError(Exception): + pass diff --git a/ceo_common/interfaces/ICloudService.py b/ceo_common/interfaces/ICloudService.py new file mode 100644 index 0000000..27bba73 --- /dev/null +++ b/ceo_common/interfaces/ICloudService.py @@ -0,0 +1,23 @@ +from typing import Dict + +from zope.interface import Interface + +from .IUser import IUser + + +class ICloudService(Interface): + """Performs operations on the CSC Cloud.""" + + def create_account(user: IUser): + """ + Activate an LDAP account in CloudStack for the given user. + """ + + def purge_accounts() -> Dict: + """ + Delete CloudStack accounts which correspond to expired CSC accounts. + A warning message will be emailed to users one week before their + cloud account is deleted. + Another message will be emailed to the users after their cloud account + has been deleted. + """ diff --git a/ceo_common/interfaces/IUser.py b/ceo_common/interfaces/IUser.py index 5c0c5a2..f288a43 100644 --- a/ceo_common/interfaces/IUser.py +++ b/ceo_common/interfaces/IUser.py @@ -83,3 +83,6 @@ class IUser(Interface): If get_forwarding_addresses is True, the forwarding addresses for the user will also be returned, if present. """ + + def membership_is_valid() -> bool: + """Returns True iff the user's has a non-expired membership.""" diff --git a/ceo_common/interfaces/__init__.py b/ceo_common/interfaces/__init__.py index 226b90f..4f32ca8 100644 --- a/ceo_common/interfaces/__init__.py +++ b/ceo_common/interfaces/__init__.py @@ -1,3 +1,4 @@ +from .ICloudService import ICloudService from .IKerberosService import IKerberosService from .IConfig import IConfig from .IUser import IUser diff --git a/ceo_common/model/Term.py b/ceo_common/model/Term.py index 84dc1ad..650f045 100644 --- a/ceo_common/model/Term.py +++ b/ceo_common/model/Term.py @@ -1,5 +1,7 @@ import datetime +import ceo_common.utils as utils + class Term: """A representation of a term in the CSC LDAP, e.g. 's2021'.""" @@ -17,7 +19,7 @@ class Term: @staticmethod def current(): """Get a Term object for the current date.""" - dt = datetime.datetime.now() + dt = utils.get_current_datetime() c = 'w' if 5 <= dt.month <= 8: c = 's' @@ -27,18 +29,19 @@ class Term: return Term(s_term) def __add__(self, other): - assert type(other) is int and other >= 0 + assert type(other) is int c = self.s_term[0] season_idx = self.seasons.index(c) year = int(self.s_term[1:]) - year += other // 3 - season_idx += other % 3 - if season_idx >= 3: - year += 1 - season_idx -= 3 + season_idx += other + year += season_idx // 3 + season_idx %= 3 s_term = self.seasons[season_idx] + str(year) return Term(s_term) + def __sub__(self, other): + return self.__add__(-other) + def __eq__(self, other): return isinstance(other, Term) and self.s_term == other.s_term @@ -65,3 +68,10 @@ class Term: def __le__(self, other): return self < other or self == other + + def to_datetime(self) -> datetime.datetime: + c = self.s_term[0] + year = int(self.s_term[1:]) + month = self.seasons.index(c) * 4 + 1 + day = 1 + return datetime.datetime(year, month, day) diff --git a/ceo_common/utils.py b/ceo_common/utils.py new file mode 100644 index 0000000..bd3f3be --- /dev/null +++ b/ceo_common/utils.py @@ -0,0 +1,7 @@ +import datetime + + +def get_current_datetime() -> datetime.datetime: + # We place this in a separate function so that we can mock it out + # in our unit tests. + return datetime.datetime.now() diff --git a/ceod/api/app_factory.py b/ceod/api/app_factory.py index bd67765..1ab12b9 100644 --- a/ceod/api/app_factory.py +++ b/ceod/api/app_factory.py @@ -7,11 +7,12 @@ from zope import component from .error_handlers import register_error_handlers from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \ - IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService + IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \ + ICloudService from ceo_common.model import Config, HTTPClient, RemoteMailmanService from ceod.api.spnego import init_spnego from ceod.model import KerberosService, LDAPService, FileService, \ - MailmanService, MailService, UWLDAPService + MailmanService, MailService, UWLDAPService, CloudService from ceod.db import MySQLService, PostgreSQLService @@ -41,6 +42,10 @@ def create_app(flask_config={}): from ceod.api import database app.register_blueprint(database.bp, url_prefix='/api/db') + if hostname == cfg.get('ceod_cloud_host'): + from ceod.api import cloud + app.register_blueprint(cloud.bp, url_prefix='/api/cloud') + from ceod.api import groups app.register_blueprint(groups.bp, url_prefix='/api/groups') @@ -118,3 +123,8 @@ def register_services(app): if hostname == cfg.get('ceod_database_host'): psql_srv = PostgreSQLService() component.provideUtility(psql_srv, IDatabaseService, 'postgresql') + + # CloudService + if hostname == cfg.get('ceod_cloud_host'): + cloud_srv = CloudService() + component.provideUtility(cloud_srv, ICloudService) diff --git a/ceod/api/cloud.py b/ceod/api/cloud.py new file mode 100644 index 0000000..497d09f --- /dev/null +++ b/ceod/api/cloud.py @@ -0,0 +1,24 @@ +from flask import Blueprint +from zope import component + +from .utils import requires_authentication_no_realm, authz_restrict_to_syscom +from ceo_common.interfaces import ICloudService, ILDAPService + +bp = Blueprint('cloud', __name__) + + +@bp.route('/accounts/create', methods=['POST']) +@requires_authentication_no_realm +def create_account(auth_user: str): + cloud_srv = component.getUtility(ICloudService) + ldap_srv = component.getUtility(ILDAPService) + user = ldap_srv.get_user(auth_user) + cloud_srv.create_account(user) + return {'status': 'OK'} + + +@bp.route('/accounts/purge', methods=['POST']) +@authz_restrict_to_syscom +def purge_accounts(): + cloud_srv = component.getUtility(ICloudService) + return cloud_srv.purge_accounts() diff --git a/ceod/api/error_handlers.py b/ceod/api/error_handlers.py index 4183d80..d9e17f0 100644 --- a/ceod/api/error_handlers.py +++ b/ceod/api/error_handlers.py @@ -1,10 +1,14 @@ import traceback +from flask import request from flask.app import Flask import ldap3 from werkzeug.exceptions import HTTPException -from ceo_common.errors import UserNotFoundError, GroupNotFoundError +from ceo_common.errors import UserNotFoundError, GroupNotFoundError, \ + UserAlreadyExistsError, GroupAlreadyExistsError, BadRequest, \ + UserAlreadySubscribedError, InvalidMembershipError, \ + CloudStackAPIError from ceo_common.logger_factory import logger_factory __all__ = ['register_error_handlers'] @@ -20,11 +24,26 @@ def generic_error_handler(err: Exception): """Return JSON for all errors.""" if isinstance(err, HTTPException): status_code = err.code + elif isinstance(err, BadRequest): + status_code = 400 + elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult) \ + or isinstance(err, InvalidMembershipError): + status_code = 403 elif isinstance(err, UserNotFoundError) or isinstance(err, GroupNotFoundError): status_code = 404 - elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult): - status_code = 403 + elif any(isinstance(err, cls) for cls in [ + UserAlreadyExistsError, GroupAlreadyExistsError, UserAlreadySubscribedError + ]): + status_code = 409 + elif isinstance(err, CloudStackAPIError): + status_code = 500 else: status_code = 500 logger.error(traceback.format_exc()) + if request.path.startswith('/api/cloud'): + # I've noticed that the requests library spits out the + # full URL when an Exception is raised, which will cause + # our CloudStack API key to be leaked. So we're going to mask + # it here instead. + err = Exception('Please contact the Systems Committee') return {'error': type(err).__name__ + ': ' + str(err)}, status_code diff --git a/ceod/model/CloudService.py b/ceod/model/CloudService.py new file mode 100644 index 0000000..c86d6a8 --- /dev/null +++ b/ceod/model/CloudService.py @@ -0,0 +1,174 @@ +from base64 import b64encode +import datetime +import hashlib +import hmac +import json +import os +from typing import Dict, List +from urllib.parse import quote + +import requests +from zope import component +from zope.interface import implementer + +from ceo_common.errors import InvalidMembershipError, CloudStackAPIError +from ceo_common.logger_factory import logger_factory +from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \ + IMailService +from ceo_common.model import Term +import ceo_common.utils as utils + +logger = logger_factory(__name__) + + +@implementer(ICloudService) +class CloudService: + def __init__(self): + cfg = component.getUtility(IConfig) + self.api_key = cfg.get('cloudstack_api_key') + self.secret_key = cfg.get('cloudstack_secret_key') + self.base_url = cfg.get('cloudstack_base_url') + self.members_domain = 'Members' + + state_dir = '/run/ceod' + if not os.path.isdir(state_dir): + os.mkdir(state_dir) + self.pending_deletions_file = os.path.join(state_dir, 'cloudstack_pending_account_deletions.json') + + def _create_url(self, params: Dict[str, str]) -> str: + # See https://docs.cloudstack.apache.org/en/latest/developersguide/dev.html#the-cloudstack-api + if 'apiKey' not in params and 'apikey' not in params: + params['apiKey'] = self.api_key + params['response'] = 'json' + request_str = '&'.join( + key + '=' + quote(val) + for key, val in params.items() + ) + sig_str = '&'.join( + key.lower() + '=' + quote(val).lower() + for key, val in sorted(params.items()) + ) + sig = hmac.new(self.secret_key.encode(), sig_str.encode(), hashlib.sha1).digest() + encoded_sig = b64encode(sig).decode() + url = self.base_url + '?' + request_str + '&signature=' + quote(encoded_sig) + return url + + def _get_domain_id(self, domain_name: str) -> str: + url = self._create_url({ + 'command': 'listDomains', + 'details': 'min', + 'name': domain_name, + }) + resp = requests.get(url) + resp.raise_for_status() + d = resp.json()['listdomainsresponse'] + assert d['count'] == 1, 'there should be one domain found' + return d['domain'][0]['id'] + + def _get_all_accounts(self, domain_id: str) -> List[Dict]: + url = self._create_url({ + 'command': 'listAccounts', + 'domainid': domain_id, + 'details': 'min', + }) + resp = requests.get(url) + resp.raise_for_status() + d = resp.json()['listaccountsresponse'] + if 'account' not in d: + # The API returns an empty dict if there are no accounts + return [] + return d['account'] + + def _delete_account(self, account_id: str): + url = self._create_url({ + 'command': 'deleteAccount', + 'id': account_id, + }) + resp = requests.post(url) + resp.raise_for_status() + + def create_account(self, user: IUser): + if not user.membership_is_valid(): + raise InvalidMembershipError() + domain_id = self._get_domain_id(self.members_domain) + + url = self._create_url({ + 'command': 'ldapCreateAccount', + 'accounttype': '0', + 'domainid': domain_id, + 'username': user.uid, + }) + resp = requests.post(url) + d = resp.json()['createaccountresponse'] + if not resp.ok: + raise CloudStackAPIError(d['errortext']) + + def purge_accounts(self) -> Dict: + accounts_deleted = [] + accounts_to_be_deleted = [] + result = { + 'accounts_deleted': accounts_deleted, + 'accounts_to_be_deleted': accounts_to_be_deleted, + } + + current_term = Term.current() + beginning_of_term = current_term.to_datetime() + now = utils.get_current_datetime() + delta = now - beginning_of_term + if delta.days < 30: + # one-month grace period + return result + + ldap_srv = component.getUtility(ILDAPService) + mail_srv = component.getUtility(IMailService) + domain_id = self._get_domain_id(self.members_domain) + accounts = self._get_all_accounts(domain_id) + + if os.path.isfile(self.pending_deletions_file): + state = json.load(open(self.pending_deletions_file)) + last_check = datetime.datetime.fromtimestamp(state['timestamp']) + delta = now - last_check + if delta.days < 7: + logger.debug( + 'Skipping account purge because less than one week has ' + 'passed since the warning emails were sent out' + ) + accounts_to_be_deleted.extend(state['accounts_to_be_deleted']) + return result + username_to_account_id = { + account['name']: account['id'] + for account in accounts + } + for username in state['accounts_to_be_deleted']: + if username not in username_to_account_id: + continue + user = ldap_srv.get_user(username) + if user.membership_is_valid(): + continue + account_id = username_to_account_id[username] + self._delete_account(account_id) + accounts_deleted.append(username) + mail_srv.send_cloud_account_has_been_deleted_message(user) + logger.info(f'Deleted cloud account for {username}') + os.unlink(self.pending_deletions_file) + return result + + state = { + 'timestamp': int(now.timestamp()), + 'accounts_to_be_deleted': accounts_to_be_deleted, + } + for account in accounts: + username = account['name'] + account_id = account['id'] + user = ldap_srv.get_user(username) + if user.membership_is_valid(): + continue + accounts_to_be_deleted.append(username) + mail_srv.send_cloud_account_will_be_deleted_message(user) + logger.info( + f'A warning email was sent to {username} because their ' + 'cloud account will be deleted' + ) + if accounts_to_be_deleted: + json.dump(state, open(self.pending_deletions_file, 'w')) + return result diff --git a/ceod/model/MailService.py b/ceod/model/MailService.py index ce877a4..2630497 100644 --- a/ceod/model/MailService.py +++ b/ceod/model/MailService.py @@ -58,8 +58,9 @@ class MailService: def send_welcome_message_to(self, user: IUser, password: str): template = self.jinja_env.get_template('welcome_message.j2') - # TODO: store surname and givenName in LDAP - first_name = user.cn.split(' ', 1)[0] + first_name = user.given_name + if not first_name: + first_name = user.cn.split(' ', 1)[0] body = template.render(name=first_name, user=user.uid, password=password) self.send( f'Computer Science Club ', @@ -94,3 +95,29 @@ class MailService: }, body, ) + + def send_cloud_account_will_be_deleted_message(self, user: IUser): + template = self.jinja_env.get_template('cloud_account_will_be_deleted.j2') + body = template.render(user=user) + self.send( + f'cloudaccounts ', + f'{user.cn} <{user.uid}@{self.base_domain}>', + { + 'Subject': 'Your CSC Cloud account will be deleted', + 'Cc': f'ceo+cloudaccounts@{self.base_domain}', + }, + body, + ) + + def send_cloud_account_has_been_deleted_message(self, user: IUser): + template = self.jinja_env.get_template('cloud_account_has_been_deleted.j2') + body = template.render(user=user) + self.send( + f'cloudaccounts ', + f'{user.cn} <{user.uid}@{self.base_domain}>', + { + 'Subject': 'Your CSC Cloud account has been deleted', + 'Cc': f'ceo+cloudaccounts@{self.base_domain}', + }, + body, + ) diff --git a/ceod/model/User.py b/ceod/model/User.py index 1bbaa38..6495c39 100644 --- a/ceod/model/User.py +++ b/ceod/model/User.py @@ -10,6 +10,7 @@ from .utils import should_be_club_rep from .validators import is_valid_shell, is_valid_term from ceo_common.interfaces import ILDAPService, IKerberosService, IFileService, \ IUser, IConfig, IMailmanService +from ceo_common.model import Term @implementer(IUser) @@ -197,3 +198,10 @@ class User: def set_forwarding_addresses(self, addresses: List[str]): file_srv = component.getUtility(IFileService) file_srv.set_forwarding_addresses(self, addresses) + + def membership_is_valid(self) -> bool: + if not self.terms: + return False + current_term = Term.current() + most_recent_term = max(map(Term, self.terms)) + return most_recent_term >= current_term diff --git a/ceod/model/__init__.py b/ceod/model/__init__.py index 65572dc..b88889c 100644 --- a/ceod/model/__init__.py +++ b/ceod/model/__init__.py @@ -1,3 +1,4 @@ +from .CloudService import CloudService from .KerberosService import KerberosService from .LDAPService import LDAPService, UserNotFoundError, GroupNotFoundError from .User import User diff --git a/ceod/model/templates/cloud_account_has_been_deleted.j2 b/ceod/model/templates/cloud_account_has_been_deleted.j2 new file mode 100644 index 0000000..4d08100 --- /dev/null +++ b/ceod/model/templates/cloud_account_has_been_deleted.j2 @@ -0,0 +1,14 @@ +Hello {{ user.given_name }}, + +This is an automated message from ceo, the CSC Electronic Office. + +Your club membership has expired, so your CSC Cloud account +has been deleted. If you decide to renew your membership, you +may create a new cloud account, but it will not have any of the +resources from your old cloud account. + +If you have any questions or concerns, please contact the Systems +Committee: syscom@csclub.uwaterloo.ca + +Best regards, +ceo diff --git a/ceod/model/templates/cloud_account_will_be_deleted.j2 b/ceod/model/templates/cloud_account_will_be_deleted.j2 new file mode 100644 index 0000000..cfdcd50 --- /dev/null +++ b/ceod/model/templates/cloud_account_will_be_deleted.j2 @@ -0,0 +1,18 @@ +Hello {{ user.given_name }}, + +This is an automated message from ceo, the CSC Electronic Office. + +Your club membership has expired, and you have an active account in +the CSC Cloud (https://cloud.csclub.uwaterloo.ca). All of your cloud +resources (VMs, templates, DNS records, etc.) will be permanently +deleted if your membership is not renewed in one week's time. + +If you wish to keep your cloud resources, please renew your club +membership before next week. If you do not wish to keep your cloud +resources, then you may safely ignore this message. + +If you have any questions or concerns, please contact the Systems +Committee: syscom@csclub.uwaterloo.ca + +Best regards, +ceo diff --git a/etc/ceo.ini b/etc/ceo.ini index a097de0..b110888 100644 --- a/etc/ceo.ini +++ b/etc/ceo.ini @@ -9,6 +9,8 @@ admin_host = phosphoric-acid database_host = caffeine # this is the host which can make API requests to Mailman mailman_host = mail +# this is the host running a CloudStack management server +cloud_host = biloba use_https = true port = 9987 diff --git a/etc/ceod.ini b/etc/ceod.ini index 457e5ed..6693389 100644 --- a/etc/ceod.ini +++ b/etc/ceod.ini @@ -10,6 +10,8 @@ fs_root_host = phosphoric-acid database_host = caffeine # this is the host which can make API requests to Mailman mailman_host = mail +# this is the host which is running a CloudStack management server +cloud_host = biloba use_https = true port = 9987 @@ -72,3 +74,8 @@ host = localhost username = REPLACE_ME password = REPLACE_ME host = localhost + +[cloudstack] +api_key = REPLACE_ME +secret_key = REPLACE_ME +base_url = http://localhost:8080/client/api diff --git a/tests/MockCloudStackServer.py b/tests/MockCloudStackServer.py new file mode 100644 index 0000000..1b87718 --- /dev/null +++ b/tests/MockCloudStackServer.py @@ -0,0 +1,160 @@ +from uuid import uuid4 + +from aiohttp import web + +from .MockHTTPServerBase import MockHTTPServerBase + + +def gen_uuid(): + return str(uuid4()) + + +class MockCloudStackServer(MockHTTPServerBase): + def __init__(self, port=8080): + routes = [ + web.get('/client/api', self.generic_handler), + web.post('/client/api', self.generic_handler), + # for debugging purposes + web.get('/reset', self.reset_handler), + web.post('/reset', self.reset_handler), + ] + super().__init__(port, routes) + + self.users_by_accountid = {} + self.users_by_username = {} + + def clear(self): + self.users_by_accountid.clear() + self.users_by_username.clear() + + async def reset_handler(self, request): + self.clear() + return web.Response(text='OK\n') + + def _add_user(self, username: str): + account_id = gen_uuid() + user_id = gen_uuid() + user = { + "id": user_id, + "username": username, + "firstname": "Calum", + "lastname": "Dalek", + "email": username + "@csclub.internal", + "created": "2021-11-20T11:08:24-0500", + "state": "enabled", + "account": username, + "accounttype": 0, + "usersource": "ldap", + "roleid": "24422759-45de-11ec-b585-32ee6075b19b", + "roletype": "User", + "rolename": "User", + "domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0", + "domain": "Members", + "accountid": account_id, + "iscallerchilddomain": False, + "isdefault": False + } + self.users_by_accountid[account_id] = user + self.users_by_username[username] = user + return user + + def _delete_user(self, account_id: str): + user = self.users_by_accountid[account_id] + username = user['username'] + del self.users_by_accountid[account_id] + del self.users_by_username[username] + + def _account_from_username(self, username: str): + user = self.users_by_username[username] + return { + "id": user['accountid'], + "name": username, + "accounttype": 0, + "roleid": "24422759-45de-11ec-b585-32ee6075b19b", + "roletype": "User", + "rolename": "User", + "domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0", + "domain": "Members", + "domainpath": "ROOT/Members", + "state": "enabled", + "user": [user], + "isdefault": False, + "groups": [] + } + + async def generic_handler(self, request): + command = request.query['command'] + if command == 'listDomains': + return web.json_response({ + "listdomainsresponse": { + "count": 1, + "domain": [{ + "id": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0", + "name": "Members", + "level": 1, + "parentdomainid": "f0f8263c-45dd-11ec-b585-32ee6075b19b", + "parentdomainname": "ROOT", + "haschild": False, + "path": "ROOT/Members", + "state": "Active", + "secondarystoragetotal": 0.0 + }] + } + }) + elif command == 'ldapCreateAccount': + username = request.query['username'] + if username in self.users_by_username: + return web.json_response({ + "createaccountresponse": { + "uuidList": [], + "errorcode": 530, + "cserrorcode": 4250, + "errortext": f"The user {username} already exists in domain 2" + } + }, status=530) + self._add_user(username) + return web.json_response({ + "createaccountresponse": { + "account": self._account_from_username(username), + } + }) + elif command == 'listUsers': + users = list(self.users_by_username.values()) + return web.json_response({ + 'listusersresponse': { + 'count': len(users), + 'user': users, + } + }) + elif command == 'listAccounts': + usernames = list(self.users_by_username.keys()) + return web.json_response({ + 'listaccountsresponse': { + 'count': len(usernames), + 'account': [ + self._account_from_username(username) + for username in usernames + ] + } + }) + elif command == 'deleteAccount': + account_id = request.query['id'] + self._delete_user(account_id) + return web.json_response({ + 'deleteaccountresponse': { + 'jobid': gen_uuid() + } + }) + else: + return web.json_response({ + "errorresponse": { + "uuidList": [], + "errorcode": 401, + "errortext": "unable to verify user credentials and/or request signature" + } + }, status=401) + + +if __name__ == '__main__': + server = MockCloudStackServer() + server.start() diff --git a/tests/MockHTTPServerBase.py b/tests/MockHTTPServerBase.py new file mode 100644 index 0000000..8747dc7 --- /dev/null +++ b/tests/MockHTTPServerBase.py @@ -0,0 +1,29 @@ +from abc import ABC +import asyncio +from threading import Thread +from typing import List + +from aiohttp import web + + +class MockHTTPServerBase(ABC): + def __init__(self, port: int, routes: List): + self.port = port + self.app = web.Application() + self.app.add_routes(routes) + self.runner = web.AppRunner(self.app) + self.loop = asyncio.new_event_loop() + + def _start_loop(self): + asyncio.set_event_loop(self.loop) + self.loop.run_until_complete(self.runner.setup()) + site = web.TCPSite(self.runner, '127.0.0.1', self.port) + self.loop.run_until_complete(site.start()) + self.loop.run_forever() + + def start(self): + t = Thread(target=self._start_loop) + t.start() + + def stop(self): + self.loop.call_soon_threadsafe(self.loop.stop) diff --git a/tests/MockMailmanServer.py b/tests/MockMailmanServer.py index d6ff78b..e03f625 100644 --- a/tests/MockMailmanServer.py +++ b/tests/MockMailmanServer.py @@ -1,18 +1,15 @@ -import asyncio -from threading import Thread from aiohttp import web +from .MockHTTPServerBase import MockHTTPServerBase -class MockMailmanServer: + +class MockMailmanServer(MockHTTPServerBase): def __init__(self, port=8001, prefix='/3.1'): - self.port = port - self.app = web.Application() - self.app.add_routes([ + routes = [ web.post(prefix + '/members', self.subscribe), web.delete(prefix + '/lists/{mailing_list}/member/{address}', self.unsubscribe), - ]) - self.runner = web.AppRunner(self.app) - self.loop = asyncio.new_event_loop() + ] + super().__init__(port, routes) # add more as necessary self.subscriptions = { @@ -22,20 +19,6 @@ class MockMailmanServer: 'syscom-alerts': [], } - def _start_loop(self): - asyncio.set_event_loop(self.loop) - self.loop.run_until_complete(self.runner.setup()) - site = web.TCPSite(self.runner, '127.0.0.1', self.port) - self.loop.run_until_complete(site.start()) - self.loop.run_forever() - - def start(self): - t = Thread(target=self._start_loop) - t.start() - - def stop(self): - self.loop.call_soon_threadsafe(self.loop.stop) - def clear(self): for key in self.subscriptions: self.subscriptions[key].clear() diff --git a/tests/__init__.py b/tests/__init__.py index 6110bcd..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +0,0 @@ -from .MockSMTPServer import MockSMTPServer -from .MockMailmanServer import MockMailmanServer diff --git a/tests/ceo/cli/test_cloud.py b/tests/ceo/cli/test_cloud.py new file mode 100644 index 0000000..d233b70 --- /dev/null +++ b/tests/ceo/cli/test_cloud.py @@ -0,0 +1,28 @@ +from click.testing import CliRunner + +from ...utils import gssapi_token_ctx +from ceo.cli import cli + + +def test_cloud_account_activate(cli_setup, mock_cloud_server, new_user, cfg): + base_domain = cfg.get('base_domain') + mock_cloud_server.clear() + + runner = CliRunner() + with gssapi_token_ctx(new_user.uid): + result = runner.invoke(cli, ['cloud', 'account', 'activate']) + expected = ( + 'Congratulations! Your cloud account has been activated.\n' + f'You may now login into https://cloud.{base_domain} with your CSC credentials.\n' + "Make sure to enter 'Members' for the domain (no quotes).\n" + ) + assert result.exit_code == 0 + assert result.output == expected + + +def test_cloud_accounts_purge(cli_setup, mock_cloud_server): + mock_cloud_server.clear() + + runner = CliRunner() + result = runner.invoke(cli, ['cloud', 'accounts', 'purge']) + assert result.exit_code == 0 diff --git a/tests/ceo_dev.ini b/tests/ceo_dev.ini index f2d11fa..3291452 100644 --- a/tests/ceo_dev.ini +++ b/tests/ceo_dev.ini @@ -7,6 +7,7 @@ uw_domain = uwaterloo.internal admin_host = phosphoric-acid database_host = coffee mailman_host = mail +cloud_host = mail use_https = false port = 9987 diff --git a/tests/ceod/api/test_cloud.py b/tests/ceod/api/test_cloud.py new file mode 100644 index 0000000..821fce4 --- /dev/null +++ b/tests/ceod/api/test_cloud.py @@ -0,0 +1,86 @@ +import datetime +import os +from unittest.mock import patch + +import ldap3 + +from ceo_common.model import Term +import ceo_common.utils as ceo_common_utils + + +def expire_member(user, ldap_conn): + most_recent_term = max(map(Term, user.terms)) + new_term = most_recent_term - 1 + changes = { + 'term': [(ldap3.MODIFY_REPLACE, [str(new_term)])] + } + dn = user.ldap_srv.uid_to_dn(user.uid) + ldap_conn.modify(dn, changes) + + +def test_create_account(client, mock_cloud_server, new_user, ldap_conn): + uid = new_user.uid + mock_cloud_server.clear() + status, _ = client.post('/api/cloud/accounts/create', principal=uid) + assert status == 200 + assert uid in mock_cloud_server.users_by_username + + status, _ = client.post('/api/cloud/accounts/create', principal=uid) + assert status != 200 + + mock_cloud_server.clear() + expire_member(new_user, ldap_conn) + status, _ = client.post('/api/cloud/accounts/create', principal=uid) + assert status == 403 + + +def test_purge_accounts( + client, mock_cloud_server, cloud_srv, mock_mail_server, new_user, + ldap_conn, +): + uid = new_user.uid + mock_cloud_server.clear() + mock_mail_server.messages.clear() + accounts_deleted = [] + accounts_to_be_deleted = [] + if os.path.isfile(cloud_srv.pending_deletions_file): + os.unlink(cloud_srv.pending_deletions_file) + expected = { + 'accounts_deleted': accounts_deleted, + 'accounts_to_be_deleted': accounts_to_be_deleted, + } + current_term = Term.current() + beginning_of_term = current_term.to_datetime() + client.post('/api/cloud/accounts/create', principal=uid) + expire_member(new_user, ldap_conn) + with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock: + # one-month grace period - account should not be deleted + now_mock.return_value = beginning_of_term + datetime.timedelta(days=1) + status, data = client.post('/api/cloud/accounts/purge') + assert status == 200 + assert data == expected + + # grace period has passed - user should be sent a warning + now_mock.return_value += datetime.timedelta(days=32) + accounts_to_be_deleted.append(new_user.uid) + status, data = client.post('/api/cloud/accounts/purge') + assert status == 200 + assert data == expected + assert os.path.isfile(cloud_srv.pending_deletions_file) + assert len(mock_mail_server.messages) == 1 + + # user still has one week left to renew their membership + status, data = client.post('/api/cloud/accounts/purge') + assert status == 200 + assert data == expected + + # one week has passed - the account can now be deleted + now_mock.return_value += datetime.timedelta(days=8) + accounts_to_be_deleted.clear() + accounts_deleted.append(new_user.uid) + status, data = client.post('/api/cloud/accounts/purge') + assert status == 200 + assert data == expected + assert new_user.uid not in mock_cloud_server.users_by_username + assert len(mock_mail_server.messages) == 2 + mock_mail_server.messages.clear() diff --git a/tests/ceod_dev.ini b/tests/ceod_dev.ini index b4b6cc5..16944c8 100644 --- a/tests/ceod_dev.ini +++ b/tests/ceod_dev.ini @@ -8,6 +8,7 @@ admin_host = phosphoric-acid fs_root_host = phosphoric-acid mailman_host = mail database_host = coffee +cloud_host = mail use_https = false port = 9987 @@ -67,3 +68,8 @@ host = localhost username = postgres password = postgres host = localhost + +[cloudstack] +api_key = REPLACE_ME +secret_key = REPLACE_ME +base_url = http://localhost:8080/client/api diff --git a/tests/ceod_test_local.ini b/tests/ceod_test_local.ini index 7927654..fdc988f 100644 --- a/tests/ceod_test_local.ini +++ b/tests/ceod_test_local.ini @@ -8,6 +8,7 @@ admin_host = phosphoric-acid fs_root_host = phosphoric-acid mailman_host = phosphoric-acid database_host = phosphoric-acid +cloud_host = phosphoric-acid use_https = false port = 9988 @@ -66,3 +67,8 @@ host = coffee username = postgres password = postgres host = coffee + +[cloudstack] +api_key = REPLACE_ME +secret_key = REPLACE_ME +base_url = http://localhost:8080/client/api diff --git a/tests/conftest.py b/tests/conftest.py index c3ae319..add1783 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,15 +22,17 @@ from zope import component from .utils import gssapi_token_ctx, ccache_cleanup # noqa: F401 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \ IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \ - IDatabaseService -from ceo_common.model import Config, HTTPClient + IDatabaseService, ICloudService +from ceo_common.model import Config, HTTPClient, Term from ceod.api import create_app from ceod.db import MySQLService, PostgreSQLService from ceod.model import KerberosService, LDAPService, FileService, User, \ - MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService + MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \ + CloudService import ceod.utils as utils from .MockSMTPServer import MockSMTPServer from .MockMailmanServer import MockMailmanServer +from .MockCloudStackServer import MockCloudStackServer from .conftest_ceod_api import client # noqa: F401 from .conftest_ceo import cli_setup # noqa: F401 @@ -243,6 +245,14 @@ def mail_srv(cfg, mock_mail_server): return _mail_srv +@pytest.fixture(scope='session') +def mock_cloud_server(): + mock_server = MockCloudStackServer() + mock_server.start() + yield mock_server + mock_server.stop() + + @pytest.fixture(scope='session') def mysql_srv(cfg): mysql_srv = MySQLService() @@ -257,6 +267,13 @@ def postgresql_srv(cfg): return psql_srv +@pytest.fixture(scope='session') +def cloud_srv(cfg): + _cloud_srv = CloudService() + component.getGlobalSiteManager().registerUtility(_cloud_srv, ICloudService) + return _cloud_srv + + @pytest.fixture(autouse=True, scope='session') def app( cfg, @@ -268,6 +285,7 @@ def app( mail_srv, mysql_srv, postgresql_srv, + cloud_srv, ): app = create_app({'TESTING': True}) return app @@ -328,6 +346,34 @@ def krb_user(simple_user): simple_user.remove_from_kerberos() +_new_user_id_counter = 10001 +@pytest.fixture # noqa: E302 +def new_user(client, g_admin_ctx, ldap_srv_session): # noqa: F811 + global _new_user_id_counter + uid = 'test_' + str(_new_user_id_counter) + _new_user_id_counter += 1 + status, data = client.post('/api/members', json={ + 'uid': uid, + 'cn': 'John Doe', + 'given_name': 'John', + 'sn': 'Doe', + 'program': 'Math', + 'terms': [str(Term.current())], + }) + assert status == 200 + assert data[-1]['status'] == 'completed' + with g_admin_ctx(): + user = ldap_srv_session.get_user(uid) + subprocess.run([ + 'kadmin', '-k', '-p', 'ceod/admin', 'cpw', + '-pw', 'krb5', uid, + ], check=True) + yield user + status, data = client.delete(f'/api/members/{uid}') + assert status == 200 + assert data[-1]['status'] == 'completed' + + @pytest.fixture def simple_group(): return Group( diff --git a/tests/conftest_ceo.py b/tests/conftest_ceo.py index 5eb712b..78ce233 100644 --- a/tests/conftest_ceo.py +++ b/tests/conftest_ceo.py @@ -1,5 +1,3 @@ -import os - import pytest from .utils import gssapi_token_ctx @@ -7,9 +5,6 @@ from .utils import gssapi_token_ctx @pytest.fixture(scope='module') def cli_setup(app_process): - # This tells the CLI entrypoint not to register additional zope services. - os.environ['PYTEST'] = '1' - # Running the client and the server in the same process would be very # messy because they would be sharing the same environment variables, # Kerberos cache, and registered utilities (via zope). So we're just