Browse Source

Add API to manage cloud accounts (#34)

This PR adds API endpoints and a CLI to create cloud accounts and to purge accounts of expired members.

Co-authored-by: Max Erenberg <>
Reviewed-on: #34
Co-authored-by: Max Erenberg <merenber@csclub.uwaterloo.ca>
Co-committed-by: Max Erenberg <merenber@csclub.uwaterloo.ca>
pull/35/head
Max Erenberg 9 months ago
parent
commit
ac98aaf38d
  1. 6
      .drone/data.ldif
  2. 7
      .drone/mail-setup.sh
  3. 1
      .gitignore
  4. 46
      ceo/cli/cloud.py
  5. 2
      ceo/cli/entrypoint.py
  6. 2
      ceo/utils.py
  7. 9
      ceo_common/errors.py
  8. 23
      ceo_common/interfaces/ICloudService.py
  9. 3
      ceo_common/interfaces/IUser.py
  10. 1
      ceo_common/interfaces/__init__.py
  11. 24
      ceo_common/model/Term.py
  12. 7
      ceo_common/utils.py
  13. 14
      ceod/api/app_factory.py
  14. 24
      ceod/api/cloud.py
  15. 25
      ceod/api/error_handlers.py
  16. 174
      ceod/model/CloudService.py
  17. 31
      ceod/model/MailService.py
  18. 8
      ceod/model/User.py
  19. 1
      ceod/model/__init__.py
  20. 14
      ceod/model/templates/cloud_account_has_been_deleted.j2
  21. 18
      ceod/model/templates/cloud_account_will_be_deleted.j2
  22. 2
      etc/ceo.ini
  23. 7
      etc/ceod.ini
  24. 160
      tests/MockCloudStackServer.py
  25. 29
      tests/MockHTTPServerBase.py
  26. 29
      tests/MockMailmanServer.py
  27. 2
      tests/__init__.py
  28. 28
      tests/ceo/cli/test_cloud.py
  29. 1
      tests/ceo_dev.ini
  30. 86
      tests/ceod/api/test_cloud.py
  31. 6
      tests/ceod_dev.ini
  32. 6
      tests/ceod_test_local.ini
  33. 52
      tests/conftest.py
  34. 5
      tests/conftest_ceo.py

6
.drone/data.ldif

@ -94,7 +94,7 @@ objectClass: posixAccount
objectClass: shadowAccount
objectClass: member
program: MAT/Mathematics Computer Science
term: s2021
term: f2021
dn: cn=ctdalek,ou=Group,dc=csclub,dc=internal
objectClass: top
@ -119,7 +119,7 @@ objectClass: posixAccount
objectClass: shadowAccount
objectClass: member
program: MAT/Mathematics Computer Science
term: s2021
term: f2021
dn: cn=regular1,ou=Group,dc=csclub,dc=internal
objectClass: top
@ -144,7 +144,7 @@ objectClass: posixAccount
objectClass: shadowAccount
objectClass: member
program: MAT/Mathematics Computer Science
term: s2021
term: f2021
dn: cn=exec1,ou=Group,dc=csclub,dc=internal
objectClass: top

7
.drone/mail-setup.sh

@ -8,9 +8,10 @@ set -ex
add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail
add_fqdn_to_hosts $(get_ip_addr auth1) auth1
[ -f venv/bin/activate ] && . venv/bin/activate
python tests/MockMailmanServer.py &
python tests/MockSMTPServer.py &
. venv/bin/activate
python -m tests.MockMailmanServer &
python -m tests.MockSMTPServer &
python -m tests.MockCloudStackServer &
export DEBIAN_FRONTEND=noninteractive
apt update

1
.gitignore vendored

@ -9,6 +9,7 @@ __pycache__/
.vscode/
*.o
*.so
*.swp
.idea/
/docs/*.1
/docs/*.5

46
ceo/cli/cloud.py

@ -0,0 +1,46 @@
import click
from zope import component
from ceo_common.interfaces import IConfig
from ..utils import http_post
from .utils import handle_sync_response
@click.group(short_help='Perform operations on the CSC cloud')
def cloud():
pass
@cloud.group(short_help='Manage your cloud account')
def account():
pass
@account.command(short_help='Activate your cloud account')
def activate():
cfg = component.getUtility(IConfig)
base_domain = cfg.get('base_domain')
resp = http_post('/api/cloud/accounts/create')
handle_sync_response(resp)
lines = [
'Congratulations! Your cloud account has been activated.',
f'You may now login into https://cloud.{base_domain} with your CSC credentials.',
"Make sure to enter 'Members' for the domain (no quotes).",
]
for line in lines:
click.echo(line)
@cloud.group(short_help='Manage cloud accounts')
def accounts():
pass
@accounts.command(short_help='Purge expired cloud accounts')
def purge():
resp = http_post('/api/cloud/accounts/purge')
result = handle_sync_response(resp)
click.echo('Accounts to be deleted: ' + ','.join(result['accounts_to_be_deleted']))
click.echo('Accounts which were deleted: ' + ','.join(result['accounts_deleted']))

2
ceo/cli/entrypoint.py

@ -7,6 +7,7 @@ from .updateprograms import updateprograms
from .mysql import mysql
from .postgresql import postgresql
from .mailman import mailman
from .cloud import cloud
@click.group()
@ -21,3 +22,4 @@ cli.add_command(updateprograms)
cli.add_command(mysql)
cli.add_command(postgresql)
cli.add_command(mailman)
cli.add_command(cloud)

2
ceo/utils.py

@ -18,6 +18,8 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response:
host = cfg.get('ceod_database_host')
elif path.startswith('/api/mailman'):
host = cfg.get('ceod_mailman_host')
elif path.startswith('/api/cloud'):
host = cfg.get('ceod_cloud_host')
else:
host = cfg.get('ceod_admin_host')
return client.request(

9
ceo_common/errors.py

@ -64,3 +64,12 @@ class DatabaseConnectionError(Exception):
class DatabasePermissionError(Exception):
def __init__(self):
super().__init__('unable to perform action due to lack of permissions')
class InvalidMembershipError(Exception):
def __init__(self):
super().__init__('membership is invalid or expired')
class CloudStackAPIError(Exception):
pass

23
ceo_common/interfaces/ICloudService.py

@ -0,0 +1,23 @@
from typing import Dict
from zope.interface import Interface
from .IUser import IUser
class ICloudService(Interface):
"""Performs operations on the CSC Cloud."""
def create_account(user: IUser):
"""
Activate an LDAP account in CloudStack for the given user.
"""
def purge_accounts() -> Dict:
"""
Delete CloudStack accounts which correspond to expired CSC accounts.
A warning message will be emailed to users one week before their
cloud account is deleted.
Another message will be emailed to the users after their cloud account
has been deleted.
"""

3
ceo_common/interfaces/IUser.py

@ -83,3 +83,6 @@ class IUser(Interface):
If get_forwarding_addresses is True, the forwarding addresses
for the user will also be returned, if present.
"""
def membership_is_valid() -> bool:
"""Returns True iff the user's has a non-expired membership."""

1
ceo_common/interfaces/__init__.py

@ -1,3 +1,4 @@
from .ICloudService import ICloudService
from .IKerberosService import IKerberosService
from .IConfig import IConfig
from .IUser import IUser

24
ceo_common/model/Term.py

@ -1,5 +1,7 @@
import datetime
import ceo_common.utils as utils
class Term:
"""A representation of a term in the CSC LDAP, e.g. 's2021'."""
@ -17,7 +19,7 @@ class Term:
@staticmethod
def current():
"""Get a Term object for the current date."""
dt = datetime.datetime.now()
dt = utils.get_current_datetime()
c = 'w'
if 5 <= dt.month <= 8:
c = 's'
@ -27,18 +29,19 @@ class Term:
return Term(s_term)
def __add__(self, other):
assert type(other) is int and other >= 0
assert type(other) is int
c = self.s_term[0]
season_idx = self.seasons.index(c)
year = int(self.s_term[1:])
year += other // 3
season_idx += other % 3
if season_idx >= 3:
year += 1
season_idx -= 3
season_idx += other
year += season_idx // 3
season_idx %= 3
s_term = self.seasons[season_idx] + str(year)
return Term(s_term)
def __sub__(self, other):
return self.__add__(-other)
def __eq__(self, other):
return isinstance(other, Term) and self.s_term == other.s_term
@ -65,3 +68,10 @@ class Term:
def __le__(self, other):
return self < other or self == other
def to_datetime(self) -> datetime.datetime:
c = self.s_term[0]
year = int(self.s_term[1:])
month = self.seasons.index(c) * 4 + 1
day = 1
return datetime.datetime(year, month, day)

7
ceo_common/utils.py

@ -0,0 +1,7 @@
import datetime
def get_current_datetime() -> datetime.datetime:
# We place this in a separate function so that we can mock it out
# in our unit tests.
return datetime.datetime.now()

14
ceod/api/app_factory.py

@ -7,11 +7,12 @@ from zope import component
from .error_handlers import register_error_handlers
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
ICloudService
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
from ceod.api.spnego import init_spnego
from ceod.model import KerberosService, LDAPService, FileService, \
MailmanService, MailService, UWLDAPService
MailmanService, MailService, UWLDAPService, CloudService
from ceod.db import MySQLService, PostgreSQLService
@ -41,6 +42,10 @@ def create_app(flask_config={}):
from ceod.api import database
app.register_blueprint(database.bp, url_prefix='/api/db')
if hostname == cfg.get('ceod_cloud_host'):
from ceod.api import cloud
app.register_blueprint(cloud.bp, url_prefix='/api/cloud')
from ceod.api import groups
app.register_blueprint(groups.bp, url_prefix='/api/groups')
@ -118,3 +123,8 @@ def register_services(app):
if hostname == cfg.get('ceod_database_host'):
psql_srv = PostgreSQLService()
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
# CloudService
if hostname == cfg.get('ceod_cloud_host'):
cloud_srv = CloudService()
component.provideUtility(cloud_srv, ICloudService)

24
ceod/api/cloud.py

@ -0,0 +1,24 @@
from flask import Blueprint
from zope import component
from .utils import requires_authentication_no_realm, authz_restrict_to_syscom
from ceo_common.interfaces import ICloudService, ILDAPService
bp = Blueprint('cloud', __name__)
@bp.route('/accounts/create', methods=['POST'])
@requires_authentication_no_realm
def create_account(auth_user: str):
cloud_srv = component.getUtility(ICloudService)
ldap_srv = component.getUtility(ILDAPService)
user = ldap_srv.get_user(auth_user)
cloud_srv.create_account(user)
return {'status': 'OK'}
@bp.route('/accounts/purge', methods=['POST'])
@authz_restrict_to_syscom
def purge_accounts():
cloud_srv = component.getUtility(ICloudService)
return cloud_srv.purge_accounts()

25
ceod/api/error_handlers.py

@ -1,10 +1,14 @@
import traceback
from flask import request
from flask.app import Flask
import ldap3
from werkzeug.exceptions import HTTPException
from ceo_common.errors import UserNotFoundError, GroupNotFoundError
from ceo_common.errors import UserNotFoundError, GroupNotFoundError, \
UserAlreadyExistsError, GroupAlreadyExistsError, BadRequest, \
UserAlreadySubscribedError, InvalidMembershipError, \
CloudStackAPIError
from ceo_common.logger_factory import logger_factory
__all__ = ['register_error_handlers']
@ -20,11 +24,26 @@ def generic_error_handler(err: Exception):
"""Return JSON for all errors."""
if isinstance(err, HTTPException):
status_code = err.code
elif isinstance(err, BadRequest):
status_code = 400
elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult) \
or isinstance(err, InvalidMembershipError):
status_code = 403
elif isinstance(err, UserNotFoundError) or isinstance(err, GroupNotFoundError):
status_code = 404
elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult):
status_code = 403
elif any(isinstance(err, cls) for cls in [
UserAlreadyExistsError, GroupAlreadyExistsError, UserAlreadySubscribedError
]):
status_code = 409
elif isinstance(err, CloudStackAPIError):
status_code = 500
else:
status_code = 500
logger.error(traceback.format_exc())
if request.path.startswith('/api/cloud'):
# I've noticed that the requests library spits out the
# full URL when an Exception is raised, which will cause
# our CloudStack API key to be leaked. So we're going to mask
# it here instead.
err = Exception('Please contact the Systems Committee')
return {'error': type(err).__name__ + ': ' + str(err)}, status_code

174
ceod/model/CloudService.py

@ -0,0 +1,174 @@
from base64 import b64encode
import datetime
import hashlib
import hmac
import json
import os
from typing import Dict, List
from urllib.parse import quote
import requests
from zope import component
from zope.interface import implementer
from ceo_common.errors import InvalidMembershipError, CloudStackAPIError
from ceo_common.logger_factory import logger_factory
from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \
IMailService
from ceo_common.model import Term
import ceo_common.utils as utils
logger = logger_factory(__name__)
@implementer(ICloudService)
class CloudService:
def __init__(self):
cfg = component.getUtility(IConfig)
self.api_key = cfg.get('cloudstack_api_key')
self.secret_key = cfg.get('cloudstack_secret_key')
self.base_url = cfg.get('cloudstack_base_url')
self.members_domain = 'Members'
state_dir = '/run/ceod'
if not os.path.isdir(state_dir):
os.mkdir(state_dir)
self.pending_deletions_file = os.path.join(state_dir, 'cloudstack_pending_account_deletions.json')
def _create_url(self, params: Dict[str, str]) -> str:
# See https://docs.cloudstack.apache.org/en/latest/developersguide/dev.html#the-cloudstack-api
if 'apiKey' not in params and 'apikey' not in params:
params['apiKey'] = self.api_key
params['response'] = 'json'
request_str = '&'.join(
key + '=' + quote(val)
for key, val in params.items()
)
sig_str = '&'.join(
key.lower() + '=' + quote(val).lower()
for key, val in sorted(params.items())
)
sig = hmac.new(self.secret_key.encode(), sig_str.encode(), hashlib.sha1).digest()
encoded_sig = b64encode(sig).decode()
url = self.base_url + '?' + request_str + '&signature=' + quote(encoded_sig)
return url
def _get_domain_id(self, domain_name: str) -> str:
url = self._create_url({
'command': 'listDomains',
'details': 'min',
'name': domain_name,
})
resp = requests.get(url)
resp.raise_for_status()
d = resp.json()['listdomainsresponse']
assert d['count'] == 1, 'there should be one domain found'
return d['domain'][0]['id']
def _get_all_accounts(self, domain_id: str) -> List[Dict]:
url = self._create_url({
'command': 'listAccounts',
'domainid': domain_id,
'details': 'min',
})
resp = requests.get(url)
resp.raise_for_status()
d = resp.json()['listaccountsresponse']
if 'account' not in d:
# The API returns an empty dict if there are no accounts
return []
return d['account']
def _delete_account(self, account_id: str):
url = self._create_url({
'command': 'deleteAccount',
'id': account_id,
})
resp = requests.post(url)
resp.raise_for_status()
def create_account(self, user: IUser):
if not user.membership_is_valid():
raise InvalidMembershipError()
domain_id = self._get_domain_id(self.members_domain)
url = self._create_url({
'command': 'ldapCreateAccount',
'accounttype': '0',
'domainid': domain_id,
'username': user.uid,
})
resp = requests.post(url)
d = resp.json()['createaccountresponse']
if not resp.ok:
raise CloudStackAPIError(d['errortext'])
def purge_accounts(self) -> Dict:
accounts_deleted = []
accounts_to_be_deleted = []
result = {
'accounts_deleted': accounts_deleted,
'accounts_to_be_deleted': accounts_to_be_deleted,
}
current_term = Term.current()
beginning_of_term = current_term.to_datetime()
now = utils.get_current_datetime()
delta = now - beginning_of_term
if delta.days < 30:
# one-month grace period
return result
ldap_srv = component.getUtility(ILDAPService)
mail_srv = component.getUtility(IMailService)
domain_id = self._get_domain_id(self.members_domain)
accounts = self._get_all_accounts(domain_id)
if os.path.isfile(self.pending_deletions_file):
state = json.load(open(self.pending_deletions_file))
last_check = datetime.datetime.fromtimestamp(state['timestamp'])
delta = now - last_check
if delta.days < 7:
logger.debug(
'Skipping account purge because less than one week has '
'passed since the warning emails were sent out'
)
accounts_to_be_deleted.extend(state['accounts_to_be_deleted'])
return result
username_to_account_id = {
account['name']: account['id']
for account in accounts
}
for username in state['accounts_to_be_deleted']:
if username not in username_to_account_id:
continue
user = ldap_srv.get_user(username)
if user.membership_is_valid():
continue
account_id = username_to_account_id[username]
self._delete_account(account_id)
accounts_deleted.append(username)
mail_srv.send_cloud_account_has_been_deleted_message(user)
logger.info(f'Deleted cloud account for {username}')
os.unlink(self.pending_deletions_file)
return result
state = {
'timestamp': int(now.timestamp()),
'accounts_to_be_deleted': accounts_to_be_deleted,
}
for account in accounts:
username = account['name']
account_id = account['id']
user = ldap_srv.get_user(username)
if user.membership_is_valid():
continue
accounts_to_be_deleted.append(username)
mail_srv.send_cloud_account_will_be_deleted_message(user)
logger.info(
f'A warning email was sent to {username} because their '
'cloud account will be deleted'
)
if accounts_to_be_deleted:
json.dump(state, open(self.pending_deletions_file, 'w'))
return result

31
ceod/model/MailService.py

@ -58,8 +58,9 @@ class MailService:
def send_welcome_message_to(self, user: IUser, password: str):
template = self.jinja_env.get_template('welcome_message.j2')
# TODO: store surname and givenName in LDAP
first_name = user.cn.split(' ', 1)[0]
first_name = user.given_name
if not first_name:
first_name = user.cn.split(' ', 1)[0]
body = template.render(name=first_name, user=user.uid, password=password)
self.send(
f'Computer Science Club <exec@{self.base_domain}>',
@ -94,3 +95,29 @@ class MailService:
},
body,
)
def send_cloud_account_will_be_deleted_message(self, user: IUser):
template = self.jinja_env.get_template('cloud_account_will_be_deleted.j2')
body = template.render(user=user)
self.send(
f'cloudaccounts <ceo+cloudaccounts@{self.base_domain}>',
f'{user.cn} <{user.uid}@{self.base_domain}>',
{
'Subject': 'Your CSC Cloud account will be deleted',
'Cc': f'ceo+cloudaccounts@{self.base_domain}',
},
body,
)
def send_cloud_account_has_been_deleted_message(self, user: IUser):
template = self.jinja_env.get_template('cloud_account_has_been_deleted.j2')
body = template.render(user=user)
self.send(
f'cloudaccounts <ceo+cloudaccounts@{self.base_domain}>',
f'{user.cn} <{user.uid}@{self.base_domain}>',
{
'Subject': 'Your CSC Cloud account has been deleted',
'Cc': f'ceo+cloudaccounts@{self.base_domain}',
},
body,
)

8
ceod/model/User.py

@ -10,6 +10,7 @@ from .utils import should_be_club_rep
from .validators import is_valid_shell, is_valid_term
from ceo_common.interfaces import ILDAPService, IKerberosService, IFileService, \
IUser, IConfig, IMailmanService
from ceo_common.model import Term
@implementer(IUser)
@ -197,3 +198,10 @@ class User:
def set_forwarding_addresses(self, addresses: List[str]):
file_srv = component.getUtility(IFileService)
file_srv.set_forwarding_addresses(self, addresses)
def membership_is_valid(self) -> bool:
if not self.terms:
return False
current_term = Term.current()
most_recent_term = max(map(Term, self.terms))
return most_recent_term >= current_term

1
ceod/model/__init__.py

@ -1,3 +1,4 @@
from .CloudService import CloudService
from .KerberosService import KerberosService
from .LDAPService import LDAPService, UserNotFoundError, GroupNotFoundError
from .User import User

14
ceod/model/templates/cloud_account_has_been_deleted.j2

@ -0,0 +1,14 @@
Hello {{ user.given_name }},
This is an automated message from ceo, the CSC Electronic Office.
Your club membership has expired, so your CSC Cloud account
has been deleted. If you decide to renew your membership, you
may create a new cloud account, but it will not have any of the
resources from your old cloud account.
If you have any questions or concerns, please contact the Systems
Committee: syscom@csclub.uwaterloo.ca
Best regards,
ceo

18
ceod/model/templates/cloud_account_will_be_deleted.j2

@ -0,0 +1,18 @@
Hello {{ user.given_name }},
This is an automated message from ceo, the CSC Electronic Office.
Your club membership has expired, and you have an active account in
the CSC Cloud (https://cloud.csclub.uwaterloo.ca). All of your cloud
resources (VMs, templates, DNS records, etc.) will be permanently
deleted if your membership is not renewed in one week's time.
If you wish to keep your cloud resources, please renew your club
membership before next week. If you do not wish to keep your cloud
resources, then you may safely ignore this message.
If you have any questions or concerns, please contact the Systems
Committee: syscom@csclub.uwaterloo.ca
Best regards,
ceo

2
etc/ceo.ini

@ -9,6 +9,8 @@ admin_host = phosphoric-acid
database_host = caffeine
# this is the host which can make API requests to Mailman
mailman_host = mail
# this is the host running a CloudStack management server
cloud_host = biloba
use_https = true
port = 9987

7
etc/ceod.ini

@ -10,6 +10,8 @@ fs_root_host = phosphoric-acid
database_host = caffeine
# this is the host which can make API requests to Mailman
mailman_host = mail
# this is the host which is running a CloudStack management server
cloud_host = biloba
use_https = true
port = 9987
@ -72,3 +74,8 @@ host = localhost
username = REPLACE_ME
password = REPLACE_ME
host = localhost
[cloudstack]
api_key = REPLACE_ME
secret_key = REPLACE_ME
base_url = http://localhost:8080/client/api

160
tests/MockCloudStackServer.py

@ -0,0 +1,160 @@
from uuid import uuid4
from aiohttp import web
from .MockHTTPServerBase import MockHTTPServerBase
def gen_uuid():
return str(uuid4())
class MockCloudStackServer(MockHTTPServerBase):
def __init__(self, port=8080):
routes = [
web.get('/client/api', self.generic_handler),
web.post('/client/api', self.generic_handler),
# for debugging purposes
web.get('/reset', self.reset_handler),
web.post('/reset', self.reset_handler),
]
super().__init__(port, routes)
self.users_by_accountid = {}
self.users_by_username = {}
def clear(self):
self.users_by_accountid.clear()
self.users_by_username.clear()
async def reset_handler(self, request):
self.clear()
return web.Response(text='OK\n')
def _add_user(self, username: str):
account_id = gen_uuid()
user_id = gen_uuid()
user = {
"id": user_id,
"username": username,
"firstname": "Calum",
"lastname": "Dalek",
"email": username + "@csclub.internal",
"created": "2021-11-20T11:08:24-0500",
"state": "enabled",
"account": username,
"accounttype": 0,
"usersource": "ldap",
"roleid": "24422759-45de-11ec-b585-32ee6075b19b",
"roletype": "User",
"rolename": "User",
"domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
"domain": "Members",
"accountid": account_id,
"iscallerchilddomain": False,
"isdefault": False
}
self.users_by_accountid[account_id] = user
self.users_by_username[username] = user
return user
def _delete_user(self, account_id: str):
user = self.users_by_accountid[account_id]
username = user['username']
del self.users_by_accountid[account_id]
del self.users_by_username[username]
def _account_from_username(self, username: str):
user = self.users_by_username[username]
return {
"id": user['accountid'],
"name": username,
"accounttype": 0,
"roleid": "24422759-45de-11ec-b585-32ee6075b19b",
"roletype": "User",
"rolename": "User",
"domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
"domain": "Members",
"domainpath": "ROOT/Members",
"state": "enabled",
"user": [user],
"isdefault": False,
"groups": []
}
async def generic_handler(self, request):
command = request.query['command']
if command == 'listDomains':
return web.json_response({
"listdomainsresponse": {
"count": 1,
"domain": [{
"id": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
"name": "Members",
"level": 1,
"parentdomainid": "f0f8263c-45dd-11ec-b585-32ee6075b19b",
"parentdomainname": "ROOT",
"haschild": False,
"path": "ROOT/Members",
"state": "Active",
"secondarystoragetotal": 0.0
}]
}
})
elif command == 'ldapCreateAccount':
username = request.query['username']
if username in self.users_by_username:
return web.json_response({
"createaccountresponse": {
"uuidList": [],
"errorcode": 530,
"cserrorcode": 4250,
"errortext": f"The user {username} already exists in domain 2"
}
}, status=530)
self._add_user(username)
return web.json_response({
"createaccountresponse": {
"account": self._account_from_username(username),
}
})
elif command == 'listUsers':
users = list(self.users_by_username.values())
return web.json_response({
'listusersresponse': {
'count': len(users),
'user': users,
}
})
elif command == 'listAccounts':
usernames = list(self.users_by_username.keys())
return web.json_response({
'listaccountsresponse': {
'count': len(usernames),
'account': [
self._account_from_username(username)
for username in usernames
]
}
})
elif command == 'deleteAccount':
account_id = request.query['id']
self._delete_user(account_id)
return web.json_response({
'deleteaccountresponse': {
'jobid': gen_uuid()
}
})
else:
return web.json_response({
"errorresponse": {
"uuidList": [],
"errorcode": 401,
"errortext": "unable to verify user credentials and/or request signature"
}
}, status=401)
if __name__ == '__main__':
server = MockCloudStackServer()
server.start()

29
tests/MockHTTPServerBase.py

@ -0,0 +1,29 @@
from abc import ABC
import asyncio
from threading import Thread
from typing import List
from aiohttp import web
class MockHTTPServerBase(ABC):
def __init__(self, port: int, routes: List):
self.port = port
self.app = web.Application()
self.app.add_routes(routes)
self.runner = web.AppRunner(self.app)
self.loop = asyncio.new_event_loop()
def _start_loop(self):
asyncio.set_event_loop(self.loop)
self.loop.run_until_complete(self.runner.setup())
site = web.TCPSite(self.runner, '127.0.0.1', self.port)
self.loop.run_until_complete(site.start())
self.loop.run_forever()
def start(self):
t = Thread(target=self._start_loop)
t.start()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)

29
tests/MockMailmanServer.py

@ -1,18 +1,15 @@
import asyncio
from threading import Thread
from aiohttp import web
from .MockHTTPServerBase import MockHTTPServerBase
class MockMailmanServer:
class MockMailmanServer(MockHTTPServerBase):
def __init__(self, port=8001, prefix='/3.1'):
self.port = port
self.app = web.Application()
self.app.add_routes([
routes = [
web.post(prefix + '/members', self.subscribe),
web.delete(prefix + '/lists/{mailing_list}/member/{address}', self.unsubscribe),
])
self.runner = web.AppRunner(self.app)
self.loop = asyncio.new_event_loop()
]
super().__init__(port, routes)
# add more as necessary
self.subscriptions = {
@ -22,20 +19,6 @@ class MockMailmanServer:
'syscom-alerts': [],
}
def _start_loop(self):
asyncio.set_event_loop(self.loop)
self.loop.run_until_complete(self.runner.setup())
site = web.TCPSite(self.runner, '127.0.0.1', self.port)
self.loop.run_until_complete(site.start())
self.loop.run_forever()
def start(self):
t = Thread(target=self._start_loop)
t.start()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)
def clear(self):
for key in self.subscriptions:
self.subscriptions[key].clear()

2
tests/__init__.py

@ -1,2 +0,0 @@
from .MockSMTPServer import MockSMTPServer
from .MockMailmanServer import MockMailmanServer

28
tests/ceo/cli/test_cloud.py

@ -0,0 +1,28 @@
from click.testing import CliRunner
from ...utils import gssapi_token_ctx
from ceo.cli import cli
def test_cloud_account_activate(cli_setup, mock_cloud_server, new_user, cfg):
base_domain = cfg.get('base_domain')
mock_cloud_server.clear()
runner = CliRunner()
with gssapi_token_ctx(new_user.uid):
result = runner.invoke(cli, ['cloud', 'account', 'activate'])
expected = (
'Congratulations! Your cloud account has been activated.\n'
f'You may now login into https://cloud.{base_domain} with your CSC credentials.\n'
"Make sure to enter 'Members' for the domain (no quotes).\n"
)
assert result.exit_code == 0
assert result.output == expected
def test_cloud_accounts_purge(cli_setup, mock_cloud_server):
mock_cloud_server.clear()
runner = CliRunner()
result = runner.invoke(cli, ['cloud', 'accounts', 'purge'])
assert result.exit_code == 0

1
tests/ceo_dev.ini

@ -7,6 +7,7 @@ uw_domain = uwaterloo.internal
admin_host = phosphoric-acid
database_host = coffee
mailman_host = mail
cloud_host = mail
use_https = false
port = 9987

86
tests/ceod/api/test_cloud.py

@ -0,0 +1,86 @@
import datetime
import os
from unittest.mock import patch
import ldap3
from ceo_common.model import Term
import ceo_common.utils as ceo_common_utils
def expire_member(user, ldap_conn):
most_recent_term = max(map(Term, user.terms))
new_term = most_recent_term - 1
changes = {
'term': [(ldap3.MODIFY_REPLACE, [str(new_term)])]
}
dn = user.ldap_srv.uid_to_dn(user.uid)
ldap_conn.modify(dn, changes)
def test_create_account(client, mock_cloud_server, new_user, ldap_conn):
uid = new_user.uid
mock_cloud_server.clear()
status, _ = client.post('/api/cloud/accounts/create', principal=uid)
assert status == 200
assert uid in mock_cloud_server.users_by_username
status, _ = client.post('/api/cloud/accounts/create', principal=uid)
assert status != 200
mock_cloud_server.clear()
expire_member(new_user, ldap_conn)
status, _ = client.post('/api/cloud/accounts/create', principal=uid)
assert status == 403
def test_purge_accounts(
client, mock_cloud_server, cloud_srv, mock_mail_server, new_user,
ldap_conn,
):
uid = new_user.uid
mock_cloud_server.clear()
mock_mail_server.messages.clear()
accounts_deleted = []
accounts_to_be_deleted = []
if os.path.isfile(cloud_srv.pending_deletions_file):
os.unlink(cloud_srv.pending_deletions_file)
expected = {
'accounts_deleted': accounts_deleted,
'accounts_to_be_deleted': accounts_to_be_deleted,
}
current_term = Term.current()
beginning_of_term = current_term.to_datetime()
client.post('/api/cloud/accounts/create', principal=uid)
expire_member(new_user, ldap_conn)
with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock:
# one-month grace period - account should not be deleted
now_mock.return_value = beginning_of_term + datetime.timedelta(days=1)
status, data = client.post('/api/cloud/accounts/purge')
assert status == 200
assert data == expected
# grace period has passed - user should be sent a warning
now_mock.return_value += datetime.timedelta(days=32)
accounts_to_be_deleted.append(new_user.uid)
status, data = client.post('/api/cloud/accounts/purge')
assert status == 200
assert data == expected
assert os.path.isfile(cloud_srv.pending_deletions_file)
assert len(mock_mail_server.messages) == 1
# user still has one week left to renew their membership
status, data = client.post('/api/cloud/accounts/purge')
assert status == 200
assert data == expected
# one week has passed - the account can now be deleted
now_mock.return_value += datetime.timedelta(days=8)
accounts_to_be_deleted.clear()
accounts_deleted.append(new_user.uid)
status, data = client.post('/api/cloud/accounts/purge')
assert status == 200
assert data == expected
assert new_user.uid not in mock_cloud_server.users_by_username
assert len(mock_mail_server.messages) == 2
mock_mail_server.messages.clear()

6
tests/ceod_dev.ini

@ -8,6 +8,7 @@ admin_host = phosphoric-acid
fs_root_host = phosphoric-acid
mailman_host = mail
database_host = coffee
cloud_host = mail
use_https = false
port = 9987
@ -67,3 +68,8 @@ host = localhost
username = postgres
password = postgres
host = localhost
[cloudstack]
api_key = REPLACE_ME
secret_key = REPLACE_ME
base_url = http://localhost:8080/client/api

6
tests/ceod_test_local.ini

@ -8,6 +8,7 @@ admin_host = phosphoric-acid
fs_root_host = phosphoric-acid
mailman_host = phosphoric-acid
database_host = phosphoric-acid
cloud_host = phosphoric-acid
use_https = false
port = 9988
@ -66,3 +67,8 @@ host = coffee
username = postgres
password = postgres
host = coffee
[cloudstack]
api_key = REPLACE_ME
secret_key = REPLACE_ME
base_url = http://localhost:8080/client/api

52
tests/conftest.py

@ -22,15 +22,17 @@ from zope import component
from .utils import gssapi_token_ctx, ccache_cleanup # noqa: F401
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
IDatabaseService
from ceo_common.model import Config, HTTPClient
IDatabaseService, ICloudService
from ceo_common.model import Config, HTTPClient, Term
from ceod.api import create_app
from ceod.db import MySQLService, PostgreSQLService
from ceod.model import KerberosService, LDAPService, FileService, User, \
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
CloudService
import ceod.utils as utils
from .MockSMTPServer import MockSMTPServer
from .MockMailmanServer import MockMailmanServer
from .MockCloudStackServer import MockCloudStackServer
from .conftest_ceod_api import client # noqa: F401
from .conftest_ceo import cli_setup # noqa: F401
@ -243,6 +245,14 @@ def mail_srv(cfg, mock_mail_server):
return _mail_srv
@pytest.fixture(scope='session')
def mock_cloud_server():
mock_server = MockCloudStackServer()
mock_server.start()
yield mock_server
mock_server.stop()
@pytest.fixture(scope='session')
def mysql_srv(cfg):
mysql_srv = MySQLService()
@ -257,6 +267,13 @@ def postgresql_srv(cfg):
return psql_srv
@pytest.fixture(scope='session')
def cloud_srv(cfg):
_cloud_srv = CloudService()
component.getGlobalSiteManager().registerUtility(_cloud_srv, ICloudService)
return _cloud_srv
@pytest.fixture(autouse=True, scope='session')
def app(
cfg,
@ -268,6 +285,7 @@ def app(
mail_srv,
mysql_srv,
postgresql_srv,
cloud_srv,
):
app = create_app({'TESTING': True})
return app
@ -328,6 +346,34 @@ def krb_user(simple_user):
simple_user.remove_from_kerberos()
_new_user_id_counter = 10001
@pytest.fixture # noqa: E302
def new_user(client, g_admin_ctx, ldap_srv_session): # noqa: F811
global _new_user_id_counter
uid = 'test_' + str(_new_user_id_counter)
_new_user_id_counter += 1
status, data = client.post('/api/members', json={
'uid': uid,
'cn': 'John Doe',
'given_name': 'John',
'sn': 'Doe',
'program': 'Math',
'terms': [str(Term.current())],
})
assert status == 200
assert data[-1]['status'] == 'completed'
with g_admin_ctx():
user = ldap_srv_session.get_user(uid)
subprocess.run([
'kadmin', '-k', '-p', 'ceod/admin', 'cpw',
'-pw', 'krb5', uid,
], check=True)
yield user
status, data = client.delete(f'/api/members/{uid}')
assert status == 200
assert data[-1]['status'] == 'completed'
@pytest.fixture
def simple_group():
return Group(

5
tests/conftest_ceo.py

@ -1,5 +1,3 @@
import os
import pytest
from .utils import gssapi_token_ctx
@ -7,9 +5,6 @@ from .utils import gssapi_token_ctx
@pytest.fixture(scope='module')
def cli_setup(app_process):
# This tells the CLI entrypoint not to register additional zope services.
os.environ['PYTEST'] = '1'
# Running the client and the server in the same process would be very
# messy because they would be sharing the same environment variables,
# Kerberos cache, and registered utilities (via zope). So we're just

Loading…
Cancel
Save