public
/
pyceo
27
1
Fork 3
Code Issues 25 Pull Requests 5 Projects Releases 1 Wiki Activity

add cloud account API endpoints

This commit is contained in:
Max Erenberg 2021-11-20 18:31:25 -05:00
parent d7551eaf5c
commit fa9c2b33d5
20 changed files with 463 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.drone/data.ldif
View File

@ -94,7 +94,7 @@ objectClass: posixAccount
objectClass: shadowAccount
objectClass: member
program: MAT/Mathematics Computer Science
term: s2021
term: f2021
dn: cn=ctdalek,ou=Group,dc=csclub,dc=internal
objectClass: top
@ -119,7 +119,7 @@ objectClass: posixAccount
objectClass: shadowAccount
objectClass: member
program: MAT/Mathematics Computer Science
term: s2021
term: f2021
dn: cn=regular1,ou=Group,dc=csclub,dc=internal
objectClass: top
@ -144,7 +144,7 @@ objectClass: posixAccount
objectClass: shadowAccount
objectClass: member
program: MAT/Mathematics Computer Science
term: s2021
term: f2021
dn: cn=exec1,ou=Group,dc=csclub,dc=internal
objectClass: top

7
.drone/mail-setup.sh
View File

@ -8,9 +8,10 @@ set -ex
add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail
add_fqdn_to_hosts $(get_ip_addr auth1) auth1
[ -f venv/bin/activate ] && . venv/bin/activate
python tests/MockMailmanServer.py &
python tests/MockSMTPServer.py &
. venv/bin/activate
python -m tests.MockMailmanServer &
python -m tests.MockSMTPServer &
python -m tests.MockCloudStackServer &
export DEBIAN_FRONTEND=noninteractive
apt update

9
ceo_common/errors.py
View File

@ -64,3 +64,12 @@ class DatabaseConnectionError(Exception):
class DatabasePermissionError(Exception):
def __init__(self):
super().__init__('unable to perform action due to lack of permissions')
class InvalidMembershipError(Exception):
def __init__(self):
super().__init__('membership is invalid or expired')
class CloudStackAPIError(Exception):
pass

11
ceo_common/interfaces/ICloudService.py
View File

@ -1,3 +1,5 @@
from typing import Dict
from zope.interface import Interface
from .IUser import IUser
@ -10,3 +12,12 @@ class ICloudService(Interface):
"""
Activate an LDAP account in CloudStack for the given user.
"""
def purge_accounts() -> Dict:
"""
Delete CloudStack accounts which correspond to expired CSC accounts.
A warning message will be emailed to users one week before their
cloud account is deleted.
Another message will be emailed to the users after their cloud account
has been deleted.
"""

3
ceo_common/interfaces/IUser.py
View File

@ -83,3 +83,6 @@ class IUser(Interface):
If get_forwarding_addresses is True, the forwarding addresses
for the user will also be returned, if present.
"""
def membership_is_valid() -> bool:
"""Returns True iff the user's has a non-expired membership."""

7
ceo_common/model/Term.py
View File

@ -65,3 +65,10 @@ class Term:
def __le__(self, other):
return self < other or self == other
def to_datetime(self) -> datetime.datetime:
c = self.s_term[0]
year = int(self.s_term[1:])
month = self.seasons.index(c) * 4 + 1
day = 1
return datetime.datetime(year, month, day)

4
ceod/api/app_factory.py
View File

@ -42,6 +42,10 @@ def create_app(flask_config={}):
from ceod.api import database
app.register_blueprint(database.bp, url_prefix='/api/db')
if hostname == cfg.get('ceod_cloud_host'):
from ceod.api import cloud
app.register_blueprint(cloud.bp, url_prefix='/api/cloud')
from ceod.api import groups
app.register_blueprint(groups.bp, url_prefix='/api/groups')

24
ceod/api/cloud.py Normal file
View File

@ -0,0 +1,24 @@
from flask import Blueprint
from zope import component
from .utils import requires_authentication_no_realm, authz_restrict_to_syscom
from ceo_common.interfaces import ICloudService, ILDAPService
bp = Blueprint('cloud', __name__)
@bp.route('/accounts/create', methods=['POST'])
@requires_authentication_no_realm
def create_account(auth_user: str):
cloud_srv = component.getUtility(ICloudService)
ldap_srv = component.getUtility(ILDAPService)
user = ldap_srv.get_user(auth_user)
cloud_srv.create_account(user)
return {'status': 'OK'}
@bp.route('/accounts/purge', methods=['POST'])
@authz_restrict_to_syscom
def purge_accounts():
cloud_srv = component.getUtility(ICloudService)
return cloud_srv.purge_accounts()

25
ceod/api/error_handlers.py
View File

@ -1,10 +1,14 @@
import traceback
from flask import request
from flask.app import Flask
import ldap3
from werkzeug.exceptions import HTTPException
from ceo_common.errors import UserNotFoundError, GroupNotFoundError
from ceo_common.errors import UserNotFoundError, GroupNotFoundError, \
UserAlreadyExistsError, GroupAlreadyExistsError, BadRequest, \
UserAlreadySubscribedError, InvalidMembershipError, \
CloudStackAPIError
from ceo_common.logger_factory import logger_factory
__all__ = ['register_error_handlers']
@ -20,11 +24,26 @@ def generic_error_handler(err: Exception):
"""Return JSON for all errors."""
if isinstance(err, HTTPException):
status_code = err.code
elif isinstance(err, BadRequest):
status_code = 400
elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult) \
or isinstance(err, InvalidMembershipError):
status_code = 403
elif isinstance(err, UserNotFoundError) or isinstance(err, GroupNotFoundError):
status_code = 404
elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult):
status_code = 403
elif any(isinstance(err, cls) for cls in [
UserAlreadyExistsError, GroupAlreadyExistsError, UserAlreadySubscribedError
]):
status_code = 409
elif isinstance(err, CloudStackAPIError):
status_code = 500
else:
status_code = 500
logger.error(traceback.format_exc())
if request.path.startswith('/api/cloud'):
# I've noticed that the requests library spits out the
# full URL when an Exception is raised, which will cause
# our CloudStack API key to be leaked. So we're going to mask
# it here instead.
err = Exception('Please contact the Systems Committee')
return {'error': type(err).__name__ + ': ' + str(err)}, status_code

117
ceod/model/CloudService.py
View File

@ -1,16 +1,24 @@
from base64 import b64encode
import datetime
import hashlib
import hmac
from typing import Dict
import json
import os
from typing import Dict, List
from urllib.parse import quote
import requests
from zope import component
from zope.interface import implementer
from ceo_common.interfaces import ICloudService, IConfig, IUser
from ceo_common.errors import InvalidMembershipError, CloudStackAPIError
from ceo_common.logger_factory import logger_factory
from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \
IMailService
from ceo_common.model import Term
logger = logger_factory(__name__)
@implementer(ICloudService)
class CloudService:
@ -21,6 +29,11 @@ class CloudService:
self.base_url = cfg.get('cloudstack_base_url')
self.members_domain = cfg.get('cloudstack_members_domain')
state_dir = '/run/ceod'
if not os.path.isdir(state_dir):
os.mkdir(state_dir)
self.pending_deletions_file = os.path.join(state_dir, 'cloudstack_pending_account_deletions.json')
def _create_url(self, params: Dict[str, str]) -> str:
# See https://docs.cloudstack.apache.org/en/latest/developersguide/dev.html#the-cloudstack-api
if 'apiKey' not in params and 'apikey' not in params:
@ -51,12 +64,31 @@ class CloudService:
assert d['count'] == 1, 'there should be one domain found'
return d['domain'][0]['id']
def _get_all_accounts(self, domain_id: str) -> List[Dict]:
url = self._create_url({
'command': 'listAccounts',
'domainid': domain_id,
'details': 'min',
})
resp = requests.get(url)
resp.raise_for_status()
d = resp.json()['listaccountsresponse']
if 'account' not in d:
# The API returns an empty dict if there are no accounts
return []
return d['account']
def _delete_account(self, account_id: str):
url = self._create_url({
'command': 'deleteAccount',
'id': account_id,
})
resp = requests.post(url)
resp.raise_for_status()
def create_account(self, user: IUser):
if not user.terms:
raise Exception('Only members may create cloud accounts')
most_recent_term = max(map(Term, user.terms))
if most_recent_term < Term.current():
raise Exception('Membership has expired for this user')
if not user.membership_is_valid():
raise InvalidMembershipError()
domain_id = self._get_domain_id(self.members_domain)
url = self._create_url({
@ -68,4 +100,73 @@ class CloudService:
resp = requests.post(url)
d = resp.json()['createaccountresponse']
if not resp.ok:
raise Exception(d['errortext'])
raise CloudStackAPIError(d['errortext'])
def purge_accounts(self) -> Dict:
accounts_deleted = []
accounts_to_be_deleted = []
result = {
'accounts_deleted': accounts_deleted,
'accounts_to_be_deleted': accounts_to_be_deleted,
}
current_term = Term.current()
beginning_of_term = current_term.to_datetime()
now = datetime.datetime.now()
delta = now - beginning_of_term
if delta.days < 30:
# one-month grace period
return result
ldap_srv = component.getUtility(ILDAPService)
mail_srv = component.getUtility(IMailService)
domain_id = self._get_domain_id(self.members_domain)
accounts = self._get_all_accounts(domain_id)
if os.path.isfile(self.pending_deletions_file):
state = json.load(open(self.pending_deletions_file))
last_check = datetime.datetime.fromtimestamp(state['timestamp'])
delta = now - last_check
if delta.days < 7:
logger.debug(
'Skipping account purge because less than one week has '
'passed since the warning emails were sent out'
)
return result
username_to_account_id = {
account['name']: account['id']
for account in accounts
}
for username in state['accounts_to_be_deleted']:
if username not in username_to_account_id:
continue
user = ldap_srv.get_user(username)
if user.membership_is_valid():
continue
account_id = username_to_account_id[username]
self._delete_account(account_id)
accounts_deleted.append(username)
mail_srv.send_cloud_account_has_been_deleted_message(user)
logger.info(f'Deleted cloud account for {username}')
os.unlink(self.pending_deletions_file)
return result
state = {
'timestamp': int(now.timestamp()),
'accounts_to_be_deleted': accounts_to_be_deleted,
}
for account in accounts:
username = account['name']
account_id = account['id']
user = ldap_srv.get_user(username)
if user.membership_is_valid():
continue
accounts_to_be_deleted.append(username)
mail_srv.send_cloud_account_will_be_deleted_message(user)
logger.info(
f'A warning email was sent to {username} because their '
'cloud account will be deleted'
)
if accounts_to_be_deleted:
json.dump(state, open(self.pending_deletions_file, 'w'))
return result

29
ceod/model/MailService.py
View File

@ -58,7 +58,8 @@ class MailService:
def send_welcome_message_to(self, user: IUser, password: str):
template = self.jinja_env.get_template('welcome_message.j2')
# TODO: store surname and givenName in LDAP
first_name = user.given_name
if not first_name:
first_name = user.cn.split(' ', 1)[0]
body = template.render(name=first_name, user=user.uid, password=password)
self.send(
@ -94,3 +95,29 @@ class MailService:
},
body,
)
def send_cloud_account_will_be_deleted_message(self, user: IUser):
template = self.jinja_env.get_template('cloud_account_will_be_deleted.j2')
body = template.render(user=user)
self.send(
f'cloudaccounts <ceo+cloudaccounts@{self.base_domain}>',
f'{user.cn} <{user.uid}@{self.base_domain}>',
{
'Subject': 'Your CSC Cloud account will be deleted',
'Cc': f'ceo+cloudaccounts@{self.base_domain}',
},
body,
)
def send_cloud_account_has_been_deleted_message(self, user: IUser):
template = self.jinja_env.get_template('cloud_account_has_been_deleted.j2')
body = template.render(user=user)
self.send(
f'cloudaccounts <ceo+cloudaccounts@{self.base_domain}>',
f'{user.cn} <{user.uid}@{self.base_domain}>',
{
'Subject': 'Your CSC Cloud account has been deleted',
'Cc': f'ceo+cloudaccounts@{self.base_domain}',
},
body,
)

8
ceod/model/User.py
View File

@ -10,6 +10,7 @@ from .utils import should_be_club_rep
from .validators import is_valid_shell, is_valid_term
from ceo_common.interfaces import ILDAPService, IKerberosService, IFileService, \
IUser, IConfig, IMailmanService
from ceo_common.model import Term
@implementer(IUser)
@ -197,3 +198,10 @@ class User:
def set_forwarding_addresses(self, addresses: List[str]):
file_srv = component.getUtility(IFileService)
file_srv.set_forwarding_addresses(self, addresses)
def membership_is_valid(self) -> bool:
if not self.terms:
return False
current_term = Term.current()
most_recent_term = max(map(Term, self.terms))
return most_recent_term >= current_term

14
ceod/model/templates/cloud_account_has_been_deleted.j2 Normal file
View File

@ -0,0 +1,14 @@
Hello {{ user.given_name }},
This is an automated message from ceo, the CSC Electronic Office.
Your club membership has expired, so your CSC Cloud account
has been deleted. If you decide to renew your membership, you
may create a new cloud account, but it will not have any of the
resources from your old cloud account.
If you have any questions or concerns, please contact the Systems
Committee: syscom@csclub.uwaterloo.ca
Best regards,
ceo

18
ceod/model/templates/cloud_account_will_be_deleted.j2 Normal file
View File

@ -0,0 +1,18 @@
Hello {{ user.given_name }},
This is an automated message from ceo, the CSC Electronic Office.
Your club membership has expired, and you have an active account in
the CSC Cloud (https://cloud.csclub.uwaterloo.ca). All of your cloud
resources (VMs, templates, DNS records, etc.) will be permanently
deleted if your membership is not renewed in one week's time.
If you wish to keep your cloud resources, please renew your club
membership before next week. If you do not wish to keep your cloud
resources, then you may safely ignore this message.
If you have any questions or concerns, please contact the Systems
Committee: syscom@csclub.uwaterloo.ca
Best regards,
ceo

2
etc/ceod.ini
View File

@ -78,5 +78,5 @@ host = localhost
[cloudstack]
api_key = REPLACE_ME
secret_key = REPLACE_ME
base_url = http://localhost:8080/api/client
base_url = http://localhost:8080/client/api
members_domain = Members

160
tests/MockCloudStackServer.py Normal file
View File

@ -0,0 +1,160 @@
from uuid import uuid4
from aiohttp import web
from .MockHTTPServerBase import MockHTTPServerBase
def gen_uuid():
return str(uuid4())
class MockCloudStackServer(MockHTTPServerBase):
def __init__(self, port=8080):
routes = [
web.get('/client/api', self.generic_handler),
web.post('/client/api', self.generic_handler),
# for debugging purposes
web.get('/reset', self.reset_handler),
web.post('/reset', self.reset_handler),
]
super().__init__(port, routes)
self.users_by_accountid = {}
self.users_by_username = {}
def clear(self):
self.users_by_accountid.clear()
self.users_by_username.clear()
def reset_handler(self, request):
self.clear()
return web.Response(text='OK\n')
def _add_user(self, username: str):
account_id = gen_uuid()
user_id = gen_uuid()
user = {
"id": user_id,
"username": username,
"firstname": "Calum",
"lastname": "Dalek",
"email": username + "@csclub.internal",
"created": "2021-11-20T11:08:24-0500",
"state": "enabled",
"account": username,
"accounttype": 0,
"usersource": "ldap",
"roleid": "24422759-45de-11ec-b585-32ee6075b19b",
"roletype": "User",
"rolename": "User",
"domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
"domain": "Members",
"accountid": account_id,
"iscallerchilddomain": False,
"isdefault": False
}
self.users_by_accountid[account_id] = user
self.users_by_username[username] = user
return user
def _delete_user(self, account_id: str):
user = self.users_by_accountid[account_id]
username = user['username']
del self.users_by_accountid[account_id]
del self.users_by_username[username]
def _account_from_username(self, username: str):
user = self.users_by_username[username]
return {
"id": user['accountid'],
"name": username,
"accounttype": 0,
"roleid": "24422759-45de-11ec-b585-32ee6075b19b",
"roletype": "User",
"rolename": "User",
"domainid": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
"domain": "Members",
"domainpath": "ROOT/Members",
"state": "enabled",
"user": [user],
"isdefault": False,
"groups": []
}
async def generic_handler(self, request):
command = request.query['command']
if command == 'listDomains':
return web.json_response({
"listdomainsresponse": {
"count": 1,
"domain": [{
"id": "4d2a4a98-b1b4-47a8-ab8f-7e175013a0f0",
"name": "Members",
"level": 1,
"parentdomainid": "f0f8263c-45dd-11ec-b585-32ee6075b19b",
"parentdomainname": "ROOT",
"haschild": False,
"path": "ROOT/Members",
"state": "Active",
"secondarystoragetotal": 0.0
}]
}
})
elif command == 'ldapCreateAccount':
username = request.query['username']
if username in self.users_by_username:
return web.json_response({
"createaccountresponse": {
"uuidList": [],
"errorcode": 530,
"cserrorcode": 4250,
"errortext": f"The user {username} already exists in domain 2"
}
}, status=530)
self._add_user(username)
return web.json_response({
"createaccountresponse": {
"account": self._account_from_username(username),
}
})
elif command == 'listUsers':
users = list(self.users_by_username.values())
return web.json_response({
'listusersresponse': {
'count': len(users),
'user': users,
}
})
elif command == 'listAccounts':
usernames = list(self.users_by_username.keys())
return web.json_response({
'listaccountsresponse': {
'count': len(usernames),
'account': [
self._account_from_username(username)
for username in usernames
]
}
})
elif command == 'deleteAccount':
account_id = request.query['id']
self._delete_user(account_id)
return web.json_response({
'deleteaccountresponse': {
'jobid': gen_uuid()
}
})
else:
return web.json_response({
"errorresponse": {
"uuidList": [],
"errorcode": 401,
"errortext": "unable to verify user credentials and/or request signature"
}
}, status=401)
if __name__ == '__main__':
server = MockCloudStackServer()
server.start()

29
tests/MockHTTPServerBase.py Normal file
View File

@ -0,0 +1,29 @@
from abc import ABC
import asyncio
from threading import Thread
from typing import List
from aiohttp import web
class MockHTTPServerBase(ABC):
def __init__(self, port: int, routes: List):
self.port = port
self.app = web.Application()
self.app.add_routes(routes)
self.runner = web.AppRunner(self.app)
self.loop = asyncio.new_event_loop()
def _start_loop(self):
asyncio.set_event_loop(self.loop)
self.loop.run_until_complete(self.runner.setup())
site = web.TCPSite(self.runner, '127.0.0.1', self.port)
self.loop.run_until_complete(site.start())
self.loop.run_forever()
def start(self):
t = Thread(target=self._start_loop)
t.start()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)

29
tests/MockMailmanServer.py
View File

@ -1,18 +1,15 @@
import asyncio
from threading import Thread
from aiohttp import web
from .MockHTTPServerBase import MockHTTPServerBase
class MockMailmanServer:
class MockMailmanServer(MockHTTPServerBase):
def __init__(self, port=8001, prefix='/3.1'):
self.port = port
self.app = web.Application()
self.app.add_routes([
routes = [
web.post(prefix + '/members', self.subscribe),
web.delete(prefix + '/lists/{mailing_list}/member/{address}', self.unsubscribe),
])
self.runner = web.AppRunner(self.app)
self.loop = asyncio.new_event_loop()
]
super().__init__(port, routes)
# add more as necessary
self.subscriptions = {
@ -22,20 +19,6 @@ class MockMailmanServer:
'syscom-alerts': [],
}
def _start_loop(self):
asyncio.set_event_loop(self.loop)
self.loop.run_until_complete(self.runner.setup())
site = web.TCPSite(self.runner, '127.0.0.1', self.port)
self.loop.run_until_complete(site.start())
self.loop.run_forever()
def start(self):
t = Thread(target=self._start_loop)
t.start()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)
def clear(self):
for key in self.subscriptions:
self.subscriptions[key].clear()

2
tests/__init__.py
View File

@ -1,2 +0,0 @@
from .MockSMTPServer import MockSMTPServer
from .MockMailmanServer import MockMailmanServer

4
tests/ceod_dev.ini
View File

@ -8,7 +8,7 @@ admin_host = phosphoric-acid
fs_root_host = phosphoric-acid
mailman_host = mail
database_host = coffee
cloud_host = biloba
cloud_host = mail
use_https = false
port = 9987
@ -72,5 +72,5 @@ host = localhost
[cloudstack]
api_key = REPLACE_ME
secret_key = REPLACE_ME
base_url = http://localhost:8080/api/client
base_url = http://localhost:8080/client/api
members_domain = Members