From 7425d69feb688845ee529a72e38adc8887cbd155 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Fri, 26 Nov 2021 22:04:07 -0500 Subject: [PATCH 1/3] add VHostManager --- ceo_common/errors.py | 5 + ceo_common/interfaces/ICloudService.py | 21 +++- ceo_common/interfaces/IVHostManager.py | 36 +++++++ ceo_common/interfaces/__init__.py | 1 + ceod/model/CloudService.py | 40 ++++++- ceod/model/VHostManager.py | 101 ++++++++++++++++++ ceod/model/__init__.py | 1 + .../templates/nginx_cloud_vhost_config.j2 | 25 +++++ tests/ceod_dev.ini | 7 ++ tests/ceod_test_local.ini | 7 ++ 10 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 ceo_common/interfaces/IVHostManager.py create mode 100644 ceod/model/VHostManager.py create mode 100644 ceod/model/templates/nginx_cloud_vhost_config.j2 diff --git a/ceo_common/errors.py b/ceo_common/errors.py index 2550fc2..fdc31b9 100644 --- a/ceo_common/errors.py +++ b/ceo_common/errors.py @@ -73,3 +73,8 @@ class InvalidMembershipError(Exception): class CloudStackAPIError(Exception): pass + + +class InvalidDomainError(Exception): + def __init__(self): + super().__init__('domain is invalid') diff --git a/ceo_common/interfaces/ICloudService.py b/ceo_common/interfaces/ICloudService.py index 27bba73..8f58a5c 100644 --- a/ceo_common/interfaces/ICloudService.py +++ b/ceo_common/interfaces/ICloudService.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, List from zope.interface import Interface @@ -21,3 +21,22 @@ class ICloudService(Interface): Another message will be emailed to the users after their cloud account has been deleted. """ + + def create_vhost(username: str, domain: str, ip_address: str): + """ + Create a new vhost record for the given domain and IP address. + """ + + def delete_vhost(username: str, domain: str): + """ + Delete the vhost record for the given user and domain. + """ + + def get_vhosts(username: str) -> List[Dict]: + """ + Get the vhost records for the given user. Each record has the form + { + "domain": "app.username.m.csclub.cloud", + "ip_address": "172.19.134.12" + } + """ diff --git a/ceo_common/interfaces/IVHostManager.py b/ceo_common/interfaces/IVHostManager.py new file mode 100644 index 0000000..ad5debd --- /dev/null +++ b/ceo_common/interfaces/IVHostManager.py @@ -0,0 +1,36 @@ +from typing import List, Dict + +from zope.interface import Interface + + +class IVHostManager(Interface): + """Performs operations on the CSC Cloud.""" + + def create_vhost(username: str, domain: str, ip_address: str): + """ + Create a new vhost record for the given domain and IP address. + """ + + def delete_vhost(username: str, domain: str): + """ + Delete the vhost record for the given user and domain. + """ + + def delete_all_vhosts_for_user(username: str): + """ + Delete all vhost records for the given user. + """ + + def get_num_vhosts(username: str) -> int: + """ + Get the number of vhost records for the given user. + """ + + def get_vhosts(username: str) -> List[Dict]: + """ + Get the vhost records for the given user. Each record has the form + { + "domain": "app.username.m.csclub.cloud", + "ip_address": "172.19.134.12" + } + """ diff --git a/ceo_common/interfaces/__init__.py b/ceo_common/interfaces/__init__.py index 4f32ca8..44a7397 100644 --- a/ceo_common/interfaces/__init__.py +++ b/ceo_common/interfaces/__init__.py @@ -10,3 +10,4 @@ from .IMailService import IMailService from .IMailmanService import IMailmanService from .IHTTPClient import IHTTPClient from .IDatabaseService import IDatabaseService +from .IVHostManager import IVHostManager diff --git a/ceod/model/CloudService.py b/ceod/model/CloudService.py index c86d6a8..a12956a 100644 --- a/ceod/model/CloudService.py +++ b/ceod/model/CloudService.py @@ -4,6 +4,7 @@ import hashlib import hmac import json import os +import re from typing import Dict, List from urllib.parse import quote @@ -11,7 +12,9 @@ import requests from zope import component from zope.interface import implementer -from ceo_common.errors import InvalidMembershipError, CloudStackAPIError +from .VHostManager import VHostManager +from ceo_common.errors import InvalidMembershipError, CloudStackAPIError, \ + InvalidDomainError from ceo_common.logger_factory import logger_factory from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \ IMailService @@ -23,12 +26,21 @@ logger = logger_factory(__name__) @implementer(ICloudService) class CloudService: + VALID_DOMAIN_RE = re.compile(r'^(?:[0-9a-z-]+\.)+[a-z]+$') + def __init__(self): cfg = component.getUtility(IConfig) self.api_key = cfg.get('cloudstack_api_key') self.secret_key = cfg.get('cloudstack_secret_key') self.base_url = cfg.get('cloudstack_base_url') self.members_domain = 'Members' + self.vhost_mgr = VHostManager( + vhost_dir=cfg.get('cloud vhosts_config_dir'), + ssl_cert_path=cfg.get('cloud vhosts_ssl_cert_path'), + ssl_key_path=cfg.get('cloud vhosts_ssl_key_path'), + ) + self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account') + self.vhost_domain = cfg.get('cloud vhosts_members_domain') state_dir = '/run/ceod' if not os.path.isdir(state_dir): @@ -172,3 +184,29 @@ class CloudService: if accounts_to_be_deleted: json.dump(state, open(self.pending_deletions_file, 'w')) return result + + def _is_valid_domain(self, username: str, domain: str) -> bool: + subdomain = username + '.' + self.vhost_domain + if not (domain == subdomain or domain.endswith('.' + subdomain)): + return False + if self.VALID_DOMAIN_RE.match(domain) is None: + return False + if len(domain) > 80: + return False + return True + + def create_vhost(self, username: str, domain: str, ip_address: str): + if self.vhost_mgr.get_num_vhosts(username) >= self.max_vhosts_per_account: + raise Exception(f'Only {self.max_vhosts_per_account} vhosts ' + 'allowed per account') + if not self._is_valid_domain(username, domain): + raise InvalidDomainError() + self.vhost_mgr.create_vhost(username, domain, ip_address) + + def delete_vhost(self, username: str, domain: str): + if not self._is_valid_domain(username, domain): + raise InvalidDomainError() + self.vhost_mgr.delete_vhost(username, domain) + + def get_vhosts(self, username: str) -> List[Dict]: + return self.vhost_mgr.get_vhosts(username) diff --git a/ceod/model/VHostManager.py b/ceod/model/VHostManager.py new file mode 100644 index 0000000..9a97507 --- /dev/null +++ b/ceod/model/VHostManager.py @@ -0,0 +1,101 @@ +import glob +import os +import re +import subprocess +from typing import List, Dict + +from flask import current_app +import jinja2 +from zope.interface import implementer + +from ceo_common.logger_factory import logger_factory +from ceo_common.interfaces import IVHostManager + +PROXY_PASS_IP_RE = re.compile(r'^\s+proxy_pass\s+http://(?P[\d.]+);$') +VHOST_FILENAME_RE = re.compile(r'^member_(?P[0-9a-z-]+)_(?P[0-9a-z-]+)$') +logger = logger_factory(__name__) + + +@implementer(IVHostManager) +class VHostManager: + def __init__( + self, + vhost_dir: str, + ssl_cert_path: str, + ssl_key_path: str, + ): + self.vhost_dir = vhost_dir + self.ssl_cert_path = ssl_cert_path + self.ssl_key_path = ssl_key_path + self.jinja_env = jinja2.Environment( + loader=jinja2.PackageLoader('ceod.model'), + ) + + @staticmethod + def _vhost_filename(username: str, domain: str) -> str: + """Generate a filename for the vhost record""" + # sanity check... + assert '..' not in domain and '/' not in domain + return 'member' + '_' + username + '_' + domain + + def _vhost_filepath(self, username: str, domain: str) -> str: + """Generate an absolute path for the vhost record""" + return os.path.join(self.vhost_dir, self._vhost_filename(username, domain)) + + def _vhost_files(self, username: str) -> List[str]: + """Return a list of all vhost files for this user.""" + return glob.glob(os.path.join(self.vhost_dir, 'member_' + username + '_*')) + + def _reload_web_server(self): + if current_app.config.get('ENV') == 'development' or \ + current_app.config.get('TESTING'): + logger.info('Not reloading web server because we are in development') + return + subprocess.run(['systemctl', 'reload', 'nginx'], check=True) + + def create_vhost(self, username: str, domain: str, ip_address: str): + template = self.jinja_env.get_template('nginx_cloud_vhost_config.j2') + body = template.render( + username=username, domain=domain, ip_address=ip_address, + ssl_cert_path=self.ssl_cert_path, ssl_key_path=self.ssl_key_path) + filepath = self._vhost_filepath(username, domain) + logger.info(f'Writing a new vhost ({domain} -> {ip_address}) to {filepath}') + with open(filepath, 'w') as fo: + fo.write(body) + self._reload_web_server() + + def delete_vhost(self, username: str, domain: str): + filepath = self._vhost_filepath(username, domain) + logger.info(f'Deleting {filepath}') + os.unlink(filepath) + self._reload_web_server() + + def get_num_vhosts(self, username: str) -> int: + return len(self._vhost_files(username)) + + def get_vhosts(self, username: str) -> List[Dict]: + vhosts = [] + for filepath in self._vhost_files(username): + filename = os.path.basename(filepath) + match = VHOST_FILENAME_RE.match(filename) + assert match is not None, f"'{filename}' does not match expected pattern" + domain = match.group('domain') + ip_address = None + for line in open(filepath): + match = PROXY_PASS_IP_RE.match(line) + if match is None: + continue + ip_address = match.group('ip_address') + break + assert ip_address is not None, f"Could not find IP address in {filename}" + vhosts.append({'domain': domain, 'ip_address': ip_address}) + return vhosts + + def delete_all_vhosts_for_user(self, username: str): + filepaths = self._vhost_files(username) + if not filepaths: + return + for filepath in filepaths: + logger.info(f'Deleting {filepath}') + os.unlink(filepath) + self._reload_web_server() diff --git a/ceod/model/__init__.py b/ceod/model/__init__.py index b88889c..61be811 100644 --- a/ceod/model/__init__.py +++ b/ceod/model/__init__.py @@ -9,3 +9,4 @@ from .FileService import FileService from .SudoRole import SudoRole from .MailService import MailService from .MailmanService import MailmanService +from .VHostManager import VHostManager diff --git a/ceod/model/templates/nginx_cloud_vhost_config.j2 b/ceod/model/templates/nginx_cloud_vhost_config.j2 new file mode 100644 index 0000000..869606d --- /dev/null +++ b/ceod/model/templates/nginx_cloud_vhost_config.j2 @@ -0,0 +1,25 @@ +# This file is automatically managed by ceod. +# DO NOT EDIT THIS FILE MANUALLY. +# If you want to modify it, please move it to another directory. + +server { + listen 80; + listen [::]:80; + server_name {{ domain }}; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name {{ domain }}; + ssl_certificate {{ ssl_cert_path }}; + ssl_certificate_key {{ ssl_key_path }}; + + location / { + proxy_pass http://{{ ip_address }}; + } + + access_log /var/log/nginx/member-{{ username }}-access.log; + error_log /var/log/nginx/member-{{ username }}-error.log; +} diff --git a/tests/ceod_dev.ini b/tests/ceod_dev.ini index 16944c8..5aa1e64 100644 --- a/tests/ceod_dev.ini +++ b/tests/ceod_dev.ini @@ -73,3 +73,10 @@ host = localhost api_key = REPLACE_ME secret_key = REPLACE_ME base_url = http://localhost:8080/client/api + +[cloud vhosts] +config_dir = /run/ceod/member-vhosts +ssl_cert_path = /etc/ssl/private/csclub.cloud.chain +ssl_key_path = /etc/ssl/private/csclub.cloud.key +max_vhosts_per_account = 10 +members_domain = m.csclub.cloud diff --git a/tests/ceod_test_local.ini b/tests/ceod_test_local.ini index fdc988f..c89098c 100644 --- a/tests/ceod_test_local.ini +++ b/tests/ceod_test_local.ini @@ -72,3 +72,10 @@ host = coffee api_key = REPLACE_ME secret_key = REPLACE_ME base_url = http://localhost:8080/client/api + +[cloud vhosts] +config_dir = /run/ceod/member-vhosts +ssl_cert_path = /etc/ssl/private/csclub.cloud.chain +ssl_key_path = /etc/ssl/private/csclub.cloud.key +max_vhosts_per_account = 10 +members_domain = m.csclub.cloud -- 2.39.2 From 0a52b0b395aa312c40dd9da9c66da735d861b594 Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Sat, 27 Nov 2021 14:58:01 -0500 Subject: [PATCH 2/3] add unit tests --- .drone/common.sh | 3 + .drone/mail-setup.sh | 3 + ceo/cli/cloud.py | 43 +++++++++++++- ceo/utils.py | 4 ++ ceo_common/errors.py | 5 ++ ceo_common/model/HTTPClient.py | 4 +- ceod/api/cloud.py | 37 ++++++++++-- ceod/api/error_handlers.py | 6 +- ceod/api/utils.py | 11 ++++ ceod/model/CloudService.py | 20 +++++-- ceod/model/VHostManager.py | 8 +-- etc/ceod.ini | 9 +++ tests/ceo/cli/test_cloud.py | 26 +++++++++ tests/ceod/api/test_cloud.py | 100 ++++++++++++++++++++++++++++++++ tests/ceod/model/test_vhosts.py | 28 +++++++++ tests/ceod_dev.ini | 2 + tests/ceod_test_local.ini | 2 + tests/conftest.py | 16 ++++- tests/conftest_ceod_api.py | 3 + 19 files changed, 306 insertions(+), 24 deletions(-) create mode 100644 tests/ceod/model/test_vhosts.py diff --git a/.drone/common.sh b/.drone/common.sh index de52459..963e480 100644 --- a/.drone/common.sh +++ b/.drone/common.sh @@ -6,6 +6,9 @@ sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > / cp /tmp/resolv.conf /etc/resolv.conf rm /tmp/resolv.conf +# mock out systemctl +ln -s /bin/true /usr/local/bin/systemctl + get_ip_addr() { getent hosts $1 | cut -d' ' -f1 } diff --git a/.drone/mail-setup.sh b/.drone/mail-setup.sh index 0e83a78..0265936 100755 --- a/.drone/mail-setup.sh +++ b/.drone/mail-setup.sh @@ -18,5 +18,8 @@ apt update apt install -y netcat-openbsd auth_setup mail +# for the VHostManager +mkdir -p /run/ceod/member-vhosts + # sync with phosphoric-acid nc -l 0.0.0.0 9000 & diff --git a/ceo/cli/cloud.py b/ceo/cli/cloud.py index 150549a..0e44922 100644 --- a/ceo/cli/cloud.py +++ b/ceo/cli/cloud.py @@ -3,8 +3,8 @@ from zope import component from ceo_common.interfaces import IConfig -from ..utils import http_post -from .utils import handle_sync_response +from ..utils import http_post, http_put, http_get, http_delete +from .utils import Abort, handle_sync_response, print_colon_kv @click.group(short_help='Perform operations on the CSC cloud') @@ -44,3 +44,42 @@ def purge(): result = handle_sync_response(resp) click.echo('Accounts to be deleted: ' + ','.join(result['accounts_to_be_deleted'])) click.echo('Accounts which were deleted: ' + ','.join(result['accounts_deleted'])) + + +@cloud.group(short_help='Manage your virtual hosts') +def vhosts(): + pass + + +@vhosts.command(name='add', short_help='Add a virtual host') +@click.argument('domain') +@click.argument('ip_address') +def add_vhost(domain, ip_address): + body = {'ip_address': ip_address} + if '/' in domain: + raise Abort('invalid domain name') + resp = http_put('/api/cloud/vhosts/' + domain, json=body) + handle_sync_response(resp) + click.echo('Done.') + + +@vhosts.command(name='delete', short_help='Delete a virtual host') +@click.argument('domain') +def delete_vhost(domain): + if '/' in domain: + raise Abort('invalid domain name') + resp = http_delete('/api/cloud/vhosts/' + domain) + handle_sync_response(resp) + click.echo('Done.') + + +@vhosts.command(name='list', short_help='List virtual hosts') +def list_vhosts(): + resp = http_get('/api/cloud/vhosts') + result = handle_sync_response(resp) + vhosts = result['vhosts'] + if not vhosts: + click.echo('No vhosts found.') + return + pairs = [(d['domain'], d['ip_address']) for d in vhosts] + print_colon_kv(pairs) diff --git a/ceo/utils.py b/ceo/utils.py index f915a9a..e977759 100644 --- a/ceo/utils.py +++ b/ceo/utils.py @@ -42,6 +42,10 @@ def http_delete(path: str, **kwargs) -> requests.Response: return http_request('DELETE', path, **kwargs) +def http_put(path: str, **kwargs) -> requests.Response: + return http_request('PUT', path, **kwargs) + + def get_failed_operations(data: List[Dict]) -> List[str]: """ Get a list of the failed operations using the JSON objects diff --git a/ceo_common/errors.py b/ceo_common/errors.py index fdc31b9..6cf844a 100644 --- a/ceo_common/errors.py +++ b/ceo_common/errors.py @@ -78,3 +78,8 @@ class CloudStackAPIError(Exception): class InvalidDomainError(Exception): def __init__(self): super().__init__('domain is invalid') + + +class InvalidIPError(Exception): + def __init__(self): + super().__init__('IP address is invalid') diff --git a/ceo_common/model/HTTPClient.py b/ceo_common/model/HTTPClient.py index 67fbc11..6a38e23 100644 --- a/ceo_common/model/HTTPClient.py +++ b/ceo_common/model/HTTPClient.py @@ -27,8 +27,8 @@ class HTTPClient: host = host + '.' + self.base_domain if method == 'GET': - # This is the only GET endpoint which requires auth - need_auth = path.startswith('/api/members') + need_auth = path.startswith('/api/members') or \ + path.startswith('/api/cloud') delegate = False else: need_auth = True diff --git a/ceod/api/cloud.py b/ceod/api/cloud.py index 497d09f..be7467b 100644 --- a/ceod/api/cloud.py +++ b/ceod/api/cloud.py @@ -1,8 +1,9 @@ -from flask import Blueprint +from flask import Blueprint, request from zope import component -from .utils import requires_authentication_no_realm, authz_restrict_to_syscom -from ceo_common.interfaces import ICloudService, ILDAPService +from .utils import requires_authentication_no_realm, authz_restrict_to_syscom, \ + get_valid_member_or_throw +from ceo_common.interfaces import ICloudService bp = Blueprint('cloud', __name__) @@ -10,9 +11,8 @@ bp = Blueprint('cloud', __name__) @bp.route('/accounts/create', methods=['POST']) @requires_authentication_no_realm def create_account(auth_user: str): + user = get_valid_member_or_throw(auth_user) cloud_srv = component.getUtility(ICloudService) - ldap_srv = component.getUtility(ILDAPService) - user = ldap_srv.get_user(auth_user) cloud_srv.create_account(user) return {'status': 'OK'} @@ -22,3 +22,30 @@ def create_account(auth_user: str): def purge_accounts(): cloud_srv = component.getUtility(ICloudService) return cloud_srv.purge_accounts() + + +@bp.route('/vhosts/', methods=['PUT']) +@requires_authentication_no_realm +def create_vhost(auth_user: str, domain: str): + get_valid_member_or_throw(auth_user) + cloud_srv = component.getUtility(ICloudService) + body = request.get_json(force=True) + ip_address = body['ip_address'] + cloud_srv.create_vhost(auth_user, domain, ip_address) + return {'status': 'OK'} + + +@bp.route('/vhosts/', methods=['DELETE']) +@requires_authentication_no_realm +def delete_vhost(auth_user: str, domain: str): + cloud_srv = component.getUtility(ICloudService) + cloud_srv.delete_vhost(auth_user, domain) + return {'status': 'OK'} + + +@bp.route('/vhosts', methods=['GET']) +@requires_authentication_no_realm +def get_vhosts(auth_user: str): + cloud_srv = component.getUtility(ICloudService) + vhosts = cloud_srv.get_vhosts(auth_user) + return {'vhosts': vhosts} diff --git a/ceod/api/error_handlers.py b/ceod/api/error_handlers.py index d9e17f0..43ec699 100644 --- a/ceod/api/error_handlers.py +++ b/ceod/api/error_handlers.py @@ -8,7 +8,7 @@ from werkzeug.exceptions import HTTPException from ceo_common.errors import UserNotFoundError, GroupNotFoundError, \ UserAlreadyExistsError, GroupAlreadyExistsError, BadRequest, \ UserAlreadySubscribedError, InvalidMembershipError, \ - CloudStackAPIError + CloudStackAPIError, InvalidDomainError, InvalidIPError from ceo_common.logger_factory import logger_factory __all__ = ['register_error_handlers'] @@ -24,7 +24,9 @@ def generic_error_handler(err: Exception): """Return JSON for all errors.""" if isinstance(err, HTTPException): status_code = err.code - elif isinstance(err, BadRequest): + elif any(isinstance(err, cls) for cls in [ + BadRequest, InvalidDomainError, InvalidIPError + ]): status_code = 400 elif isinstance(err, ldap3.core.exceptions.LDAPStrongerAuthRequiredResult) \ or isinstance(err, InvalidMembershipError): diff --git a/ceod/api/utils.py b/ceod/api/utils.py index 5b7c2da..acd255f 100644 --- a/ceod/api/utils.py +++ b/ceod/api/utils.py @@ -7,14 +7,25 @@ import traceback from typing import Callable, List from flask import current_app, stream_with_context +from zope import component from .spnego import requires_authentication +from ceo_common.errors import InvalidMembershipError +from ceo_common.interfaces import IUser, ILDAPService from ceo_common.logger_factory import logger_factory from ceod.transactions import AbstractTransaction logger = logger_factory(__name__) +def get_valid_member_or_throw(username: str) -> IUser: + ldap_srv = component.getUtility(ILDAPService) + user = ldap_srv.get_user(username) + if not user.membership_is_valid(): + raise InvalidMembershipError() + return user + + def requires_authentication_no_realm(f: Callable) -> Callable: """ Like requires_authentication, but strips the realm out of the principal string. diff --git a/ceod/model/CloudService.py b/ceod/model/CloudService.py index a12956a..fd64e3d 100644 --- a/ceod/model/CloudService.py +++ b/ceod/model/CloudService.py @@ -2,6 +2,7 @@ from base64 import b64encode import datetime import hashlib import hmac +import ipaddress import json import os import re @@ -13,8 +14,7 @@ from zope import component from zope.interface import implementer from .VHostManager import VHostManager -from ceo_common.errors import InvalidMembershipError, CloudStackAPIError, \ - InvalidDomainError +from ceo_common.errors import CloudStackAPIError, InvalidDomainError, InvalidIPError from ceo_common.logger_factory import logger_factory from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \ IMailService @@ -41,6 +41,8 @@ class CloudService: ) self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account') self.vhost_domain = cfg.get('cloud vhosts_members_domain') + self.vhost_ip_min = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_min')) + self.vhost_ip_max = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_max')) state_dir = '/run/ceod' if not os.path.isdir(state_dir): @@ -100,8 +102,6 @@ class CloudService: resp.raise_for_status() def create_account(self, user: IUser): - if not user.membership_is_valid(): - raise InvalidMembershipError() domain_id = self._get_domain_id(self.members_domain) url = self._create_url({ @@ -158,7 +158,10 @@ class CloudService: if user.membership_is_valid(): continue account_id = username_to_account_id[username] + self._delete_account(account_id) + self.vhost_mgr.delete_all_vhosts_for_user(username) + accounts_deleted.append(username) mail_srv.send_cloud_account_has_been_deleted_message(user) logger.info(f'Deleted cloud account for {username}') @@ -195,12 +198,21 @@ class CloudService: return False return True + def _is_valid_ip_address(self, ip_address: str) -> bool: + try: + addr = ipaddress.ip_address(ip_address) + except ValueError: + return False + return self.vhost_ip_min <= addr <= self.vhost_ip_max + def create_vhost(self, username: str, domain: str, ip_address: str): if self.vhost_mgr.get_num_vhosts(username) >= self.max_vhosts_per_account: raise Exception(f'Only {self.max_vhosts_per_account} vhosts ' 'allowed per account') if not self._is_valid_domain(username, domain): raise InvalidDomainError() + if not self._is_valid_ip_address(ip_address): + raise InvalidIPError() self.vhost_mgr.create_vhost(username, domain, ip_address) def delete_vhost(self, username: str, domain: str): diff --git a/ceod/model/VHostManager.py b/ceod/model/VHostManager.py index 9a97507..a4a86a9 100644 --- a/ceod/model/VHostManager.py +++ b/ceod/model/VHostManager.py @@ -4,7 +4,6 @@ import re import subprocess from typing import List, Dict -from flask import current_app import jinja2 from zope.interface import implementer @@ -12,7 +11,7 @@ from ceo_common.logger_factory import logger_factory from ceo_common.interfaces import IVHostManager PROXY_PASS_IP_RE = re.compile(r'^\s+proxy_pass\s+http://(?P[\d.]+);$') -VHOST_FILENAME_RE = re.compile(r'^member_(?P[0-9a-z-]+)_(?P[0-9a-z-]+)$') +VHOST_FILENAME_RE = re.compile(r'^member_(?P[0-9a-z-]+)_(?P[0-9a-z.-]+)$') logger = logger_factory(__name__) @@ -47,10 +46,7 @@ class VHostManager: return glob.glob(os.path.join(self.vhost_dir, 'member_' + username + '_*')) def _reload_web_server(self): - if current_app.config.get('ENV') == 'development' or \ - current_app.config.get('TESTING'): - logger.info('Not reloading web server because we are in development') - return + logger.debug('Reloading nginx') subprocess.run(['systemctl', 'reload', 'nginx'], check=True) def create_vhost(self, username: str, domain: str, ip_address: str): diff --git a/etc/ceod.ini b/etc/ceod.ini index 6693389..d1f02a0 100644 --- a/etc/ceod.ini +++ b/etc/ceod.ini @@ -79,3 +79,12 @@ host = localhost api_key = REPLACE_ME secret_key = REPLACE_ME base_url = http://localhost:8080/client/api + +[cloud vhosts] +config_dir = /etc/nginx/ceod-member-vhosts +ssl_cert_path = /etc/ssl/private/csclub.cloud.chain +ssl_key_path = /etc/ssl/private/csclub.cloud.key +max_vhosts_per_account = 10 +members_domain = m.csclub.cloud +ip_range_min = 172.19.134.10 +ip_range_max = 172.19.134.160 diff --git a/tests/ceo/cli/test_cloud.py b/tests/ceo/cli/test_cloud.py index d233b70..326a85b 100644 --- a/tests/ceo/cli/test_cloud.py +++ b/tests/ceo/cli/test_cloud.py @@ -26,3 +26,29 @@ def test_cloud_accounts_purge(cli_setup, mock_cloud_server): runner = CliRunner() result = runner.invoke(cli, ['cloud', 'accounts', 'purge']) assert result.exit_code == 0 + + +def test_cloud_vhosts(cli_setup, new_user, cfg): + members_domain = cfg.get('cloud vhosts_members_domain') + uid = new_user.uid + domain1 = uid + '.' + members_domain + ip1 = '172.19.134.11' + + runner = CliRunner() + with gssapi_token_ctx(uid): + result = runner.invoke(cli, ['cloud', 'vhosts', 'add', domain1, ip1]) + expected = 'Done.\n' + assert result.exit_code == 0 + assert result.output == expected + + with gssapi_token_ctx(uid): + result = runner.invoke(cli, ['cloud', 'vhosts', 'list']) + expected = domain1 + ': ' + ip1 + '\n' + assert result.exit_code == 0 + assert result.output == expected + + with gssapi_token_ctx(uid): + result = runner.invoke(cli, ['cloud', 'vhosts', 'delete', domain1]) + expected = 'Done.\n' + assert result.exit_code == 0 + assert result.output == expected diff --git a/tests/ceod/api/test_cloud.py b/tests/ceod/api/test_cloud.py index 821fce4..9787489 100644 --- a/tests/ceod/api/test_cloud.py +++ b/tests/ceod/api/test_cloud.py @@ -84,3 +84,103 @@ def test_purge_accounts( assert new_user.uid not in mock_cloud_server.users_by_username assert len(mock_mail_server.messages) == 2 mock_mail_server.messages.clear() + + +def test_cloud_vhosts(cfg, client, new_user, ldap_conn): + members_domain = cfg.get('cloud vhosts_members_domain') + max_vhosts = cfg.get('cloud vhosts_max_vhosts_per_account') + uid = new_user.uid + + domain1 = uid + '.' + members_domain + ip1 = '172.19.134.11' + status, _ = client.put( + f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1}, + principal=uid) + assert status == 200 + status, data = client.get('/api/cloud/vhosts', principal=uid) + assert status == 200 + assert data == {'vhosts': [{'domain': domain1, 'ip_address': ip1}]} + + # invalid domain name + domain2 = uid + 'm.cloud.' + cfg.get('base_domain') + ip2 = ip1 + status, _ = client.put( + f'/api/cloud/vhosts/{domain2}', json={'ip_address': ip2}, + principal=uid) + assert status == 400 + + # invalid IP address + domain3 = domain1 + ip3 = '129.97.134.10' + status, _ = client.put( + f'/api/cloud/vhosts/{domain3}', json={'ip_address': ip3}, + principal=uid) + assert status == 400 + + # new vhost with same domain should replace old one + domain4 = domain1 + ip4 = '172.19.134.14' + status, _ = client.put( + f'/api/cloud/vhosts/{domain4}', json={'ip_address': ip4}, + principal=uid) + assert status == 200 + status, data = client.get('/api/cloud/vhosts', principal=uid) + assert status == 200 + assert data == {'vhosts': [{'domain': domain4, 'ip_address': ip4}]} + + # maximum number of vhosts + for i in range(max_vhosts): + domain = 'app' + str(i + 1) + '.' + uid + '.' + members_domain + status, _ = client.put( + f'/api/cloud/vhosts/{domain}', json={'ip_address': ip1}, + principal=uid) + if i < max_vhosts - 1: + assert status == 200 + else: + assert status != 200 + + # delete a vhost + status, _ = client.delete(f'/api/cloud/vhosts/{domain1}', principal=uid) + assert status == 200 + + # expired members may not create vhosts + expire_member(new_user, ldap_conn) + status, _ = client.put( + f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1}, + principal=uid) + assert status == 403 + + +def test_cloud_vhosts_purged_account( + cfg, client, mock_cloud_server, mock_mail_server, cloud_srv, new_user, + ldap_conn, +): + uid = new_user.uid + members_domain = cfg.get('cloud vhosts_members_domain') + mock_cloud_server.clear() + current_term = Term.current() + beginning_of_term = current_term.to_datetime() + domain1 = uid + '.' + members_domain + ip1 = '172.19.134.11' + + client.post('/api/cloud/accounts/create', principal=uid) + client.put( + f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1}, + principal=uid) + + expire_member(new_user, ldap_conn) + with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock: + # grace period has passed - user should be sent a warning + now_mock.return_value = beginning_of_term + datetime.timedelta(days=31) + client.post('/api/cloud/accounts/purge') + + # one week has passed - the account can now be deleted + now_mock.return_value += datetime.timedelta(days=8) + client.post('/api/cloud/accounts/purge') + + # vhosts should have been deleted + status, data = client.get('/api/cloud/vhosts', principal=uid) + assert status == 200 + assert data == {'vhosts': []} + + mock_mail_server.messages.clear() diff --git a/tests/ceod/model/test_vhosts.py b/tests/ceod/model/test_vhosts.py new file mode 100644 index 0000000..441a6a9 --- /dev/null +++ b/tests/ceod/model/test_vhosts.py @@ -0,0 +1,28 @@ +import os + + +def test_vhost_mgr(cloud_srv): + vhost_mgr = cloud_srv.vhost_mgr + username = 'test1' + domain = username + '.m.csclub.cloud' + filename = f'member_{username}_{domain}' + ip_address = '172.19.134.11' + vhost_mgr.create_vhost(username, domain, ip_address) + path = os.path.join(vhost_mgr.vhost_dir, filename) + assert os.path.isfile(path) + + assert vhost_mgr.get_num_vhosts(username) == 1 + + assert vhost_mgr.get_vhosts(username) == [{ + 'domain': domain, 'ip_address': ip_address, + }] + + domain2 = 'app.' + domain + vhost_mgr.create_vhost(username, domain2, ip_address) + assert vhost_mgr.get_num_vhosts(username) == 2 + + vhost_mgr.delete_vhost(username, domain) + assert vhost_mgr.get_num_vhosts(username) == 1 + + vhost_mgr.delete_all_vhosts_for_user(username) + assert vhost_mgr.get_num_vhosts(username) == 0 diff --git a/tests/ceod_dev.ini b/tests/ceod_dev.ini index 5aa1e64..9fe19b5 100644 --- a/tests/ceod_dev.ini +++ b/tests/ceod_dev.ini @@ -80,3 +80,5 @@ ssl_cert_path = /etc/ssl/private/csclub.cloud.chain ssl_key_path = /etc/ssl/private/csclub.cloud.key max_vhosts_per_account = 10 members_domain = m.csclub.cloud +ip_range_min = 172.19.134.10 +ip_range_max = 172.19.134.160 diff --git a/tests/ceod_test_local.ini b/tests/ceod_test_local.ini index c89098c..ff1b340 100644 --- a/tests/ceod_test_local.ini +++ b/tests/ceod_test_local.ini @@ -79,3 +79,5 @@ ssl_cert_path = /etc/ssl/private/csclub.cloud.chain ssl_key_path = /etc/ssl/private/csclub.cloud.key max_vhosts_per_account = 10 members_domain = m.csclub.cloud +ip_range_min = 172.19.134.10 +ip_range_max = 172.19.134.160 diff --git a/tests/conftest.py b/tests/conftest.py index add1783..32543fe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,7 +59,7 @@ def cfg(_drone_hostname_mock): def delete_test_princs(krb_srv): proc = subprocess.run([ - 'kadmin', '-k', '-p', krb_srv.admin_principal, 'listprincs', 'test_*', + 'kadmin', '-k', '-p', krb_srv.admin_principal, 'listprincs', 'test*', ], text=True, capture_output=True, check=True) princs = [line.strip() for line in proc.stdout.splitlines()] for princ in princs: @@ -268,7 +268,17 @@ def postgresql_srv(cfg): @pytest.fixture(scope='session') -def cloud_srv(cfg): +def vhost_dir_setup(cfg): + vhost_dir = cfg.get('cloud vhosts_config_dir') + if os.path.isdir(vhost_dir): + shutil.rmtree(vhost_dir) + os.makedirs(vhost_dir) + yield + shutil.rmtree(vhost_dir) + + +@pytest.fixture(scope='session') +def cloud_srv(cfg, vhost_dir_setup): _cloud_srv = CloudService() component.getGlobalSiteManager().registerUtility(_cloud_srv, ICloudService) return _cloud_srv @@ -350,7 +360,7 @@ _new_user_id_counter = 10001 @pytest.fixture # noqa: E302 def new_user(client, g_admin_ctx, ldap_srv_session): # noqa: F811 global _new_user_id_counter - uid = 'test_' + str(_new_user_id_counter) + uid = 'test' + str(_new_user_id_counter) _new_user_id_counter += 1 status, data = client.post('/api/members', json={ 'uid': uid, diff --git a/tests/conftest_ceod_api.py b/tests/conftest_ceod_api.py index f60c37b..8d12f35 100644 --- a/tests/conftest_ceod_api.py +++ b/tests/conftest_ceod_api.py @@ -73,3 +73,6 @@ class CeodTestClient: def delete(self, path, principal=None, need_auth=True, delegate=True, **kwargs): return self.request('DELETE', path, principal, need_auth, delegate, **kwargs) + + def put(self, path, principal=None, need_auth=True, delegate=True, **kwargs): + return self.request('PUT', path, principal, need_auth, delegate, **kwargs) -- 2.39.2 From 9cbb0f299d6b8d640c1b4d0d5bee8743bbc4b490 Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Sat, 27 Nov 2021 15:27:35 -0500 Subject: [PATCH 3/3] add docs --- docs/openapi.yaml | 95 +++++++++++++++++++++++++++++++++++++----- docs/redoc-static.html | 25 +++++++---- 2 files changed, 103 insertions(+), 17 deletions(-) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index a017305..fe3250c 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -713,16 +713,7 @@ paths: description: Activate a cloud account for the calling user responses: "200": - description: Success - content: - application/json: - schema: - type: object - properties: - status: - type: string - description: '"OK"' - example: {"status": "OK"} + "$ref": "#/components/responses/SimpleSuccessResponse" "403": "$ref": "#/components/responses/InvalidMembershipErrorResponse" /cloud/accounts/purge: @@ -757,6 +748,79 @@ paths: description: usernames of accounts which were deleted items: type: string + /cloud/vhosts/{domain}: + put: + tags: ['cloud'] + servers: + - url: https://biloba.csclub.uwaterloo.ca:9987/api + summary: Create a vhost + description: Add a new virtual host configuration. + parameters: + - name: domain + in: path + description: domain name of the virtual host + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + ip_address: + type: string + description: IP address of the virtual host + example: {"ip_address": "172.19.134.11"} + responses: + "200": + "$ref": "#/components/responses/SimpleSuccessResponse" + "403": + "$ref": "#/components/responses/InvalidMembershipErrorResponse" + delete: + tags: ['cloud'] + servers: + - url: https://biloba.csclub.uwaterloo.ca:9987/api + summary: Delete a vhost + description: Delete a virtual host configuration. + parameters: + - name: domain + in: path + description: domain name of the virtual host + required: true + schema: + type: string + responses: + "200": + "$ref": "#/components/responses/SimpleSuccessResponse" + /cloud/vhosts: + get: + tags: ['cloud'] + servers: + - url: https://biloba.csclub.uwaterloo.ca:9987/api + summary: List all vhosts + description: List all virtual host configurations for the calling user. + responses: + "200": + description: Success + content: + application/json: + schema: + type: object + properties: + vhosts: + type: array + description: virtual hosts + items: + type: object + properties: + domain: + type: string + description: domain name of the virtual host + ip_address: + type: string + description: IP address of the virtual host + example: {"vhosts": [{"domain": "ctdalek.m.csclub.cloud", "ip_address": "172.19.134.11"}]} components: securitySchemes: GSSAPIAuth: @@ -932,3 +996,14 @@ components: InvalidMembershipErrorResponse: <<: *ErrorResponse description: Membership is invalid or expired + SimpleSuccessResponse: + description: Success + content: + application/json: + schema: + type: object + properties: + status: + type: string + description: '"OK"' + example: {"status": "OK"} diff --git a/docs/redoc-static.html b/docs/redoc-static.html index 0ccf8ea..708eff9 100644 --- a/docs/redoc-static.html +++ b/docs/redoc-static.html @@ -352,7 +352,8 @@ data-styled.g108[id="sc-dWBRfb"]{content:"jnEbBv,"}/*!sc*/ .cAOCuf{font-size:0.929em;line-height:20px;background-color:#2F8132;color:#ffffff;padding:3px 10px;text-transform:uppercase;font-family:Montserrat,sans-serif;margin:0;}/*!sc*/ .iZkjfb{font-size:0.929em;line-height:20px;background-color:#bf581d;color:#ffffff;padding:3px 10px;text-transform:uppercase;font-family:Montserrat,sans-serif;margin:0;}/*!sc*/ .gemyvL{font-size:0.929em;line-height:20px;background-color:#cc3333;color:#ffffff;padding:3px 10px;text-transform:uppercase;font-family:Montserrat,sans-serif;margin:0;}/*!sc*/ -data-styled.g109[id="sc-jHcXXw"]{content:"bXnXQF,cAOCuf,iZkjfb,gemyvL,"}/*!sc*/ +.inNGOu{font-size:0.929em;line-height:20px;background-color:#95507c;color:#ffffff;padding:3px 10px;text-transform:uppercase;font-family:Montserrat,sans-serif;margin:0;}/*!sc*/ +data-styled.g109[id="sc-jHcXXw"]{content:"bXnXQF,cAOCuf,iZkjfb,gemyvL,inNGOu,"}/*!sc*/ .gBwOdz{position:absolute;width:100%;z-index:100;background:#fafafa;color:#263238;box-sizing:border-box;box-shadow:0px 0px 6px rgba(0,0,0,0.33);overflow:hidden;border-bottom-left-radius:4px;border-bottom-right-radius:4px;-webkit-transition:all 0.25s ease;transition:all 0.25s ease;visibility:hidden;-webkit-transform:translateY(-50%) scaleY(0);-ms-transform:translateY(-50%) scaleY(0);transform:translateY(-50%) scaleY(0);}/*!sc*/ data-styled.g110[id="sc-bQCEYZ"]{content:"gBwOdz,"}/*!sc*/ .fKFAhr{padding:10px;}/*!sc*/ @@ -429,7 +430,7 @@ data-styled.g140[id="sc-amkrK"]{content:"icZuVc,"}/*!sc*/ -
Documentation Powered by ReDoc
Documentation Powered by ReDoc
Authorizations:
GSSAPIAuth

Responses

Response samples

Content type
application/json
{
  • "accounts_to_be_deleted": [
    ],
  • "accounts_deleted": [
    ]
}

positions

Show current positions

Shows the list of positions and members holding them.

+

Response samples

Content type
application/json
{
  • "accounts_to_be_deleted": [
    ],
  • "accounts_deleted": [
    ]
}

Create a vhost

Add a new virtual host configuration.

+
Authorizations:
GSSAPIAuth
path Parameters
domain
required
string

domain name of the virtual host

+
Request Body schema: application/json
ip_address
string

IP address of the virtual host

+

Responses

Request samples

Content type
application/json
{
  • "ip_address": "172.19.134.11"
}

Response samples

Content type
application/json
{
  • "status": "OK"
}

Delete a vhost

Delete a virtual host configuration.

+
Authorizations:
GSSAPIAuth
path Parameters
domain
required
string

domain name of the virtual host

+

Responses

Response samples

Content type
application/json
{
  • "status": "OK"
}

List all vhosts

List all virtual host configurations for the calling user.

Authorizations:
GSSAPIAuth

Responses

Response samples

Content type
application/json
{
  • "president": "user0",
  • "vice-president": "user1",
  • "sysadmin": "user2",
  • "treasurer": null
}

Update positions

Update members for each positions. Members not specified in the parameters will be removed from the position and unsubscribed from the exec's mailing list. New position holders will be subscribed to the mailing list.

+

Response samples

Content type
application/json
{
  • "vhosts": [
    ]
}

positions

Show current positions

Shows the list of positions and members holding them.

+
Authorizations:
GSSAPIAuth

Responses

Response samples

Content type
application/json
{
  • "president": "user0",
  • "vice-president": "user1",
  • "sysadmin": "user2",
  • "treasurer": null
}

Update positions

Update members for each positions. Members not specified in the parameters will be removed from the position and unsubscribed from the exec's mailing list. New position holders will be subscribed to the mailing list.

Authorizations:
GSSAPIAuth
Request Body schema: application/json

New position holders

property name*
string

Responses

Request samples

Content type
application/json
{
  • "president": "user0",
  • "vice-president": "user1",
  • "sysadmin": "user2",
  • "treasurer": null
}

Response samples

Content type
text/plain
{"status": "in progress", "operation": "update_positions_ldap"}
+

Request samples

Content type
application/json
{
  • "president": "user0",
  • "vice-president": "user1",
  • "sysadmin": "user2",
  • "treasurer": null
}

Response samples

Content type
text/plain
{"status": "in progress", "operation": "update_positions_ldap"}
 {"status": "in progress", "operation": "update_exec_group_ldap"}
 {"status": "in progress", "operation": "subscribe_to_mailing_list"}
 {"status": "completed", "result": "OK"}
-
+