diff --git a/.drone.yml b/.drone.yml index 4424b38..8557338 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,7 +10,7 @@ steps: # way to share system packages between steps commands: # install dependencies - - apt update && apt install -y libkrb5-dev libpq-dev python3-dev + - apt update && apt install -y libkrb5-dev libpq-dev python3-dev libaugeas0 - python3 -m venv venv - . venv/bin/activate - pip install -r dev-requirements.txt diff --git a/.drone/auth1-setup.sh b/.drone/auth1-setup.sh index d79bf69..db648eb 100755 --- a/.drone/auth1-setup.sh +++ b/.drone/auth1-setup.sh @@ -16,8 +16,6 @@ if [ -n "$CI" ]; then rm /tmp/hosts fi -export DEBIAN_FRONTEND=noninteractive -apt update apt install -y psmisc # If we don't do this then OpenLDAP uses a lot of RAM @@ -98,7 +96,6 @@ while true; do fi done -apt install -y netcat-openbsd # sync with phosphoric-acid nc -l 0.0.0.0 9000 & if [ -z "$CI" ]; then diff --git a/.drone/coffee-setup.sh b/.drone/coffee-setup.sh index 0773aa3..f347099 100755 --- a/.drone/coffee-setup.sh +++ b/.drone/coffee-setup.sh @@ -8,9 +8,6 @@ set -ex add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee add_fqdn_to_hosts $(get_ip_addr auth1) auth1 -export DEBIAN_FRONTEND=noninteractive -apt update - apt install --no-install-recommends -y default-mysql-server postgresql # MYSQL @@ -49,7 +46,6 @@ REVOKE ALL ON SCHEMA public FROM public; GRANT ALL ON SCHEMA public TO postgres; EOF" postgres -apt install -y netcat-openbsd if [ -z "$CI" ]; then auth_setup coffee fi diff --git a/.drone/common.sh b/.drone/common.sh index f59dbdd..aaf452a 100644 --- a/.drone/common.sh +++ b/.drone/common.sh @@ -23,6 +23,16 @@ cp .drone/k8s-authority.crt /etc/csc/k8s-authority.crt # openssl is actually already present in the python Docker image, # so we don't need to mock it out +# netcat is used for synchronization between the containers +export DEBIAN_FRONTEND=noninteractive +apt update +apt install -y netcat-openbsd +if [ "$(hostname)" != auth1 ]; then + # ceod uses Augeas, which is not installed by default in the Python + # Docker container + apt install -y libaugeas0 +fi + get_ip_addr() { getent hosts $1 | cut -d' ' -f1 } diff --git a/.drone/mail-setup.sh b/.drone/mail-setup.sh index 2751736..cbb0b78 100755 --- a/.drone/mail-setup.sh +++ b/.drone/mail-setup.sh @@ -14,9 +14,6 @@ python -m tests.MockSMTPServer & python -m tests.MockCloudStackServer & python -m tests.MockHarborServer & -export DEBIAN_FRONTEND=noninteractive -apt update -apt install -y netcat-openbsd auth_setup mail # for the VHostManager diff --git a/.drone/phosphoric-acid-setup.sh b/.drone/phosphoric-acid-setup.sh index 80c70cf..c862258 100755 --- a/.drone/phosphoric-acid-setup.sh +++ b/.drone/phosphoric-acid-setup.sh @@ -13,9 +13,6 @@ if [ -z "$CI" ]; then add_fqdn_to_hosts $(get_ip_addr mail) mail fi -export DEBIAN_FRONTEND=noninteractive -apt update -apt install -y netcat-openbsd auth_setup phosphoric-acid # initialize the skel directory diff --git a/README.md b/README.md index 4357256..2c5d828 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,7 @@ Docker containers instead, which are much easier to work with than the VM. First, make sure you create the virtualenv: ```sh -docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \ - sh -c 'python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt' +docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt' ``` Then bring up the containers: ```sh diff --git a/ceo/cli/entrypoint.py b/ceo/cli/entrypoint.py index 0f680b1..9a9a877 100644 --- a/ceo/cli/entrypoint.py +++ b/ceo/cli/entrypoint.py @@ -10,6 +10,7 @@ from .mailman import mailman from .cloud import cloud from .k8s import k8s from .registry import registry +from .webhosting import webhosting @click.group() @@ -27,3 +28,4 @@ cli.add_command(mailman) cli.add_command(cloud) cli.add_command(k8s) cli.add_command(registry) +cli.add_command(webhosting) diff --git a/ceo/cli/webhosting.py b/ceo/cli/webhosting.py new file mode 100644 index 0000000..dbcbd95 --- /dev/null +++ b/ceo/cli/webhosting.py @@ -0,0 +1,37 @@ +import click + +from ..utils import http_post +from .utils import handle_sync_response + + +@click.group(short_help='Manage websites hosted by the main CSC web server') +def webhosting(): + pass + + +@webhosting.command(short_help='Disable club sites with no active club reps') +@click.option('--dry-run', is_flag=True, default=False) +@click.option('--remove-inactive-club-reps', is_flag=True, default=False) +def disableclubsites(dry_run, remove_inactive_club_reps): + params = {} + if dry_run: + params['dry_run'] = 'true' + if remove_inactive_club_reps: + params['remove_inactive_club_reps'] = 'true' + if not dry_run: + click.confirm('Are you sure you want to disable the websites of clubs with no active club reps?', abort=True) + + resp = http_post('/api/webhosting/disableclubsites', params=params) + disabled_club_names = handle_sync_response(resp) + if len(disabled_club_names) == 0: + if dry_run: + click.echo('No websites would have been disabled.') + else: + click.echo('No websites were disabled.') + else: + if dry_run: + click.echo('The following club websites would have been disabled:') + else: + click.echo('The following club websites were disabled:') + for club_name in disabled_club_names: + click.echo(club_name) diff --git a/ceo/utils.py b/ceo/utils.py index 00a86a8..57c62ba 100644 --- a/ceo/utils.py +++ b/ceo/utils.py @@ -21,6 +21,8 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response: host = cfg.get('ceod_mailman_host') elif path.startswith('/api/cloud'): host = cfg.get('ceod_cloud_host') + elif path.startswith('/api/webhosting'): + host = cfg.get('ceod_webhosting_host') else: host = cfg.get('ceod_admin_host') return client.request( diff --git a/ceo_common/interfaces/IClubWebHostingService.py b/ceo_common/interfaces/IClubWebHostingService.py new file mode 100644 index 0000000..d44dc95 --- /dev/null +++ b/ceo_common/interfaces/IClubWebHostingService.py @@ -0,0 +1,45 @@ +from typing import List + +from zope.interface import Interface + + +class IClubWebHostingService(Interface): + """ + Performs operations on the configuration files of club websites hosted + by the CSC. + """ + + def begin_transaction(): + """ + Performs any steps necessary to query the website config files. + This must be called BEFORE any of the methods below, using a context + expression like so: + + with club_site_mgr.begin_transaction(): + club_site_mgr.disable_club_site('club1') + ... + club_site_mgr.commit() + """ + + def commit(): + """ + Writes the config file changes to disk. This must be called at the + end of the context expression. + """ + + def disable_club_site(club_name: str): + """ + Disables the site for the club. Note that commit() must still be + called to commit this change. + """ + + def disable_sites_for_inactive_clubs( + dry_run: bool = False, + remove_inactive_club_reps: bool = False, + ) -> List[str]: + """ + Disables sites for inactive clubs. If remove_inactive_club_reps is set + to True, then inactive club reps will be removed from club groups. + The list of clubs whose sites were disabled (or would have been + disabled, if dry_run is True) is returned. + """ diff --git a/ceo_common/interfaces/ILDAPService.py b/ceo_common/interfaces/ILDAPService.py index 4b928ed..09738fb 100644 --- a/ceo_common/interfaces/ILDAPService.py +++ b/ceo_common/interfaces/ILDAPService.py @@ -9,10 +9,10 @@ from .IGroup import IGroup class ILDAPService(Interface): """An interface to the LDAP database.""" - def uid_to_dn(self, uid: str) -> str: + def uid_to_dn(uid: str) -> str: """Get the LDAP DN for the user with this UID.""" - def group_cn_to_dn(self, cn: str) -> str: + def group_cn_to_dn(cn: str) -> str: """Get the LDAP DN for the group with this CN.""" def get_user(username: str) -> IUser: @@ -24,7 +24,7 @@ class ILDAPService(Interface): Useful for displaying a list of users in a compact way. """ - def get_users_with_positions(self) -> List[IUser]: + def get_users_with_positions() -> List[IUser]: """Retrieve users who have a non-empty position attribute.""" def add_user(user: IUser): @@ -36,7 +36,7 @@ class ILDAPService(Interface): def remove_user(user: IUser): """Remove this user from the database.""" - def get_group(cn: str, is_club: bool = False) -> IGroup: + def get_group(cn: str) -> IGroup: """Retrieve the group with the given cn (Unix group name).""" def add_group(group: IGroup): @@ -88,15 +88,33 @@ class ILDAPService(Interface): described above. """ - def get_nonflagged_expired_users(self) -> List[IUser]: + def get_nonflagged_expired_users() -> List[IUser]: """ Retrieves members whose term or nonMemberTerm does not contain the current or the last term. """ - def get_expiring_users(self) -> List[IUser]: + def get_expiring_users() -> List[IUser]: """ Retrieves members whose membership will expire in less than a month. This is used to send membership renewal reminders at the beginning of a term, during the one-month grace period. """ + + def get_clubs() -> List[IGroup]: + """ + Retrieves all clubs. + """ + + # couldn't import the Term class from ceo_common.model due to some + # circular import issue... + def get_club_reps_non_member_terms(club_reps: List[str]) -> Dict[str, List['Term']]: # noqa: F821 + """ + Retrieves the non-member terms for the given club reps. + e.g. + { + "user1": [w2022, s2022], + "user2": [s2022], + ... + } + """ diff --git a/ceo_common/interfaces/IMailService.py b/ceo_common/interfaces/IMailService.py index 53b2125..0e52cab 100644 --- a/ceo_common/interfaces/IMailService.py +++ b/ceo_common/interfaces/IMailService.py @@ -24,20 +24,26 @@ class IMailService(Interface): during the transaction. """ - def send_membership_renewal_reminder(self, user: IUser): + def send_membership_renewal_reminder(user: IUser): """ Send a reminder to the user that their membership will expire soon. """ - def send_cloud_account_will_be_deleted_message(self, user: IUser): + def send_cloud_account_will_be_deleted_message(user: IUser): """ Send a warning message to the user that their cloud resources will be deleted if they do not renew their membership. """ - def send_cloud_account_has_been_deleted_message(self, user: IUser): + def send_cloud_account_has_been_deleted_message(user: IUser): """ Send a message to the user that their cloud resources have been deleted. """ + + def send_club_website_has_been_disabled_message(club: str, address: str): + """ + Send a message to the club stating that their website has been + disabled. + """ diff --git a/ceo_common/interfaces/__init__.py b/ceo_common/interfaces/__init__.py index 25c0ffd..b819459 100644 --- a/ceo_common/interfaces/__init__.py +++ b/ceo_common/interfaces/__init__.py @@ -14,3 +14,4 @@ from .IDatabaseService import IDatabaseService from .IVHostManager import IVHostManager from .IKubernetesService import IKubernetesService from .IContainerRegistryService import IContainerRegistryService +from .IClubWebHostingService import IClubWebHostingService diff --git a/ceo_common/model/HTTPClient.py b/ceo_common/model/HTTPClient.py index 6a38e23..1348a9f 100644 --- a/ceo_common/model/HTTPClient.py +++ b/ceo_common/model/HTTPClient.py @@ -43,7 +43,7 @@ class HTTPClient: if flask.has_request_context() and 'client_token' in g: # This is reached when we are the server and the client has # forwarded their credentials to us. - spnego_kwargs['creds'] = gssapi.Credentials(token=flask.g.client_token) + spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token) elif delegate: # This is reached when we are the client and we want to # forward our credentials to the server. diff --git a/ceod/api/app_factory.py b/ceod/api/app_factory.py index 84bf7c5..221c3d5 100644 --- a/ceod/api/app_factory.py +++ b/ceod/api/app_factory.py @@ -9,13 +9,13 @@ from .error_handlers import register_error_handlers from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \ IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \ ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager, \ - IContainerRegistryService + IContainerRegistryService, IClubWebHostingService from ceo_common.model import Config, HTTPClient, RemoteMailmanService from ceod.api.spnego import init_spnego from ceod.model import KerberosService, LDAPService, FileService, \ MailmanService, MailService, UWLDAPService, CloudStackService, \ CloudResourceManager, KubernetesService, VHostManager, \ - ContainerRegistryService + ContainerRegistryService, ClubWebHostingService from ceod.db import MySQLService, PostgreSQLService @@ -49,6 +49,10 @@ def create_app(flask_config={}): from ceod.api import cloud app.register_blueprint(cloud.bp, url_prefix='/api/cloud') + if hostname == cfg.get('ceod_webhosting_host'): + from ceod.api import webhosting + app.register_blueprint(webhosting.bp, url_prefix='/api/webhosting') + from ceod.api import groups app.register_blueprint(groups.bp, url_prefix='/api/groups') @@ -112,6 +116,11 @@ def register_services(app): uwldap_srv = UWLDAPService() component.provideUtility(uwldap_srv, IUWLDAPService) + # ClubWebHostingService + if hostname == cfg.get('ceod_webhosting_host'): + webhosting_srv = ClubWebHostingService() + component.provideUtility(webhosting_srv, IClubWebHostingService) + # MySQLService, PostgreSQLService if hostname == cfg.get('ceod_database_host'): mysql_srv = MySQLService() diff --git a/ceod/api/webhosting.py b/ceod/api/webhosting.py new file mode 100644 index 0000000..7b476ef --- /dev/null +++ b/ceod/api/webhosting.py @@ -0,0 +1,22 @@ +from flask import Blueprint, request +from flask.json import jsonify +from zope import component + +from .utils import authz_restrict_to_syscom, is_truthy +from ceo_common.interfaces import IClubWebHostingService + + +bp = Blueprint('webhosting', __name__) + + +@bp.route('/disableclubsites', methods=['POST']) +@authz_restrict_to_syscom +def expire_club_sites(): + dry_run = is_truthy(request.args.get('dry_run', 'false')) + remove_inactive_club_reps = is_truthy(request.args.get('remove_inactive_club_reps', 'false')) + webhosting_srv = component.getUtility(IClubWebHostingService) + disabled_sites = webhosting_srv.disable_sites_for_inactive_clubs( + dry_run=dry_run, + remove_inactive_club_reps=remove_inactive_club_reps, + ) + return jsonify(disabled_sites) diff --git a/ceod/model/ClubWebHostingService.py b/ceod/model/ClubWebHostingService.py new file mode 100644 index 0000000..c57b4c0 --- /dev/null +++ b/ceod/model/ClubWebHostingService.py @@ -0,0 +1,250 @@ +from collections import defaultdict +import contextlib +import glob +import os +import re +import subprocess +from threading import Lock +import traceback +from typing import List + +from augeas import Augeas +from zope import component +from zope.interface import implementer + +from ceo_common.interfaces import ILDAPService, IMailService, \ + IClubWebHostingService, IConfig +from ceo_common.logger_factory import logger_factory +from ceo_common.model import Term + +# NOTE: We are assuming that the name of a club is the same as its home +# directory (under /users). While this rule is not enforced, it has been true +# up until now. +# FIXME: don't assume this +APACHE_USERDIR_RE = re.compile(r'^/users/(?P[0-9a-z-]+)/www/?$') + +# This is where all the directives for disabled clubs are stored. +APACHE_DISABLED_CLUBS_FILE = 'conf-available/disable-club.conf' +# This is the file which contains the snippet to disable a single club. +APACHE_DISABLING_SNIPPET_FILE = 'snippets/disable-club.conf' +# This is the maximum number of consecutive terms for which a club is allowed +# to have no active club reps. After this many terms have passed, the club's +# website may be disabled. +MAX_TERMS_WITH_NO_ACTIVE_CLUB_REPS = 3 + +logger = logger_factory(__name__) + + +def is_a_disabling_snippet(snippet_path: str) -> bool: + return snippet_path.startswith('snippets/disable-') + + +@implementer(IClubWebHostingService) +class ClubWebHostingService: + def __init__(self, augeas_root='/'): + cfg = component.getUtility(IConfig) + self.aug_root = augeas_root + self.apache_dir = os.path.join(augeas_root, 'etc/apache2') + self.sites_available_dir = os.path.join(self.apache_dir, 'sites-available') + self.conf_available_dir = os.path.join(self.apache_dir, 'conf-available') + self.clubs_home = cfg.get('clubs_home') + self.aug = None + self.clubs = None + self.made_at_least_one_change = False + self.lock = Lock() + + @contextlib.contextmanager + def _hold_lock(self): + try: + self.lock.acquire() + yield + finally: + self.lock.release() + + @contextlib.contextmanager + def begin_transaction(self): + with self._hold_lock(): + try: + self.aug = Augeas(self.aug_root) + self.clubs = defaultdict(lambda: {'disabled': False, 'email': None}) + self._get_club_emails() + self._get_disabled_sites() + yield + finally: + if self.aug is not None: + self.aug.close() + self.aug = None + self.clubs = None + self.made_at_least_one_change = False + + def _run(self, args: List[str], **kwargs): + subprocess.run(args, check=True, **kwargs) + + def _reload_web_server(self): + logger.debug('Reloading Apache') + self._run(['systemctl', 'reload', 'apache2']) + + # This requires the APACHE_CONFIG_CRON environment variable to be + # set to 1 (e.g. in a systemd drop-in) + # See /etc/apache2/.git/hooks/pre-commit on caffeine + def _git_commit(self): + if not os.path.isdir(os.path.join(self.apache_dir, '.git')): + logger.debug('No git folder found in Apache directory') + return + logger.debug('Committing changes to git repository') + self._run(['git', 'add', APACHE_DISABLED_CLUBS_FILE], cwd=self.apache_dir) + self._run(['git', 'commit', '-m', '[ceo] disable club websites'], cwd=self.apache_dir) + + def commit(self): + if not self.made_at_least_one_change: + return + logger.debug('Saving Augeas changes') + self.aug.save() + self._reload_web_server() + self._git_commit() + + def _get_club_emails(self): + config_file_paths = glob.glob(self.sites_available_dir + '/club-*.conf') + for config_file_path in config_file_paths: + club_name = None + club_email = None + filename = os.path.basename(config_file_path) + directive_paths = self.aug.match(f'/files/etc/apache2/sites-available/{filename}/VirtualHost/directive') + for directive_path in directive_paths: + directive = self.aug.get(directive_path) + directive_value = self.aug.get(directive_path + '/arg') + if directive == 'DocumentRoot': + match = APACHE_USERDIR_RE.match(directive_value) + if match is not None: + club_name = match.group('club_name') + elif directive == 'ServerAdmin': + club_email = directive_value + if club_name is not None: + self.clubs[club_name]['email'] = club_email + + def _get_disabled_sites(self): + config_file_paths = glob.glob(self.conf_available_dir + '/disable-*.conf') + for config_file_path in config_file_paths: + filename = os.path.basename(config_file_path) + directory_paths = self.aug.match(f'/files/etc/apache2/conf-available/{filename}/Directory') + for directory_path in directory_paths: + directory = self.aug.get(directory_path + '/arg') + match = APACHE_USERDIR_RE.match(directory) + if match is None: + continue + club_name = match.group('club_name') + directive_paths = self.aug.match(directory_path + '/directive') + for directive_path in directive_paths: + if self.aug.get(directive_path) != 'Include': + continue + included_file = self.aug.get(directive_path + '/arg') + if is_a_disabling_snippet(included_file): + self.clubs[club_name]['disabled'] = True + + def disable_club_site(self, club_name: str): + logger.info(f'Disabling website for {club_name}') + directory_path = f'/files/etc/apache2/{APACHE_DISABLED_CLUBS_FILE}/Directory' + num_directories = len(self.aug.match(directory_path)) + # Create a new section + directory_path += '[%d]' % (num_directories + 1) + # FIXME: use self.clubs_home here instead (need to update unit tests) + self.aug.set(directory_path + '/arg', f'/users/{club_name}/www') + self.aug.set(directory_path + '/directive', 'Include') + self.aug.set(directory_path + '/directive/arg', APACHE_DISABLING_SNIPPET_FILE) + + self.clubs[club_name]['disabled'] = True + self.made_at_least_one_change = True + + def _site_uses_php(self, club_name: str) -> bool: + www = f'{self.clubs_home}/{club_name}/www' + if os.path.isdir(www): + # We're just going to look one level deep; that should be good enough. + for filename in os.listdir(www): + filepath = os.path.join(www, filename) + if os.path.isfile(filepath) and filename.endswith('.php'): + return True + return False + + # This method needs to be called from within a transaction (uses self.clubs) + def _need_to_disable_inactive_club_site(self, club_name: str) -> bool: + if self.clubs[club_name]['disabled']: + # already disabled - nothing to do + return False + if not self._site_uses_php(club_name): + # These are the only sites which are actually a security concern + # to us - it's OK for a purely static website to be unmaintained. + return False + return True + + def disable_sites_for_inactive_clubs( + self, + dry_run: bool = False, + remove_inactive_club_reps: bool = False, + ) -> List[str]: + ldap_srv = component.getUtility(ILDAPService) + mail_srv = component.getUtility(IMailService) + all_clubs = ldap_srv.get_clubs() + club_rep_uids = list({ + member + for club in all_clubs + for member in club.members + }) + club_reps = ldap_srv.get_club_reps_non_member_terms(club_rep_uids) + # If a club rep's last non-member term is before cutoff_term, then + # they are considered inactive + cutoff_term = Term.current() - MAX_TERMS_WITH_NO_ACTIVE_CLUB_REPS + active_club_reps = { + club_rep + for club_rep, non_member_terms in club_reps.items() + if len(non_member_terms) > 0 and max(non_member_terms) >= cutoff_term + } + # a club is inactive if it does not have at least one active club rep + inactive_clubs = [ + club + for club in all_clubs + if not any(map(lambda member: member in active_club_reps, club.members)) + ] + + # STEP 1: update the Apache configs + with self.begin_transaction(): + clubs_to_disable = [ + club.cn + for club in inactive_clubs + if self._need_to_disable_inactive_club_site(club.cn) + ] + if dry_run: + return clubs_to_disable + + for club_name in clubs_to_disable: + self.disable_club_site(club_name) + self.commit() + # self.clubs is set to None once the transaction closes, so make a copy now + clubs_info = self.clubs.copy() + + # STEP 2: send emails to clubs whose websites were disabled + clubs_who_were_not_notified = set() + for club_name in clubs_to_disable: + address = clubs_info['email'] + if address is None: + clubs_who_were_not_notified.add(club_name) + continue + try: + mail_srv.send_club_website_has_been_disabled_message(club_name, address) + except Exception: + trace = traceback.format_exc() + logger.error(f'Failed to send email to {address}:\n{trace}') + clubs_who_were_not_notified.add(club_name) + + # STEP 3: remove inactive club reps from Unix groups + if remove_inactive_club_reps: + for club in all_clubs: + if club.cn in clubs_who_were_not_notified: + continue + # club.members gets modified after calling club.remove_member(), + # so we need to make a copy + members = club.members.copy() + for member in members: + if member not in active_club_reps: + logger.info(f'Removing {member} from {club.cn}') + club.remove_member(member) + return clubs_to_disable diff --git a/ceod/model/LDAPService.py b/ceod/model/LDAPService.py index 8c54c65..74c79f3 100644 --- a/ceod/model/LDAPService.py +++ b/ceod/model/LDAPService.py @@ -344,3 +344,37 @@ class LDAPService: conn.modify(self.uid_to_dn(uid), changes) return users_to_change + + def _get_club_uids(self, conn: ldap3.Connection) -> List[str]: + conn.search(self.ldap_users_base, '(objectClass=club)', attributes=['uid']) + return [entry.uid.value for entry in conn.entries] + + def get_clubs(self) -> List[IGroup]: + batch_size = 100 + conn = self._get_ldap_conn() + club_uids = self._get_club_uids(conn) + clubs = [] + for i in range(0, len(club_uids), batch_size): + club_uids_slice = club_uids[i:i + batch_size] + filter = '(|' + ''.join([f'(cn={uid})' for uid in club_uids_slice]) + ')' + conn.search(self.ldap_groups_base, filter, attributes=ldap3.ALL_ATTRIBUTES) + for entry in conn.entries: + clubs.append(Group.deserialize_from_ldap(entry)) + return clubs + + def get_club_reps_non_member_terms(self, club_reps: List[str]) -> Dict[str, List[Term]]: + batch_size = 100 + conn = self._get_ldap_conn() + club_reps_terms = {} + for i in range(0, len(club_reps), batch_size): + club_reps_slice = club_reps[i:i + batch_size] + filter = '(|' + ''.join([f'(uid={uid})' for uid in club_reps_slice]) + ')' + conn.search(self.ldap_users_base, filter, attributes=['uid', 'nonMemberTerm']) + for entry in conn.entries: + uid = entry.uid.value + if 'nonMemberTerm' in entry.entry_attributes: + non_member_terms = list(map(Term, entry.nonMemberTerm.values)) + else: + non_member_terms = [] + club_reps_terms[uid] = non_member_terms + return club_reps_terms diff --git a/ceod/model/MailService.py b/ceod/model/MailService.py index 7439336..3a015de 100644 --- a/ceod/model/MailService.py +++ b/ceod/model/MailService.py @@ -137,3 +137,17 @@ class MailService: }, body, ) + + def send_club_website_has_been_disabled_message(self, club: str, address: str): + template = self.jinja_env.get_template('club_website_has_been_disabled.j2') + body = template.render(club=club) + self.send( + f'Computer Science Club ', + f'{club} <{address}>', + { + 'Subject': 'Your club website has been disabled', + 'Cc': f'root@{self.base_domain}', + 'Reply-To': f'syscom@{self.base_domain}', + }, + body, + ) diff --git a/ceod/model/__init__.py b/ceod/model/__init__.py index a8b16ec..dd54b8e 100644 --- a/ceod/model/__init__.py +++ b/ceod/model/__init__.py @@ -13,3 +13,4 @@ from .MailmanService import MailmanService from .VHostManager import VHostManager from .KubernetesService import KubernetesService from .ContainerRegistryService import ContainerRegistryService +from .ClubWebHostingService import ClubWebHostingService diff --git a/ceod/model/templates/club_website_has_been_disabled.j2 b/ceod/model/templates/club_website_has_been_disabled.j2 new file mode 100644 index 0000000..fada926 --- /dev/null +++ b/ceod/model/templates/club_website_has_been_disabled.j2 @@ -0,0 +1,20 @@ +Hello {{ club }} representatives, + +This is an automated message from ceo, the CSC Electronic Office. + +Since your club has not had any active CSC club reps for 3 or more consecutive +terms, your club's website has been disabled. Websites require active +maintenance and software upgrades, and an unmaintained website poses a +security risk to CSC and to the University of Waterloo. + +If you wish for your site to be re-enabled, please register at least +one club rep with CSC and, if necessary, upgrade the software running +your website. A club rep membership is free. You can find instructions +on our website for how to obtain a membership: + +https://csclub.uwaterloo.ca/get-involved/ + +If you have any questions or concerns, please contact syscom@csclub.uwaterloo.ca. + +Sincerely, +ceo diff --git a/debian/control b/debian/control index 8918721..a09676e 100644 --- a/debian/control +++ b/debian/control @@ -11,6 +11,7 @@ Build-Depends: debhelper (>= 12.1.1), python3-venv (>= 3.7), libkrb5-dev (>= 1.17), libpq-dev (>= 11.13), + libaugeas0 (>= 1.11), scdoc (>= 1.9) Package: ceo-common @@ -19,6 +20,7 @@ Depends: python3 (>= 3.7), krb5-user (>= 1.17), libkrb5-3 (>= 1.17), libpq5 (>= 11.13), + libaugeas0 (>= 1.11), ${python3:Depends}, ${misc:Depends} Description: CSC Electronic Office common files diff --git a/etc/ceo.ini b/etc/ceo.ini index c0337dd..c3a5e95 100644 --- a/etc/ceo.ini +++ b/etc/ceo.ini @@ -7,6 +7,8 @@ uw_domain = uwaterloo.ca admin_host = phosphoric-acid # this is the host with root access to the databases database_host = caffeine +# this is the host where clubs' websites are hosted +webhosting_host = caffeine # this is the host which can make API requests to Mailman mailman_host = mailman # this is the host running a CloudStack management server diff --git a/etc/ceod.ini b/etc/ceod.ini index e62cdf9..dc0e048 100644 --- a/etc/ceod.ini +++ b/etc/ceod.ini @@ -8,6 +8,8 @@ admin_host = phosphoric-acid fs_root_host = phosphoric-acid # this is the host with root access to the databases database_host = caffeine +# this is the host where clubs' websites are hosted +webhosting_host = caffeine # this is the host which can make API requests to Mailman mailman_host = mailman # this is the host which is running a CloudStack management server diff --git a/requirements.txt b/requirements.txt index 2be9a85..b5a2642 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,9 +7,10 @@ Jinja2==3.1.2 ldap3==2.9.1 mysql-connector-python==8.0.26 psycopg2==2.9.1 +python-augeas==1.1.0 requests==2.26.0 requests-gssapi==1.2.3 urwid==2.1.2 -werkzeug==2.1.2 +Werkzeug==2.1.2 zope.component==5.0.1 zope.interface==5.4.0 diff --git a/tests/ceo/cli/test_webhosting.py b/tests/ceo/cli/test_webhosting.py new file mode 100644 index 0000000..f4817c7 --- /dev/null +++ b/tests/ceo/cli/test_webhosting.py @@ -0,0 +1,62 @@ +from click.testing import CliRunner + +from ceo.cli import cli +from ceo_common.model import Term +from tests.utils import ( + create_php_file_for_club, + reset_disable_club_conf, + set_datetime_in_app_process, + restore_datetime_in_app_process, +) + + +def test_disable_club_sites( + cfg, cli_setup, app_process, webhosting_srv, webhosting_srv_resources, + new_club_gen, new_user_gen, g_admin_ctx, ldap_srv_session, +): + runner = CliRunner() + term = Term.current() + clubs_home = cfg.get('clubs_home') + with new_club_gen() as group, new_user_gen() as user: + create_php_file_for_club(clubs_home, group.cn) + user.add_non_member_terms([str(Term.current())]) + group.add_member(user.uid) + + set_datetime_in_app_process(app_process, (term + 4).to_datetime()) + + result = runner.invoke(cli, ['webhosting', 'disableclubsites', '--dry-run']) + assert result.exit_code == 0 + assert result.output.endswith( + 'The following club websites would have been disabled:\n' + f'{group.cn}\n' + ) + + result = runner.invoke(cli, ['webhosting', 'disableclubsites'], input='y\n') + assert result.exit_code == 0 + assert result.output.endswith( + 'The following club websites were disabled:\n' + f'{group.cn}\n' + ) + + with g_admin_ctx(): + group = ldap_srv_session.get_group(group.cn) + assert group.members == [user.uid] + + reset_disable_club_conf(webhosting_srv) + + result = runner.invoke( + cli, + ['webhosting', 'disableclubsites', '--remove-inactive-club-reps'], + input='y\n' + ) + assert result.exit_code == 0 + assert result.output.endswith( + 'The following club websites were disabled:\n' + f'{group.cn}\n' + ) + + with g_admin_ctx(): + group = ldap_srv_session.get_group(group.cn) + assert group.members == [] + + restore_datetime_in_app_process(app_process) diff --git a/tests/ceo_dev.ini b/tests/ceo_dev.ini index 3291452..12ad1e3 100644 --- a/tests/ceo_dev.ini +++ b/tests/ceo_dev.ini @@ -6,6 +6,7 @@ uw_domain = uwaterloo.internal # this is the host with the ceod/admin Kerberos key admin_host = phosphoric-acid database_host = coffee +webhosting_host = coffee mailman_host = mail cloud_host = mail use_https = false diff --git a/tests/ceod/api/test_webhosting.py b/tests/ceod/api/test_webhosting.py new file mode 100644 index 0000000..db39f19 --- /dev/null +++ b/tests/ceod/api/test_webhosting.py @@ -0,0 +1,43 @@ +import datetime +from unittest.mock import patch + +from ceo_common.model import Term +import ceo_common.utils +from tests.utils import create_php_file_for_club, reset_disable_club_conf + + +def test_disable_club_sites( + cfg, client, webhosting_srv, webhosting_srv_resources, new_club_gen, + new_user_gen, g_admin_ctx, ldap_srv_session, +): + term = Term.current() + clubs_home = cfg.get('clubs_home') + with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock: + now_mock.return_value = datetime.datetime.now() + with new_club_gen() as group, new_user_gen() as user: + create_php_file_for_club(clubs_home, group.cn) + user.add_non_member_terms([str(Term.current())]) + group.add_member(user.uid) + + now_mock.return_value = (term + 4).to_datetime() + status, data = client.post('/api/webhosting/disableclubsites?dry_run=true') + assert status == 200 + assert data == [group.cn] + + status, data = client.post('/api/webhosting/disableclubsites') + assert status == 200 + assert data == [group.cn] + + with g_admin_ctx(): + group = ldap_srv_session.get_group(group.cn) + assert group.members == [user.uid] + + reset_disable_club_conf(webhosting_srv) + + status, data = client.post('/api/webhosting/disableclubsites?remove_inactive_club_reps=true') + assert status == 200 + assert data == [group.cn] + + with g_admin_ctx(): + group = ldap_srv_session.get_group(group.cn) + assert group.members == [] diff --git a/tests/ceod/model/test_webhosting.py b/tests/ceod/model/test_webhosting.py new file mode 100644 index 0000000..e359c79 --- /dev/null +++ b/tests/ceod/model/test_webhosting.py @@ -0,0 +1,163 @@ +import datetime +import os +import re +import subprocess +from unittest.mock import patch + +from ceo_common.model import Term +import ceo_common.utils +from tests.utils import create_php_file_for_club + + +def create_website_config_for_club(sites_available_dir, cn, filename=None): + if filename is None: + filename = f'club-{cn}.conf' + filepath = os.path.join(sites_available_dir, filename) + with open(filepath, 'w') as fo: + fo.write(f""" + + ServerName {cn}.uwaterloo.internal + ServerAdmin {cn}@{cn}.uwaterloo.internal + DocumentRoot /users/{cn}/www/ + + """) + + +def get_enabled_sites(webhosting_srv): + return [ + club_name + for club_name, club_info in webhosting_srv.clubs.items() + if not club_info['disabled'] + ] + + +def test_disable_club_sites(webhosting_srv, webhosting_srv_resources, new_club_gen): + sites_available_dir = webhosting_srv.sites_available_dir + with new_club_gen() as group1, new_club_gen() as group2: + create_website_config_for_club(sites_available_dir, group1.cn) + create_website_config_for_club(sites_available_dir, group2.cn, 'club-somerandomname.conf') + with webhosting_srv.begin_transaction(): + enabled_clubs = get_enabled_sites(webhosting_srv) + assert sorted(enabled_clubs) == [group1.cn, group2.cn] + webhosting_srv.disable_club_site(group1.cn) + # Make sure that if we don't call commit(), nothing gets saved + with webhosting_srv.begin_transaction(): + enabled_clubs = get_enabled_sites(webhosting_srv) + assert sorted(enabled_clubs) == [group1.cn, group2.cn] + webhosting_srv.disable_club_site(group1.cn) + webhosting_srv.commit() + # Now that we committed the changes, they should be persistent + with webhosting_srv.begin_transaction(): + enabled_clubs = get_enabled_sites(webhosting_srv) + assert enabled_clubs == [group2.cn] + + +def test_disable_inactive_club_sites( + cfg, webhosting_srv, webhosting_srv_resources, g_admin_ctx, new_club_gen, + new_user_gen, mock_mail_server, +): + sites_available_dir = webhosting_srv.sites_available_dir + term = Term.current() + clubs_home = cfg.get('clubs_home') + with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock: + now_mock.return_value = datetime.datetime.now() + with new_club_gen() as group1, \ + new_club_gen() as group2, \ + new_user_gen() as user1, \ + new_user_gen() as user2: + create_website_config_for_club(sites_available_dir, group1.cn) + create_website_config_for_club(sites_available_dir, group2.cn) + create_php_file_for_club(clubs_home, group1.cn) + with g_admin_ctx(): + # group1 has no club reps so it should be disabled + # group2 has no club reps but it doesn't use PHP + assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [group1.cn] + + user1.add_non_member_terms([str(Term.current())]) + group1.add_member(user1.uid) + with g_admin_ctx(): + # group1 has an active club rep, so it shouldn't be disabled anymore + assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [] + + now_mock.return_value = (term + 3).to_datetime() + with g_admin_ctx(): + # club reps are allowed to be inactive for up to 3 terms + assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [] + + now_mock.return_value = (term + 4).to_datetime() + with g_admin_ctx(): + # club site should be disabled now that club rep is inactive + assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [group1.cn] + + user2.add_non_member_terms([str(Term.current())]) + group1.add_member(user2.uid) + with g_admin_ctx(): + # group1 has a new club rep, so it shouldn't be disabled anymore + assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [] + + create_php_file_for_club(clubs_home, group2.cn) + group2.add_member(user2.uid) + with g_admin_ctx(): + # user2 is an active club rep for both group1 and group2 + assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [] + + now_mock.return_value = (term + 8).to_datetime() + mock_mail_server.messages.clear() + subprocess.run(['git', 'init'], cwd=webhosting_srv.apache_dir, check=True) + with g_admin_ctx(): + disabled_sites = webhosting_srv.disable_sites_for_inactive_clubs() + # user2 expired and both sites use PHP, so they should both be disabled + assert sorted(disabled_sites) == [group1.cn, group2.cn] + # since each club had a ServerAdmin directive, they should both have received + # notification emails + assert len(mock_mail_server.messages) == 2 + with open(webhosting_srv.conf_available_dir + '/disable-club.conf') as fi: + disable_club_conf_content = fi.read() + print(disable_club_conf_content) + for group in [group1, group2]: + pat = re.compile( + ( + rf'^\n' + r'\s*Include snippets/disable-club\.conf\n' + r'$' + ), + re.MULTILINE + ) + assert pat.search(disable_club_conf_content) is not None + + with g_admin_ctx(): + # Club sites should only be disabled once + assert webhosting_srv.disable_sites_for_inactive_clubs() == [] + + mock_mail_server.messages.clear() + + +def test_remove_inactive_club_reps( + cfg, webhosting_srv, webhosting_srv_resources, g_admin_ctx, new_club_gen, + new_user_gen, ldap_srv_session, +): + term = Term.current() + clubs_home = cfg.get('clubs_home') + with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock: + now_mock.return_value = datetime.datetime.now() + with new_club_gen() as group, \ + new_user_gen() as user1, \ + new_user_gen() as user2: + create_php_file_for_club(clubs_home, group.cn) + + for user in [user1, user2]: + user.add_non_member_terms([str(Term.current())]) + group.add_member(user.uid) + now_mock.return_value = (term + 4).to_datetime() + with g_admin_ctx(): + webhosting_srv.disable_sites_for_inactive_clubs(remove_inactive_club_reps=True) + group = ldap_srv_session.get_group(group.cn) + assert group.members == [] + + # Make sure that inactive club reps get removed even if the site + # has already been disabled + group.add_member(user1.uid) + with g_admin_ctx(): + webhosting_srv.disable_sites_for_inactive_clubs(remove_inactive_club_reps=True) + group = ldap_srv_session.get_group(group.cn) + assert group.members == [] diff --git a/tests/ceod_dev.ini b/tests/ceod_dev.ini index e6941c6..7404777 100644 --- a/tests/ceod_dev.ini +++ b/tests/ceod_dev.ini @@ -8,6 +8,8 @@ admin_host = phosphoric-acid fs_root_host = phosphoric-acid mailman_host = mail database_host = coffee +# this is the host where clubs' websites are hosted +webhosting_host = coffee cloud_host = mail use_https = false port = 9987 diff --git a/tests/ceod_test_local.ini b/tests/ceod_test_local.ini index e933a92..8e78cef 100644 --- a/tests/ceod_test_local.ini +++ b/tests/ceod_test_local.ini @@ -9,6 +9,7 @@ fs_root_host = phosphoric-acid mailman_host = phosphoric-acid database_host = phosphoric-acid cloud_host = phosphoric-acid +webhosting_host = phosphoric-acid use_https = false port = 9988 diff --git a/tests/conftest.py b/tests/conftest.py index 599e2fc..224b742 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ import shutil import subprocess from subprocess import DEVNULL import sys +import tempfile import threading import time from unittest.mock import Mock @@ -24,11 +25,12 @@ from .utils import ( # noqa: F401 gssapi_token_ctx, ccache_cleanup, mocks_for_create_user_ctx, + reset_disable_club_conf, ) from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \ IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \ IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \ - ICloudResourceManager, IContainerRegistryService + ICloudResourceManager, IContainerRegistryService, IClubWebHostingService from ceo_common.model import Config, HTTPClient, Term import ceo_common.utils from ceod.api import create_app @@ -36,7 +38,7 @@ from ceod.db import MySQLService, PostgreSQLService from ceod.model import KerberosService, LDAPService, FileService, User, \ MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \ CloudStackService, KubernetesService, VHostManager, CloudResourceManager, \ - ContainerRegistryService + ContainerRegistryService, ClubWebHostingService from .MockSMTPServer import MockSMTPServer from .MockMailmanServer import MockMailmanServer from .MockCloudStackServer import MockCloudStackServer @@ -132,12 +134,12 @@ def g_syscom(app): """ with gssapi_token_ctx('ctdalek') as token, app.app_context(): try: - flask.g.sasl_user = 'ctdalek' + flask.g.auth_user = 'ctdalek' flask.g.client_token = token yield finally: flask.g.pop('client_token') - flask.g.pop('sasl_user') + flask.g.pop('auth_user') @pytest.fixture(scope='session') @@ -237,6 +239,14 @@ def uwldap_srv(cfg, ldap_conn): delete_subtree(conn, base_dn) +@pytest.fixture(scope='session') +def webhosting_srv(): + with tempfile.TemporaryDirectory() as tmpdir: + srv = ClubWebHostingService(tmpdir) + component.getGlobalSiteManager().registerUtility(srv, IClubWebHostingService) + yield srv + + @pytest.fixture(scope='session') def mock_mail_server(): mock_server = MockSMTPServer() @@ -343,6 +353,7 @@ def app( k8s_srv, registry_srv, cloud_mgr, + webhosting_srv, ): app = create_app({'TESTING': True}) return app @@ -442,6 +453,31 @@ def new_user(new_user_gen): yield user +@pytest.fixture +def new_club_gen(client, g_admin_ctx, ldap_srv_session): # noqa: F811 + new_club_id_counter = 31001 + + @contextlib.contextmanager + def wrapper(): + nonlocal new_club_id_counter + cn = 'test' + str(new_club_id_counter) + new_club_id_counter += 1 + status, data = client.post('/api/groups', json={ + 'cn': cn, + 'description': 'Test ' + str(new_club_id_counter), + }) + assert status == 200 + assert data[-1]['status'] == 'completed' + with g_admin_ctx(): + group = ldap_srv_session.get_group(cn) + yield group + status, data = client.delete(f'/api/groups/{cn}') + assert status == 200 + assert data[-1]['status'] == 'completed' + + return wrapper + + @pytest.fixture def simple_group(): return Group( @@ -507,6 +543,17 @@ def uwldap_user(cfg, uwldap_srv, ldap_conn): conn.delete(dn) +@pytest.fixture +def webhosting_srv_resources(webhosting_srv): + os.makedirs(webhosting_srv.conf_available_dir, exist_ok=True) + os.makedirs(webhosting_srv.sites_available_dir, exist_ok=True) + reset_disable_club_conf(webhosting_srv) + for file in os.listdir(webhosting_srv.sites_available_dir): + os.unlink(os.path.join(webhosting_srv.sites_available_dir, file)) + if os.path.isdir(webhosting_srv.apache_dir + '/.git'): + shutil.rmtree(webhosting_srv.apache_dir + '/.git') + + @pytest.fixture(scope='module') def app_process(cfg, app, http_client): port = cfg.get('ceod_port') diff --git a/tests/resources/apache2/disable-club.conf b/tests/resources/apache2/disable-club.conf new file mode 100644 index 0000000..300c947 --- /dev/null +++ b/tests/resources/apache2/disable-club.conf @@ -0,0 +1,8 @@ +# This file would normally be in /etc/apache2/conf-available/disable-club.conf +# on caffeine. + +Alias /~sysadmin/disabled /users/sysadmin/www/disabled + + + Include snippets/disable-club.conf + diff --git a/tests/utils.py b/tests/utils.py index 2a57cce..8eb5daf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,7 @@ import ceod.utils as ceod_utils import contextlib import os +import shutil import subprocess from subprocess import DEVNULL import tempfile @@ -70,3 +71,13 @@ def set_datetime_in_app_process(app_process, value): def restore_datetime_in_app_process(app_process): set_datetime_in_app_process(app_process, None) + + +def reset_disable_club_conf(webhosting_srv): + shutil.copy('tests/resources/apache2/disable-club.conf', webhosting_srv.conf_available_dir) + + +def create_php_file_for_club(clubs_home, club_name): + www = f'{clubs_home}/{club_name}/www' + os.makedirs(www, exist_ok=True) + open(www + '/index.php', 'w').close()