Disable inactive club sites (#68)

Closes #51.

An API argument called `remove_inactive_club_reps` was added so that we can dynamically control whether we want to remove inactive club reps or not. The default action is only to disable club websites without changing group membership.

Reviewed-on: #68
Co-authored-by: Max Erenberg <merenber@csclub.uwaterloo.ca>
Co-committed-by: Max Erenberg <merenber@csclub.uwaterloo.ca>

.drone.yml

@@ -10,7 +10,7 @@ steps:
   # way to share system packages between steps
   commands:
     # install dependencies
-    - apt update && apt install -y libkrb5-dev libpq-dev python3-dev
+    - apt update && apt install -y libkrb5-dev libpq-dev python3-dev libaugeas0
     - python3 -m venv venv
     - . venv/bin/activate
     - pip install -r dev-requirements.txt

.drone/auth1-setup.sh

@@ -16,8 +16,6 @@ if [ -n "$CI" ]; then
     rm /tmp/hosts
 fi
 
-export DEBIAN_FRONTEND=noninteractive
-apt update
 apt install -y psmisc
 
 # If we don't do this then OpenLDAP uses a lot of RAM
@@ -98,7 +96,6 @@ while true; do
     fi
 done
 
-apt install -y netcat-openbsd
 # sync with phosphoric-acid
 nc -l 0.0.0.0 9000 &
 if [ -z "$CI" ]; then

.drone/coffee-setup.sh

@@ -8,9 +8,6 @@ set -ex
 add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee
 add_fqdn_to_hosts $(get_ip_addr auth1) auth1
 
-export DEBIAN_FRONTEND=noninteractive
-apt update
-
 apt install --no-install-recommends -y default-mysql-server postgresql
 
 # MYSQL
@@ -49,7 +46,6 @@ REVOKE ALL ON SCHEMA public FROM public;
 GRANT ALL ON SCHEMA public TO postgres;
 EOF" postgres
 
-apt install -y netcat-openbsd
 if [ -z "$CI" ]; then
     auth_setup coffee
 fi

.drone/common.sh

@@ -23,6 +23,16 @@ cp .drone/k8s-authority.crt /etc/csc/k8s-authority.crt
 # openssl is actually already present in the python Docker image,
 # so we don't need to mock it out
 
+# netcat is used for synchronization between the containers
+export DEBIAN_FRONTEND=noninteractive
+apt update
+apt install -y netcat-openbsd
+if [ "$(hostname)" != auth1 ]; then
+    # ceod uses Augeas, which is not installed by default in the Python
+    # Docker container
+    apt install -y libaugeas0
+fi
+
 get_ip_addr() {
     getent hosts $1 | cut -d' ' -f1
 }

.drone/mail-setup.sh

@@ -14,9 +14,6 @@ python -m tests.MockSMTPServer &
 python -m tests.MockCloudStackServer &
 python -m tests.MockHarborServer &
 
-export DEBIAN_FRONTEND=noninteractive
-apt update
-apt install -y netcat-openbsd
 auth_setup mail
 
 # for the VHostManager

.drone/phosphoric-acid-setup.sh

@@ -13,9 +13,6 @@ if [ -z "$CI" ]; then
     add_fqdn_to_hosts $(get_ip_addr mail) mail
 fi
 
-export DEBIAN_FRONTEND=noninteractive
-apt update
-apt install -y netcat-openbsd
 auth_setup phosphoric-acid
 
 # initialize the skel directory

README.md

@@ -16,8 +16,7 @@ Docker containers instead, which are much easier to work with than the VM.
 
 First, make sure you create the virtualenv:
 ```sh
-docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \
-    sh -c 'python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
+docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
 ```
 Then bring up the containers:
 ```sh

ceo/cli/entrypoint.py

@@ -10,6 +10,7 @@ from .mailman import mailman
 from .cloud import cloud
 from .k8s import k8s
 from .registry import registry
+from .webhosting import webhosting
 
 
 @click.group()
@@ -27,3 +28,4 @@ cli.add_command(mailman)
 cli.add_command(cloud)
 cli.add_command(k8s)
 cli.add_command(registry)
+cli.add_command(webhosting)

ceo/cli/webhosting.py

@@ -0,0 +1,37 @@
+import click
+
+from ..utils import http_post
+from .utils import handle_sync_response
+
+
+@click.group(short_help='Manage websites hosted by the main CSC web server')
+def webhosting():
+    pass
+
+
+@webhosting.command(short_help='Disable club sites with no active club reps')
+@click.option('--dry-run', is_flag=True, default=False)
+@click.option('--remove-inactive-club-reps', is_flag=True, default=False)
+def disableclubsites(dry_run, remove_inactive_club_reps):
+    params = {}
+    if dry_run:
+        params['dry_run'] = 'true'
+    if remove_inactive_club_reps:
+        params['remove_inactive_club_reps'] = 'true'
+    if not dry_run:
+        click.confirm('Are you sure you want to disable the websites of clubs with no active club reps?', abort=True)
+
+    resp = http_post('/api/webhosting/disableclubsites', params=params)
+    disabled_club_names = handle_sync_response(resp)
+    if len(disabled_club_names) == 0:
+        if dry_run:
+            click.echo('No websites would have been disabled.')
+        else:
+            click.echo('No websites were disabled.')
+    else:
+        if dry_run:
+            click.echo('The following club websites would have been disabled:')
+        else:
+            click.echo('The following club websites were disabled:')
+        for club_name in disabled_club_names:
+            click.echo(club_name)

ceo/utils.py

@@ -21,6 +21,8 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response:
         host = cfg.get('ceod_mailman_host')
     elif path.startswith('/api/cloud'):
         host = cfg.get('ceod_cloud_host')
+    elif path.startswith('/api/webhosting'):
+        host = cfg.get('ceod_webhosting_host')
     else:
         host = cfg.get('ceod_admin_host')
     return client.request(

ceo_common/interfaces/IClubWebHostingService.py

@@ -0,0 +1,45 @@
+from typing import List
+
+from zope.interface import Interface
+
+
+class IClubWebHostingService(Interface):
+    """
+    Performs operations on the configuration files of club websites hosted
+    by the CSC.
+    """
+
+    def begin_transaction():
+        """
+        Performs any steps necessary to query the website config files.
+        This must be called BEFORE any of the methods below, using a context
+        expression like so:
+
+        with club_site_mgr.begin_transaction():
+            club_site_mgr.disable_club_site('club1')
+            ...
+            club_site_mgr.commit()
+        """
+
+    def commit():
+        """
+        Writes the config file changes to disk. This must be called at the
+        end of the context expression.
+        """
+
+    def disable_club_site(club_name: str):
+        """
+        Disables the site for the club. Note that commit() must still be
+        called to commit this change.
+        """
+
+    def disable_sites_for_inactive_clubs(
+        dry_run: bool = False,
+        remove_inactive_club_reps: bool = False,
+    ) -> List[str]:
+        """
+        Disables sites for inactive clubs. If remove_inactive_club_reps is set
+        to True, then inactive club reps will be removed from club groups.
+        The list of clubs whose sites were disabled (or would have been
+        disabled, if dry_run is True) is returned.
+        """

ceo_common/interfaces/ILDAPService.py

@@ -9,10 +9,10 @@ from .IGroup import IGroup
 class ILDAPService(Interface):
     """An interface to the LDAP database."""
 
-    def uid_to_dn(self, uid: str) -> str:
+    def uid_to_dn(uid: str) -> str:
         """Get the LDAP DN for the user with this UID."""
 
-    def group_cn_to_dn(self, cn: str) -> str:
+    def group_cn_to_dn(cn: str) -> str:
         """Get the LDAP DN for the group with this CN."""
 
     def get_user(username: str) -> IUser:
@@ -24,7 +24,7 @@ class ILDAPService(Interface):
         Useful for displaying a list of users in a compact way.
         """
 
-    def get_users_with_positions(self) -> List[IUser]:
+    def get_users_with_positions() -> List[IUser]:
         """Retrieve users who have a non-empty position attribute."""
 
     def add_user(user: IUser):
@@ -36,7 +36,7 @@ class ILDAPService(Interface):
     def remove_user(user: IUser):
         """Remove this user from the database."""
 
-    def get_group(cn: str, is_club: bool = False) -> IGroup:
+    def get_group(cn: str) -> IGroup:
         """Retrieve the group with the given cn (Unix group name)."""
 
     def add_group(group: IGroup):
@@ -88,15 +88,33 @@ class ILDAPService(Interface):
         described above.
         """
 
-    def get_nonflagged_expired_users(self) -> List[IUser]:
+    def get_nonflagged_expired_users() -> List[IUser]:
         """
         Retrieves members whose term or nonMemberTerm does not contain the
         current or the last term.
         """
 
-    def get_expiring_users(self) -> List[IUser]:
+    def get_expiring_users() -> List[IUser]:
         """
         Retrieves members whose membership will expire in less than a month.
         This is used to send membership renewal reminders at the beginning
         of a term, during the one-month grace period.
         """
+
+    def get_clubs() -> List[IGroup]:
+        """
+        Retrieves all clubs.
+        """
+
+    # couldn't import the Term class from ceo_common.model due to some
+    # circular import issue...
+    def get_club_reps_non_member_terms(club_reps: List[str]) -> Dict[str, List['Term']]:  # noqa: F821
+        """
+        Retrieves the non-member terms for the given club reps.
+        e.g.
+        {
+            "user1": [w2022, s2022],
+            "user2": [s2022],
+            ...
+        }
+        """

ceo_common/interfaces/IMailService.py

@@ -24,20 +24,26 @@ class IMailService(Interface):
         during the transaction.
         """
 
-    def send_membership_renewal_reminder(self, user: IUser):
+    def send_membership_renewal_reminder(user: IUser):
         """
         Send a reminder to the user that their membership will expire
         soon.
         """
 
-    def send_cloud_account_will_be_deleted_message(self, user: IUser):
+    def send_cloud_account_will_be_deleted_message(user: IUser):
         """
         Send a warning message to the user that their cloud resources
         will be deleted if they do not renew their membership.
         """
 
-    def send_cloud_account_has_been_deleted_message(self, user: IUser):
+    def send_cloud_account_has_been_deleted_message(user: IUser):
         """
         Send a message to the user that their cloud resources have
         been deleted.
         """
+
+    def send_club_website_has_been_disabled_message(club: str, address: str):
+        """
+        Send a message to the club stating that their website has been
+        disabled.
+        """

ceo_common/interfaces/__init__.py

@@ -14,3 +14,4 @@ from .IDatabaseService import IDatabaseService
 from .IVHostManager import IVHostManager
 from .IKubernetesService import IKubernetesService
 from .IContainerRegistryService import IContainerRegistryService
+from .IClubWebHostingService import IClubWebHostingService

ceo_common/model/HTTPClient.py

@@ -43,7 +43,7 @@ class HTTPClient:
             if flask.has_request_context() and 'client_token' in g:
                 # This is reached when we are the server and the client has
                 # forwarded their credentials to us.
-                spnego_kwargs['creds'] = gssapi.Credentials(token=flask.g.client_token)
+                spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token)
             elif delegate:
                 # This is reached when we are the client and we want to
                 # forward our credentials to the server.

ceod/api/app_factory.py

@@ -9,13 +9,13 @@ from .error_handlers import register_error_handlers
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
     IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
     ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager, \
-    IContainerRegistryService
+    IContainerRegistryService, IClubWebHostingService
 from ceo_common.model import Config, HTTPClient, RemoteMailmanService
 from ceod.api.spnego import init_spnego
 from ceod.model import KerberosService, LDAPService, FileService, \
     MailmanService, MailService, UWLDAPService, CloudStackService, \
     CloudResourceManager, KubernetesService, VHostManager, \
-    ContainerRegistryService
+    ContainerRegistryService, ClubWebHostingService
 from ceod.db import MySQLService, PostgreSQLService
 
 
@@ -49,6 +49,10 @@ def create_app(flask_config={}):
         from ceod.api import cloud
         app.register_blueprint(cloud.bp, url_prefix='/api/cloud')
 
+    if hostname == cfg.get('ceod_webhosting_host'):
+        from ceod.api import webhosting
+        app.register_blueprint(webhosting.bp, url_prefix='/api/webhosting')
+
     from ceod.api import groups
     app.register_blueprint(groups.bp, url_prefix='/api/groups')
 
@@ -112,6 +116,11 @@ def register_services(app):
     uwldap_srv = UWLDAPService()
     component.provideUtility(uwldap_srv, IUWLDAPService)
 
+    # ClubWebHostingService
+    if hostname == cfg.get('ceod_webhosting_host'):
+        webhosting_srv = ClubWebHostingService()
+        component.provideUtility(webhosting_srv, IClubWebHostingService)
+
     # MySQLService, PostgreSQLService
     if hostname == cfg.get('ceod_database_host'):
         mysql_srv = MySQLService()

ceod/api/webhosting.py

@@ -0,0 +1,22 @@
+from flask import Blueprint, request
+from flask.json import jsonify
+from zope import component
+
+from .utils import authz_restrict_to_syscom, is_truthy
+from ceo_common.interfaces import IClubWebHostingService
+
+
+bp = Blueprint('webhosting', __name__)
+
+
+@bp.route('/disableclubsites', methods=['POST'])
+@authz_restrict_to_syscom
+def expire_club_sites():
+    dry_run = is_truthy(request.args.get('dry_run', 'false'))
+    remove_inactive_club_reps = is_truthy(request.args.get('remove_inactive_club_reps', 'false'))
+    webhosting_srv = component.getUtility(IClubWebHostingService)
+    disabled_sites = webhosting_srv.disable_sites_for_inactive_clubs(
+        dry_run=dry_run,
+        remove_inactive_club_reps=remove_inactive_club_reps,
+    )
+    return jsonify(disabled_sites)

ceod/model/ClubWebHostingService.py

@@ -0,0 +1,250 @@
+from collections import defaultdict
+import contextlib
+import glob
+import os
+import re
+import subprocess
+from threading import Lock
+import traceback
+from typing import List
+
+from augeas import Augeas
+from zope import component
+from zope.interface import implementer
+
+from ceo_common.interfaces import ILDAPService, IMailService, \
+    IClubWebHostingService, IConfig
+from ceo_common.logger_factory import logger_factory
+from ceo_common.model import Term
+
+# NOTE: We are assuming that the name of a club is the same as its home
+# directory (under /users). While this rule is not enforced, it has been true
+# up until now.
+# FIXME: don't assume this
+APACHE_USERDIR_RE = re.compile(r'^/users/(?P<club_name>[0-9a-z-]+)/www/?$')
+
+# This is where all the <Directory> directives for disabled clubs are stored.
+APACHE_DISABLED_CLUBS_FILE = 'conf-available/disable-club.conf'
+# This is the file which contains the snippet to disable a single club.
+APACHE_DISABLING_SNIPPET_FILE = 'snippets/disable-club.conf'
+# This is the maximum number of consecutive terms for which a club is allowed
+# to have no active club reps. After this many terms have passed, the club's
+# website may be disabled.
+MAX_TERMS_WITH_NO_ACTIVE_CLUB_REPS = 3
+
+logger = logger_factory(__name__)
+
+
+def is_a_disabling_snippet(snippet_path: str) -> bool:
+    return snippet_path.startswith('snippets/disable-')
+
+
+@implementer(IClubWebHostingService)
+class ClubWebHostingService:
+    def __init__(self, augeas_root='/'):
+        cfg = component.getUtility(IConfig)
+        self.aug_root = augeas_root
+        self.apache_dir = os.path.join(augeas_root, 'etc/apache2')
+        self.sites_available_dir = os.path.join(self.apache_dir, 'sites-available')
+        self.conf_available_dir = os.path.join(self.apache_dir, 'conf-available')
+        self.clubs_home = cfg.get('clubs_home')
+        self.aug = None
+        self.clubs = None
+        self.made_at_least_one_change = False
+        self.lock = Lock()
+
+    @contextlib.contextmanager
+    def _hold_lock(self):
+        try:
+            self.lock.acquire()
+            yield
+        finally:
+            self.lock.release()
+
+    @contextlib.contextmanager
+    def begin_transaction(self):
+        with self._hold_lock():
+            try:
+                self.aug = Augeas(self.aug_root)
+                self.clubs = defaultdict(lambda: {'disabled': False, 'email': None})
+                self._get_club_emails()
+                self._get_disabled_sites()
+                yield
+            finally:
+                if self.aug is not None:
+                    self.aug.close()
+                self.aug = None
+                self.clubs = None
+                self.made_at_least_one_change = False
+
+    def _run(self, args: List[str], **kwargs):
+        subprocess.run(args, check=True, **kwargs)
+
+    def _reload_web_server(self):
+        logger.debug('Reloading Apache')
+        self._run(['systemctl', 'reload', 'apache2'])
+
+    # This requires the APACHE_CONFIG_CRON environment variable to be
+    # set to 1 (e.g. in a systemd drop-in)
+    # See /etc/apache2/.git/hooks/pre-commit on caffeine
+    def _git_commit(self):
+        if not os.path.isdir(os.path.join(self.apache_dir, '.git')):
+            logger.debug('No git folder found in Apache directory')
+            return
+        logger.debug('Committing changes to git repository')
+        self._run(['git', 'add', APACHE_DISABLED_CLUBS_FILE], cwd=self.apache_dir)
+        self._run(['git', 'commit', '-m', '[ceo] disable club websites'], cwd=self.apache_dir)
+
+    def commit(self):
+        if not self.made_at_least_one_change:
+            return
+        logger.debug('Saving Augeas changes')
+        self.aug.save()
+        self._reload_web_server()
+        self._git_commit()
+
+    def _get_club_emails(self):
+        config_file_paths = glob.glob(self.sites_available_dir + '/club-*.conf')
+        for config_file_path in config_file_paths:
+            club_name = None
+            club_email = None
+            filename = os.path.basename(config_file_path)
+            directive_paths = self.aug.match(f'/files/etc/apache2/sites-available/{filename}/VirtualHost/directive')
+            for directive_path in directive_paths:
+                directive = self.aug.get(directive_path)
+                directive_value = self.aug.get(directive_path + '/arg')
+                if directive == 'DocumentRoot':
+                    match = APACHE_USERDIR_RE.match(directive_value)
+                    if match is not None:
+                        club_name = match.group('club_name')
+                elif directive == 'ServerAdmin':
+                    club_email = directive_value
+            if club_name is not None:
+                self.clubs[club_name]['email'] = club_email
+
+    def _get_disabled_sites(self):
+        config_file_paths = glob.glob(self.conf_available_dir + '/disable-*.conf')
+        for config_file_path in config_file_paths:
+            filename = os.path.basename(config_file_path)
+            directory_paths = self.aug.match(f'/files/etc/apache2/conf-available/{filename}/Directory')
+            for directory_path in directory_paths:
+                directory = self.aug.get(directory_path + '/arg')
+                match = APACHE_USERDIR_RE.match(directory)
+                if match is None:
+                    continue
+                club_name = match.group('club_name')
+                directive_paths = self.aug.match(directory_path + '/directive')
+                for directive_path in directive_paths:
+                    if self.aug.get(directive_path) != 'Include':
+                        continue
+                    included_file = self.aug.get(directive_path + '/arg')
+                    if is_a_disabling_snippet(included_file):
+                        self.clubs[club_name]['disabled'] = True
+
+    def disable_club_site(self, club_name: str):
+        logger.info(f'Disabling website for {club_name}')
+        directory_path = f'/files/etc/apache2/{APACHE_DISABLED_CLUBS_FILE}/Directory'
+        num_directories = len(self.aug.match(directory_path))
+        # Create a new <Directory> section
+        directory_path += '[%d]' % (num_directories + 1)
+        # FIXME: use self.clubs_home here instead (need to update unit tests)
+        self.aug.set(directory_path + '/arg', f'/users/{club_name}/www')
+        self.aug.set(directory_path + '/directive', 'Include')
+        self.aug.set(directory_path + '/directive/arg', APACHE_DISABLING_SNIPPET_FILE)
+
+        self.clubs[club_name]['disabled'] = True
+        self.made_at_least_one_change = True
+
+    def _site_uses_php(self, club_name: str) -> bool:
+        www = f'{self.clubs_home}/{club_name}/www'
+        if os.path.isdir(www):
+            # We're just going to look one level deep; that should be good enough.
+            for filename in os.listdir(www):
+                filepath = os.path.join(www, filename)
+                if os.path.isfile(filepath) and filename.endswith('.php'):
+                    return True
+        return False
+
+    # This method needs to be called from within a transaction (uses self.clubs)
+    def _need_to_disable_inactive_club_site(self, club_name: str) -> bool:
+        if self.clubs[club_name]['disabled']:
+            # already disabled - nothing to do
+            return False
+        if not self._site_uses_php(club_name):
+            # These are the only sites which are actually a security concern
+            # to us - it's OK for a purely static website to be unmaintained.
+            return False
+        return True
+
+    def disable_sites_for_inactive_clubs(
+        self,
+        dry_run: bool = False,
+        remove_inactive_club_reps: bool = False,
+    ) -> List[str]:
+        ldap_srv = component.getUtility(ILDAPService)
+        mail_srv = component.getUtility(IMailService)
+        all_clubs = ldap_srv.get_clubs()
+        club_rep_uids = list({
+            member
+            for club in all_clubs
+            for member in club.members
+        })
+        club_reps = ldap_srv.get_club_reps_non_member_terms(club_rep_uids)
+        # If a club rep's last non-member term is before cutoff_term, then
+        # they are considered inactive
+        cutoff_term = Term.current() - MAX_TERMS_WITH_NO_ACTIVE_CLUB_REPS
+        active_club_reps = {
+            club_rep
+            for club_rep, non_member_terms in club_reps.items()
+            if len(non_member_terms) > 0 and max(non_member_terms) >= cutoff_term
+        }
+        # a club is inactive if it does not have at least one active club rep
+        inactive_clubs = [
+            club
+            for club in all_clubs
+            if not any(map(lambda member: member in active_club_reps, club.members))
+        ]
+
+        # STEP 1: update the Apache configs
+        with self.begin_transaction():
+            clubs_to_disable = [
+                club.cn
+                for club in inactive_clubs
+                if self._need_to_disable_inactive_club_site(club.cn)
+            ]
+            if dry_run:
+                return clubs_to_disable
+
+            for club_name in clubs_to_disable:
+                self.disable_club_site(club_name)
+            self.commit()
+            # self.clubs is set to None once the transaction closes, so make a copy now
+            clubs_info = self.clubs.copy()
+
+        # STEP 2: send emails to clubs whose websites were disabled
+        clubs_who_were_not_notified = set()
+        for club_name in clubs_to_disable:
+            address = clubs_info['email']
+            if address is None:
+                clubs_who_were_not_notified.add(club_name)
+                continue
+            try:
+                mail_srv.send_club_website_has_been_disabled_message(club_name, address)
+            except Exception:
+                trace = traceback.format_exc()
+                logger.error(f'Failed to send email to {address}:\n{trace}')
+                clubs_who_were_not_notified.add(club_name)
+
+        # STEP 3: remove inactive club reps from Unix groups
+        if remove_inactive_club_reps:
+            for club in all_clubs:
+                if club.cn in clubs_who_were_not_notified:
+                    continue
+                # club.members gets modified after calling club.remove_member(),
+                # so we need to make a copy
+                members = club.members.copy()
+                for member in members:
+                    if member not in active_club_reps:
+                        logger.info(f'Removing {member} from {club.cn}')
+                        club.remove_member(member)
+        return clubs_to_disable

ceod/model/LDAPService.py

@@ -344,3 +344,37 @@ class LDAPService:
             conn.modify(self.uid_to_dn(uid), changes)
 
         return users_to_change
+
+    def _get_club_uids(self, conn: ldap3.Connection) -> List[str]:
+        conn.search(self.ldap_users_base, '(objectClass=club)', attributes=['uid'])
+        return [entry.uid.value for entry in conn.entries]
+
+    def get_clubs(self) -> List[IGroup]:
+        batch_size = 100
+        conn = self._get_ldap_conn()
+        club_uids = self._get_club_uids(conn)
+        clubs = []
+        for i in range(0, len(club_uids), batch_size):
+            club_uids_slice = club_uids[i:i + batch_size]
+            filter = '(|' + ''.join([f'(cn={uid})' for uid in club_uids_slice]) + ')'
+            conn.search(self.ldap_groups_base, filter, attributes=ldap3.ALL_ATTRIBUTES)
+            for entry in conn.entries:
+                clubs.append(Group.deserialize_from_ldap(entry))
+        return clubs
+
+    def get_club_reps_non_member_terms(self, club_reps: List[str]) -> Dict[str, List[Term]]:
+        batch_size = 100
+        conn = self._get_ldap_conn()
+        club_reps_terms = {}
+        for i in range(0, len(club_reps), batch_size):
+            club_reps_slice = club_reps[i:i + batch_size]
+            filter = '(|' + ''.join([f'(uid={uid})' for uid in club_reps_slice]) + ')'
+            conn.search(self.ldap_users_base, filter, attributes=['uid', 'nonMemberTerm'])
+            for entry in conn.entries:
+                uid = entry.uid.value
+                if 'nonMemberTerm' in entry.entry_attributes:
+                    non_member_terms = list(map(Term, entry.nonMemberTerm.values))
+                else:
+                    non_member_terms = []
+                club_reps_terms[uid] = non_member_terms
+        return club_reps_terms

ceod/model/MailService.py

@@ -137,3 +137,17 @@ class MailService:
             },
             body,
         )
+
+    def send_club_website_has_been_disabled_message(self, club: str, address: str):
+        template = self.jinja_env.get_template('club_website_has_been_disabled.j2')
+        body = template.render(club=club)
+        self.send(
+            f'Computer Science Club <ceo+webhosting@{self.base_domain}>',
+            f'{club} <{address}>',
+            {
+                'Subject': 'Your club website has been disabled',
+                'Cc': f'root@{self.base_domain}',
+                'Reply-To': f'syscom@{self.base_domain}',
+            },
+            body,
+        )

ceod/model/__init__.py

@@ -13,3 +13,4 @@ from .MailmanService import MailmanService
 from .VHostManager import VHostManager
 from .KubernetesService import KubernetesService
 from .ContainerRegistryService import ContainerRegistryService
+from .ClubWebHostingService import ClubWebHostingService

ceod/model/templates/club_website_has_been_disabled.j2

@@ -0,0 +1,20 @@
+Hello {{ club }} representatives,
+
+This is an automated message from ceo, the CSC Electronic Office.
+
+Since your club has not had any active CSC club reps for 3 or more consecutive
+terms, your club's website has been disabled. Websites require active
+maintenance and software upgrades, and an unmaintained website poses a
+security risk to CSC and to the University of Waterloo.
+
+If you wish for your site to be re-enabled, please register at least
+one club rep with CSC and, if necessary, upgrade the software running
+your website. A club rep membership is free. You can find instructions
+on our website for how to obtain a membership:
+
+https://csclub.uwaterloo.ca/get-involved/
+
+If you have any questions or concerns, please contact syscom@csclub.uwaterloo.ca.
+
+Sincerely,
+ceo

debian/control

@@ -11,6 +11,7 @@ Build-Depends: debhelper (>= 12.1.1),
     python3-venv (>= 3.7),
     libkrb5-dev (>= 1.17),
     libpq-dev (>= 11.13),
+    libaugeas0 (>= 1.11),
     scdoc (>= 1.9)
 
 Package: ceo-common
@@ -19,6 +20,7 @@ Depends: python3 (>= 3.7),
     krb5-user (>= 1.17),
     libkrb5-3 (>= 1.17),
     libpq5 (>= 11.13),
+    libaugeas0 (>= 1.11),
     ${python3:Depends},
     ${misc:Depends}
 Description: CSC Electronic Office common files

etc/ceo.ini

@@ -7,6 +7,8 @@ uw_domain = uwaterloo.ca
 admin_host = phosphoric-acid
 # this is the host with root access to the databases
 database_host = caffeine
+# this is the host where clubs' websites are hosted
+webhosting_host = caffeine
 # this is the host which can make API requests to Mailman
 mailman_host = mailman
 # this is the host running a CloudStack management server

etc/ceod.ini

@@ -8,6 +8,8 @@ admin_host = phosphoric-acid
 fs_root_host = phosphoric-acid
 # this is the host with root access to the databases
 database_host = caffeine
+# this is the host where clubs' websites are hosted
+webhosting_host = caffeine
 # this is the host which can make API requests to Mailman
 mailman_host = mailman
 # this is the host which is running a CloudStack management server

requirements.txt

@@ -7,9 +7,10 @@ Jinja2==3.1.2
 ldap3==2.9.1
 mysql-connector-python==8.0.26
 psycopg2==2.9.1
+python-augeas==1.1.0
 requests==2.26.0
 requests-gssapi==1.2.3
 urwid==2.1.2
-werkzeug==2.1.2
+Werkzeug==2.1.2
 zope.component==5.0.1
 zope.interface==5.4.0

tests/ceo/cli/test_webhosting.py

@@ -0,0 +1,62 @@
+from click.testing import CliRunner
+
+from ceo.cli import cli
+from ceo_common.model import Term
+from tests.utils import (
+    create_php_file_for_club,
+    reset_disable_club_conf,
+    set_datetime_in_app_process,
+    restore_datetime_in_app_process,
+)
+
+
+def test_disable_club_sites(
+    cfg, cli_setup, app_process, webhosting_srv, webhosting_srv_resources,
+    new_club_gen, new_user_gen, g_admin_ctx, ldap_srv_session,
+):
+    runner = CliRunner()
+    term = Term.current()
+    clubs_home = cfg.get('clubs_home')
+    with new_club_gen() as group, new_user_gen() as user:
+        create_php_file_for_club(clubs_home, group.cn)
+        user.add_non_member_terms([str(Term.current())])
+        group.add_member(user.uid)
+
+        set_datetime_in_app_process(app_process, (term + 4).to_datetime())
+
+        result = runner.invoke(cli, ['webhosting', 'disableclubsites', '--dry-run'])
+        assert result.exit_code == 0
+        assert result.output.endswith(
+            'The following club websites would have been disabled:\n'
+            f'{group.cn}\n'
+        )
+
+        result = runner.invoke(cli, ['webhosting', 'disableclubsites'], input='y\n')
+        assert result.exit_code == 0
+        assert result.output.endswith(
+            'The following club websites were disabled:\n'
+            f'{group.cn}\n'
+        )
+
+        with g_admin_ctx():
+            group = ldap_srv_session.get_group(group.cn)
+        assert group.members == [user.uid]
+
+        reset_disable_club_conf(webhosting_srv)
+
+        result = runner.invoke(
+            cli,
+            ['webhosting', 'disableclubsites', '--remove-inactive-club-reps'],
+            input='y\n'
+        )
+        assert result.exit_code == 0
+        assert result.output.endswith(
+            'The following club websites were disabled:\n'
+            f'{group.cn}\n'
+        )
+
+        with g_admin_ctx():
+            group = ldap_srv_session.get_group(group.cn)
+        assert group.members == []
+
+    restore_datetime_in_app_process(app_process)

tests/ceo_dev.ini

@@ -6,6 +6,7 @@ uw_domain = uwaterloo.internal
 # this is the host with the ceod/admin Kerberos key
 admin_host = phosphoric-acid
 database_host = coffee
+webhosting_host = coffee
 mailman_host = mail
 cloud_host = mail
 use_https = false

tests/ceod/api/test_webhosting.py

@@ -0,0 +1,43 @@
+import datetime
+from unittest.mock import patch
+
+from ceo_common.model import Term
+import ceo_common.utils
+from tests.utils import create_php_file_for_club, reset_disable_club_conf
+
+
+def test_disable_club_sites(
+    cfg, client, webhosting_srv, webhosting_srv_resources, new_club_gen,
+    new_user_gen, g_admin_ctx, ldap_srv_session,
+):
+    term = Term.current()
+    clubs_home = cfg.get('clubs_home')
+    with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
+        now_mock.return_value = datetime.datetime.now()
+        with new_club_gen() as group, new_user_gen() as user:
+            create_php_file_for_club(clubs_home, group.cn)
+            user.add_non_member_terms([str(Term.current())])
+            group.add_member(user.uid)
+
+            now_mock.return_value = (term + 4).to_datetime()
+            status, data = client.post('/api/webhosting/disableclubsites?dry_run=true')
+            assert status == 200
+            assert data == [group.cn]
+
+            status, data = client.post('/api/webhosting/disableclubsites')
+            assert status == 200
+            assert data == [group.cn]
+
+            with g_admin_ctx():
+                group = ldap_srv_session.get_group(group.cn)
+            assert group.members == [user.uid]
+
+            reset_disable_club_conf(webhosting_srv)
+
+            status, data = client.post('/api/webhosting/disableclubsites?remove_inactive_club_reps=true')
+            assert status == 200
+            assert data == [group.cn]
+
+            with g_admin_ctx():
+                group = ldap_srv_session.get_group(group.cn)
+            assert group.members == []

tests/ceod/model/test_webhosting.py

@@ -0,0 +1,163 @@
+import datetime
+import os
+import re
+import subprocess
+from unittest.mock import patch
+
+from ceo_common.model import Term
+import ceo_common.utils
+from tests.utils import create_php_file_for_club
+
+
+def create_website_config_for_club(sites_available_dir, cn, filename=None):
+    if filename is None:
+        filename = f'club-{cn}.conf'
+    filepath = os.path.join(sites_available_dir, filename)
+    with open(filepath, 'w') as fo:
+        fo.write(f"""
+        <VirtualHost *:80>
+            ServerName {cn}.uwaterloo.internal
+            ServerAdmin {cn}@{cn}.uwaterloo.internal
+            DocumentRoot /users/{cn}/www/
+        </VirtualHost>
+        """)
+
+
+def get_enabled_sites(webhosting_srv):
+    return [
+        club_name
+        for club_name, club_info in webhosting_srv.clubs.items()
+        if not club_info['disabled']
+    ]
+
+
+def test_disable_club_sites(webhosting_srv, webhosting_srv_resources, new_club_gen):
+    sites_available_dir = webhosting_srv.sites_available_dir
+    with new_club_gen() as group1, new_club_gen() as group2:
+        create_website_config_for_club(sites_available_dir, group1.cn)
+        create_website_config_for_club(sites_available_dir, group2.cn, 'club-somerandomname.conf')
+        with webhosting_srv.begin_transaction():
+            enabled_clubs = get_enabled_sites(webhosting_srv)
+            assert sorted(enabled_clubs) == [group1.cn, group2.cn]
+            webhosting_srv.disable_club_site(group1.cn)
+        # Make sure that if we don't call commit(), nothing gets saved
+        with webhosting_srv.begin_transaction():
+            enabled_clubs = get_enabled_sites(webhosting_srv)
+            assert sorted(enabled_clubs) == [group1.cn, group2.cn]
+            webhosting_srv.disable_club_site(group1.cn)
+            webhosting_srv.commit()
+        # Now that we committed the changes, they should be persistent
+        with webhosting_srv.begin_transaction():
+            enabled_clubs = get_enabled_sites(webhosting_srv)
+            assert enabled_clubs == [group2.cn]
+
+
+def test_disable_inactive_club_sites(
+    cfg, webhosting_srv, webhosting_srv_resources, g_admin_ctx, new_club_gen,
+    new_user_gen, mock_mail_server,
+):
+    sites_available_dir = webhosting_srv.sites_available_dir
+    term = Term.current()
+    clubs_home = cfg.get('clubs_home')
+    with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
+        now_mock.return_value = datetime.datetime.now()
+        with new_club_gen() as group1, \
+                new_club_gen() as group2, \
+                new_user_gen() as user1, \
+                new_user_gen() as user2:
+            create_website_config_for_club(sites_available_dir, group1.cn)
+            create_website_config_for_club(sites_available_dir, group2.cn)
+            create_php_file_for_club(clubs_home, group1.cn)
+            with g_admin_ctx():
+                # group1 has no club reps so it should be disabled
+                # group2 has no club reps but it doesn't use PHP
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [group1.cn]
+
+            user1.add_non_member_terms([str(Term.current())])
+            group1.add_member(user1.uid)
+            with g_admin_ctx():
+                # group1 has an active club rep, so it shouldn't be disabled anymore
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
+
+            now_mock.return_value = (term + 3).to_datetime()
+            with g_admin_ctx():
+                # club reps are allowed to be inactive for up to 3 terms
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
+
+            now_mock.return_value = (term + 4).to_datetime()
+            with g_admin_ctx():
+                # club site should be disabled now that club rep is inactive
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [group1.cn]
+
+            user2.add_non_member_terms([str(Term.current())])
+            group1.add_member(user2.uid)
+            with g_admin_ctx():
+                # group1 has a new club rep, so it shouldn't be disabled anymore
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
+
+            create_php_file_for_club(clubs_home, group2.cn)
+            group2.add_member(user2.uid)
+            with g_admin_ctx():
+                # user2 is an active club rep for both group1 and group2
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
+
+            now_mock.return_value = (term + 8).to_datetime()
+            mock_mail_server.messages.clear()
+            subprocess.run(['git', 'init'], cwd=webhosting_srv.apache_dir, check=True)
+            with g_admin_ctx():
+                disabled_sites = webhosting_srv.disable_sites_for_inactive_clubs()
+            # user2 expired and both sites use PHP, so they should both be disabled
+            assert sorted(disabled_sites) == [group1.cn, group2.cn]
+            # since each club had a ServerAdmin directive, they should both have received
+            # notification emails
+            assert len(mock_mail_server.messages) == 2
+            with open(webhosting_srv.conf_available_dir + '/disable-club.conf') as fi:
+                disable_club_conf_content = fi.read()
+            print(disable_club_conf_content)
+            for group in [group1, group2]:
+                pat = re.compile(
+                    (
+                        rf'^<Directory "?/users/{group.cn}/www"?>\n'
+                        r'\s*Include snippets/disable-club\.conf\n'
+                        r'</Directory>$'
+                    ),
+                    re.MULTILINE
+                )
+                assert pat.search(disable_club_conf_content) is not None
+
+            with g_admin_ctx():
+                # Club sites should only be disabled once
+                assert webhosting_srv.disable_sites_for_inactive_clubs() == []
+
+    mock_mail_server.messages.clear()
+
+
+def test_remove_inactive_club_reps(
+    cfg, webhosting_srv, webhosting_srv_resources, g_admin_ctx, new_club_gen,
+    new_user_gen, ldap_srv_session,
+):
+    term = Term.current()
+    clubs_home = cfg.get('clubs_home')
+    with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
+        now_mock.return_value = datetime.datetime.now()
+        with new_club_gen() as group, \
+                new_user_gen() as user1, \
+                new_user_gen() as user2:
+            create_php_file_for_club(clubs_home, group.cn)
+
+            for user in [user1, user2]:
+                user.add_non_member_terms([str(Term.current())])
+                group.add_member(user.uid)
+            now_mock.return_value = (term + 4).to_datetime()
+            with g_admin_ctx():
+                webhosting_srv.disable_sites_for_inactive_clubs(remove_inactive_club_reps=True)
+                group = ldap_srv_session.get_group(group.cn)
+                assert group.members == []
+
+            # Make sure that inactive club reps get removed even if the site
+            # has already been disabled
+            group.add_member(user1.uid)
+            with g_admin_ctx():
+                webhosting_srv.disable_sites_for_inactive_clubs(remove_inactive_club_reps=True)
+                group = ldap_srv_session.get_group(group.cn)
+                assert group.members == []

tests/ceod_dev.ini

@@ -8,6 +8,8 @@ admin_host = phosphoric-acid
 fs_root_host = phosphoric-acid
 mailman_host = mail
 database_host = coffee
+# this is the host where clubs' websites are hosted
+webhosting_host = coffee
 cloud_host = mail
 use_https = false
 port = 9987

tests/ceod_test_local.ini

@@ -9,6 +9,7 @@ fs_root_host = phosphoric-acid
 mailman_host = phosphoric-acid
 database_host = phosphoric-acid
 cloud_host = phosphoric-acid
+webhosting_host = phosphoric-acid
 use_https = false
 port = 9988
 

tests/conftest.py

@@ -7,6 +7,7 @@ import shutil
 import subprocess
 from subprocess import DEVNULL
 import sys
+import tempfile
 import threading
 import time
 from unittest.mock import Mock
@@ -24,11 +25,12 @@ from .utils import (  # noqa: F401
     gssapi_token_ctx,
     ccache_cleanup,
     mocks_for_create_user_ctx,
+    reset_disable_club_conf,
 )
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
     IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
     IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \
-    ICloudResourceManager, IContainerRegistryService
+    ICloudResourceManager, IContainerRegistryService, IClubWebHostingService
 from ceo_common.model import Config, HTTPClient, Term
 import ceo_common.utils
 from ceod.api import create_app
@@ -36,7 +38,7 @@ from ceod.db import MySQLService, PostgreSQLService
 from ceod.model import KerberosService, LDAPService, FileService, User, \
     MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
     CloudStackService, KubernetesService, VHostManager, CloudResourceManager, \
-    ContainerRegistryService
+    ContainerRegistryService, ClubWebHostingService
 from .MockSMTPServer import MockSMTPServer
 from .MockMailmanServer import MockMailmanServer
 from .MockCloudStackServer import MockCloudStackServer
@@ -132,12 +134,12 @@ def g_syscom(app):
     """
     with gssapi_token_ctx('ctdalek') as token, app.app_context():
         try:
-            flask.g.sasl_user = 'ctdalek'
+            flask.g.auth_user = 'ctdalek'
             flask.g.client_token = token
             yield
         finally:
             flask.g.pop('client_token')
-            flask.g.pop('sasl_user')
+            flask.g.pop('auth_user')
 
 
 @pytest.fixture(scope='session')
@@ -237,6 +239,14 @@ def uwldap_srv(cfg, ldap_conn):
     delete_subtree(conn, base_dn)
 
 
+@pytest.fixture(scope='session')
+def webhosting_srv():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        srv = ClubWebHostingService(tmpdir)
+        component.getGlobalSiteManager().registerUtility(srv, IClubWebHostingService)
+        yield srv
+
+
 @pytest.fixture(scope='session')
 def mock_mail_server():
     mock_server = MockSMTPServer()
@@ -343,6 +353,7 @@ def app(
     k8s_srv,
     registry_srv,
     cloud_mgr,
+    webhosting_srv,
 ):
     app = create_app({'TESTING': True})
     return app
@@ -442,6 +453,31 @@ def new_user(new_user_gen):
         yield user
 
 
+@pytest.fixture
+def new_club_gen(client, g_admin_ctx, ldap_srv_session):  # noqa: F811
+    new_club_id_counter = 31001
+
+    @contextlib.contextmanager
+    def wrapper():
+        nonlocal new_club_id_counter
+        cn = 'test' + str(new_club_id_counter)
+        new_club_id_counter += 1
+        status, data = client.post('/api/groups', json={
+            'cn': cn,
+            'description': 'Test ' + str(new_club_id_counter),
+        })
+        assert status == 200
+        assert data[-1]['status'] == 'completed'
+        with g_admin_ctx():
+            group = ldap_srv_session.get_group(cn)
+        yield group
+        status, data = client.delete(f'/api/groups/{cn}')
+        assert status == 200
+        assert data[-1]['status'] == 'completed'
+
+    return wrapper
+
+
 @pytest.fixture
 def simple_group():
     return Group(
@@ -507,6 +543,17 @@ def uwldap_user(cfg, uwldap_srv, ldap_conn):
     conn.delete(dn)
 
 
+@pytest.fixture
+def webhosting_srv_resources(webhosting_srv):
+    os.makedirs(webhosting_srv.conf_available_dir, exist_ok=True)
+    os.makedirs(webhosting_srv.sites_available_dir, exist_ok=True)
+    reset_disable_club_conf(webhosting_srv)
+    for file in os.listdir(webhosting_srv.sites_available_dir):
+        os.unlink(os.path.join(webhosting_srv.sites_available_dir, file))
+    if os.path.isdir(webhosting_srv.apache_dir + '/.git'):
+        shutil.rmtree(webhosting_srv.apache_dir + '/.git')
+
+
 @pytest.fixture(scope='module')
 def app_process(cfg, app, http_client):
     port = cfg.get('ceod_port')

tests/resources/apache2/disable-club.conf

@@ -0,0 +1,8 @@
+# This file would normally be in /etc/apache2/conf-available/disable-club.conf
+# on caffeine.
+
+Alias /~sysadmin/disabled /users/sysadmin/www/disabled
+
+<Directory "/users/somerandomclub/www">
+    Include snippets/disable-club.conf
+</Directory>

tests/utils.py

@@ -1,6 +1,7 @@
 import ceod.utils as ceod_utils
 import contextlib
 import os
+import shutil
 import subprocess
 from subprocess import DEVNULL
 import tempfile
@@ -70,3 +71,13 @@ def set_datetime_in_app_process(app_process, value):
 
 def restore_datetime_in_app_process(app_process):
     set_datetime_in_app_process(app_process, None)
+
+
+def reset_disable_club_conf(webhosting_srv):
+    shutil.copy('tests/resources/apache2/disable-club.conf', webhosting_srv.conf_available_dir)
+
+
+def create_php_file_for_club(clubs_home, club_name):
+    www = f'{clubs_home}/{club_name}/www'
+    os.makedirs(www, exist_ok=True)
+    open(www + '/index.php', 'w').close()