.drone.yml

@@ -10,7 +10,7 @@ steps:
   # way to share system packages between steps
   commands:
     # install dependencies
-    - apt update && apt install -y libkrb5-dev libpq-dev python3-dev
+    - apt update && apt install -y libkrb5-dev libpq-dev python3-dev libaugeas0
     - python3 -m venv venv
     - . venv/bin/activate
     - pip install -r dev-requirements.txt

.drone/auth1-setup.sh

@@ -16,8 +16,6 @@ if [ -n "$CI" ]; then
     rm /tmp/hosts
 fi
 
-export DEBIAN_FRONTEND=noninteractive
-apt update
 apt install -y psmisc
 
 # If we don't do this then OpenLDAP uses a lot of RAM
@@ -98,7 +96,6 @@ while true; do
     fi
 done
 
-apt install -y netcat-openbsd
 # sync with phosphoric-acid
 nc -l 0.0.0.0 9000 &
 if [ -z "$CI" ]; then

.drone/coffee-setup.sh

@@ -8,9 +8,6 @@ set -ex
 add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee
 add_fqdn_to_hosts $(get_ip_addr auth1) auth1
 
-export DEBIAN_FRONTEND=noninteractive
-apt update
-
 apt install --no-install-recommends -y default-mysql-server postgresql
 
 # MYSQL
@@ -49,7 +46,6 @@ REVOKE ALL ON SCHEMA public FROM public;
 GRANT ALL ON SCHEMA public TO postgres;
 EOF" postgres
 
-apt install -y netcat-openbsd
 if [ -z "$CI" ]; then
     auth_setup coffee
 fi

.drone/common.sh

@@ -23,6 +23,16 @@ cp .drone/k8s-authority.crt /etc/csc/k8s-authority.crt
 # openssl is actually already present in the python Docker image,
 # so we don't need to mock it out
 
+# netcat is used for synchronization between the containers
+export DEBIAN_FRONTEND=noninteractive
+apt update
+apt install -y netcat-openbsd
+if [ "$(hostname)" != auth1 ]; then
+    # ceod uses Augeas, which is not installed by default in the Python
+    # Docker container
+    apt install -y libaugeas0
+fi
+
 get_ip_addr() {
     getent hosts $1 | cut -d' ' -f1
 }

.drone/mail-setup.sh

@@ -14,9 +14,6 @@ python -m tests.MockSMTPServer &
 python -m tests.MockCloudStackServer &
 python -m tests.MockHarborServer &
 
-export DEBIAN_FRONTEND=noninteractive
-apt update
-apt install -y netcat-openbsd
 auth_setup mail
 
 # for the VHostManager

.drone/phosphoric-acid-setup.sh

@@ -13,9 +13,6 @@ if [ -z "$CI" ]; then
     add_fqdn_to_hosts $(get_ip_addr mail) mail
 fi
 
-export DEBIAN_FRONTEND=noninteractive
-apt update
-apt install -y netcat-openbsd
 auth_setup phosphoric-acid
 
 # initialize the skel directory

README.md

@@ -16,8 +16,7 @@ Docker containers instead, which are much easier to work with than the VM.
 
 First, make sure you create the virtualenv:
 ```sh
-docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \
-    sh -c 'python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
+docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
 ```
 Then bring up the containers:
 ```sh

ceo/cli/entrypoint.py

@@ -10,6 +10,7 @@ from .mailman import mailman
 from .cloud import cloud
 from .k8s import k8s
 from .registry import registry
+from .webhosting import webhosting
 
 
 @click.group()
@@ -27,3 +28,4 @@ cli.add_command(mailman)
 cli.add_command(cloud)
 cli.add_command(k8s)
 cli.add_command(registry)
+cli.add_command(webhosting)

ceo/cli/webhosting.py

@@ -0,0 +1,37 @@
+import click
+
+from ..utils import http_post
+from .utils import handle_sync_response
+
+
+@click.group(short_help='Manage websites hosted by the main CSC web server')
+def webhosting():
+    pass
+
+
+@webhosting.command(short_help='Disable club sites with no active club reps')
+@click.option('--dry-run', is_flag=True, default=False)
+@click.option('--remove-inactive-club-reps', is_flag=True, default=False)
+def disableclubsites(dry_run, remove_inactive_club_reps):
+    params = {}
+    if dry_run:
+        params['dry_run'] = 'true'
+    if remove_inactive_club_reps:
+        params['remove_inactive_club_reps'] = 'true'
+    if not dry_run:
+        click.confirm('Are you sure you want to disable the websites of clubs with no active club reps?', abort=True)
+
+    resp = http_post('/api/webhosting/disableclubsites', params=params)
+    disabled_club_names = handle_sync_response(resp)
+    if len(disabled_club_names) == 0:
+        if dry_run:
+            click.echo('No websites would have been disabled.')
+        else:
+            click.echo('No websites were disabled.')
+    else:
+        if dry_run:
+            click.echo('The following club websites would have been disabled:')
+        else:
+            click.echo('The following club websites were disabled:')
+        for club_name in disabled_club_names:
+            click.echo(club_name)

ceo/utils.py

@@ -21,6 +21,8 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response:
         host = cfg.get('ceod_mailman_host')
     elif path.startswith('/api/cloud'):
         host = cfg.get('ceod_cloud_host')
+    elif path.startswith('/api/webhosting'):
+        host = cfg.get('ceod_webhosting_host')
     else:
         host = cfg.get('ceod_admin_host')
     return client.request(

ceo_common/interfaces/IClubWebHostingService.py

@@ -0,0 +1,45 @@
+from typing import List
+
+from zope.interface import Interface
+
+
+class IClubWebHostingService(Interface):
+    """
+    Performs operations on the configuration files of club websites hosted
+    by the CSC.
+    """
+
+    def begin_transaction():
+        """
+        Performs any steps necessary to query the website config files.
+        This must be called BEFORE any of the methods below, using a context
+        expression like so:
+
+        with club_site_mgr.begin_transaction():
+            club_site_mgr.disable_club_site('club1')
+            ...
+            club_site_mgr.commit()
+        """
+
+    def commit():
+        """
+        Writes the config file changes to disk. This must be called at the
+        end of the context expression.
+        """
+
+    def disable_club_site(club_name: str):
+        """
+        Disables the site for the club. Note that commit() must still be
+        called to commit this change.
+        """
+
+    def disable_sites_for_inactive_clubs(
+        dry_run: bool = False,
+        remove_inactive_club_reps: bool = False,
+    ) -> List[str]:
+        """
+        Disables sites for inactive clubs. If remove_inactive_club_reps is set
+        to True, then inactive club reps will be removed from club groups.
+        The list of clubs whose sites were disabled (or would have been
+        disabled, if dry_run is True) is returned.
+        """

ceo_common/interfaces/ILDAPService.py

@@ -9,10 +9,10 @@ from .IGroup import IGroup
 class ILDAPService(Interface):
     """An interface to the LDAP database."""
 
-    def uid_to_dn(self, uid: str) -> str:
+    def uid_to_dn(uid: str) -> str:
         """Get the LDAP DN for the user with this UID."""
 
-    def group_cn_to_dn(self, cn: str) -> str:
+    def group_cn_to_dn(cn: str) -> str:
         """Get the LDAP DN for the group with this CN."""
 
     def get_user(username: str) -> IUser:
@@ -24,7 +24,7 @@ class ILDAPService(Interface):
         Useful for displaying a list of users in a compact way.
         """
 
-    def get_users_with_positions(self) -> List[IUser]:
+    def get_users_with_positions() -> List[IUser]:
         """Retrieve users who have a non-empty position attribute."""
 
     def add_user(user: IUser):
@@ -36,7 +36,7 @@ class ILDAPService(Interface):
     def remove_user(user: IUser):
         """Remove this user from the database."""
 
-    def get_group(cn: str, is_club: bool = False) -> IGroup:
+    def get_group(cn: str) -> IGroup:
         """Retrieve the group with the given cn (Unix group name)."""
 
     def add_group(group: IGroup):
@@ -88,15 +88,33 @@ class ILDAPService(Interface):
         described above.
         """
 
-    def get_nonflagged_expired_users(self) -> List[IUser]:
+    def get_nonflagged_expired_users() -> List[IUser]:
         """
         Retrieves members whose term or nonMemberTerm does not contain the
         current or the last term.
         """
 
-    def get_expiring_users(self) -> List[IUser]:
+    def get_expiring_users() -> List[IUser]:
         """
         Retrieves members whose membership will expire in less than a month.
         This is used to send membership renewal reminders at the beginning
         of a term, during the one-month grace period.
         """
+
+    def get_clubs() -> List[IGroup]:
+        """
+        Retrieves all clubs.
+        """
+
+    # couldn't import the Term class from ceo_common.model due to some
+    # circular import issue...
+    def get_club_reps_non_member_terms(club_reps: List[str]) -> Dict[str, List['Term']]:  # noqa: F821
+        """
+        Retrieves the non-member terms for the given club reps.
+        e.g.
+        {
+            "user1": [w2022, s2022],
+            "user2": [s2022],
+            ...
+        }
+        """

ceo_common/interfaces/IMailService.py

@@ -24,20 +24,26 @@ class IMailService(Interface):
         during the transaction.
         """
 
-    def send_membership_renewal_reminder(self, user: IUser):
+    def send_membership_renewal_reminder(user: IUser):
         """
         Send a reminder to the user that their membership will expire
         soon.
         """
 
-    def send_cloud_account_will_be_deleted_message(self, user: IUser):
+    def send_cloud_account_will_be_deleted_message(user: IUser):
         """
         Send a warning message to the user that their cloud resources
         will be deleted if they do not renew their membership.
         """
 
-    def send_cloud_account_has_been_deleted_message(self, user: IUser):
+    def send_cloud_account_has_been_deleted_message(user: IUser):
         """
         Send a message to the user that their cloud resources have
         been deleted.
         """
+
+    def send_club_website_has_been_disabled_message(club: str, address: str):
+        """
+        Send a message to the club stating that their website has been
+        disabled.
+        """

ceo_common/interfaces/__init__.py

@@ -14,3 +14,4 @@ from .IDatabaseService import IDatabaseService
 from .IVHostManager import IVHostManager
 from .IKubernetesService import IKubernetesService
 from .IContainerRegistryService import IContainerRegistryService
+from .IClubWebHostingService import IClubWebHostingService

ceo_common/model/HTTPClient.py

@@ -43,7 +43,7 @@ class HTTPClient:
             if flask.has_request_context() and 'client_token' in g:
                 # This is reached when we are the server and the client has
                 # forwarded their credentials to us.
-                spnego_kwargs['creds'] = gssapi.Credentials(token=flask.g.client_token)
+                spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token)
             elif delegate:
                 # This is reached when we are the client and we want to
                 # forward our credentials to the server.

ceod/api/app_factory.py

@@ -9,13 +9,13 @@ from .error_handlers import register_error_handlers
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
     IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
     ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager, \
-    IContainerRegistryService
+    IContainerRegistryService, IClubWebHostingService
 from ceo_common.model import Config, HTTPClient, RemoteMailmanService
 from ceod.api.spnego import init_spnego
 from ceod.model import KerberosService, LDAPService, FileService, \
     MailmanService, MailService, UWLDAPService, CloudStackService, \
     CloudResourceManager, KubernetesService, VHostManager, \
-    ContainerRegistryService
+    ContainerRegistryService, ClubWebHostingService
 from ceod.db import MySQLService, PostgreSQLService
 
 
@@ -49,6 +49,10 @@ def create_app(flask_config={}):
         from ceod.api import cloud
         app.register_blueprint(cloud.bp, url_prefix='/api/cloud')
 
+    if hostname == cfg.get('ceod_webhosting_host'):
+        from ceod.api import webhosting
+        app.register_blueprint(webhosting.bp, url_prefix='/api/webhosting')
+
     from ceod.api import groups
     app.register_blueprint(groups.bp, url_prefix='/api/groups')
 
@@ -112,6 +116,11 @@ def register_services(app):
     uwldap_srv = UWLDAPService()
     component.provideUtility(uwldap_srv, IUWLDAPService)
 
+    # ClubWebHostingService
+    if hostname == cfg.get('ceod_webhosting_host'):
+        webhosting_srv = ClubWebHostingService()
+        component.provideUtility(webhosting_srv, IClubWebHostingService)
+
     # MySQLService, PostgreSQLService
     if hostname == cfg.get('ceod_database_host'):
         mysql_srv = MySQLService()

ceod/api/webhosting.py

@@ -0,0 +1,22 @@
+from flask import Blueprint, request
+from flask.json import jsonify
+from zope import component
+
+from .utils import authz_restrict_to_syscom, is_truthy
+from ceo_common.interfaces import IClubWebHostingService
+
+
+bp = Blueprint('webhosting', __name__)
+
+
+@bp.route('/disableclubsites', methods=['POST'])
+@authz_restrict_to_syscom
+def expire_club_sites():
+    dry_run = is_truthy(request.args.get('dry_run', 'false'))
+    remove_inactive_club_reps = is_truthy(request.args.get('remove_inactive_club_reps', 'false'))
+    webhosting_srv = component.getUtility(IClubWebHostingService)
+    disabled_sites = webhosting_srv.disable_sites_for_inactive_clubs(
+        dry_run=dry_run,
+        remove_inactive_club_reps=remove_inactive_club_reps,
+    )
+    return jsonify(disabled_sites)

ceod/model/ClubWebHostingService.py

@@ -0,0 +1,250 @@
+from collections import defaultdict
+import contextlib
+import glob
+import os
+import re
+import subprocess
+from threading import Lock
+import traceback
+from typing import List
+
+from augeas import Augeas
+from zope import component
+from zope.interface import implementer
+
+from ceo_common.interfaces import ILDAPService, IMailService, \
+    IClubWebHostingService, IConfig
+from ceo_common.logger_factory import logger_factory
+from ceo_common.model import Term
+
+# NOTE: We are assuming that the name of a club is the same as its home
+# directory (under /users). While this rule is not enforced, it has been true
+# up until now.
+# FIXME: don't assume this
+APACHE_USERDIR_RE = re.compile(r'^/users/(?P<club_name>[0-9a-z-]+)/www/?$')
+
+# This is where all the <Directory> directives for disabled clubs are stored.
+APACHE_DISABLED_CLUBS_FILE = 'conf-available/disable-club.conf'
+# This is the file which contains the snippet to disable a single club.
+APACHE_DISABLING_SNIPPET_FILE = 'snippets/disable-club.conf'
+# This is the maximum number of consecutive terms for which a club is allowed
+# to have no active club reps. After this many terms have passed, the club's
+# website may be disabled.
+MAX_TERMS_WITH_NO_ACTIVE_CLUB_REPS = 3
+
+logger = logger_factory(__name__)
+
+
+def is_a_disabling_snippet(snippet_path: str) -> bool:
+    return snippet_path.startswith('snippets/disable-')
+
+
+@implementer(IClubWebHostingService)
+class ClubWebHostingService:
+    def __init__(self, augeas_root='/'):
+        cfg = component.getUtility(IConfig)
+        self.aug_root = augeas_root
+        self.apache_dir = os.path.join(augeas_root, 'etc/apache2')
+        self.sites_available_dir = os.path.join(self.apache_dir, 'sites-available')
+        self.conf_available_dir = os.path.join(self.apache_dir, 'conf-available')
+        self.clubs_home = cfg.get('clubs_home')
+        self.aug = None
+        self.clubs = None
+        self.made_at_least_one_change = False
+        self.lock = Lock()
+
+    @contextlib.contextmanager
+    def _hold_lock(self):
+        try:
+            self.lock.acquire()
+            yield
+        finally:
+            self.lock.release()
+
+    @contextlib.contextmanager
+    def begin_transaction(self):
+        with self._hold_lock():
+            try:
+                self.aug = Augeas(self.aug_root)
+                self.clubs = defaultdict(lambda: {'disabled': False, 'email': None})
+                self._get_club_emails()
+                self._get_disabled_sites()
+                yield
+            finally:
+                if self.aug is not None:
+                    self.aug.close()
+                self.aug = None
+                self.clubs = None
+                self.made_at_least_one_change = False
+
+    def _run(self, args: List[str], **kwargs):
+        subprocess.run(args, check=True, **kwargs)
+
+    def _reload_web_server(self):
+        logger.debug('Reloading Apache')
+        self._run(['systemctl', 'reload', 'apache2'])
+
+    # This requires the APACHE_CONFIG_CRON environment variable to be
+    # set to 1 (e.g. in a systemd drop-in)
+    # See /etc/apache2/.git/hooks/pre-commit on caffeine
+    def _git_commit(self):
+        if not os.path.isdir(os.path.join(self.apache_dir, '.git')):
+            logger.debug('No git folder found in Apache directory')
+            return
+        logger.debug('Committing changes to git repository')
+        self._run(['git', 'add', APACHE_DISABLED_CLUBS_FILE], cwd=self.apache_dir)
+        self._run(['git', 'commit', '-m', '[ceo] disable club websites'], cwd=self.apache_dir)
+
+    def commit(self):
+        if not self.made_at_least_one_change:
+            return
+        logger.debug('Saving Augeas changes')
+        self.aug.save()
+        self._reload_web_server()
+        self._git_commit()
+
+    def _get_club_emails(self):
+        config_file_paths = glob.glob(self.sites_available_dir + '/club-*.conf')
+        for config_file_path in config_file_paths:
+            club_name = None
+            club_email = None
+            filename = os.path.basename(config_file_path)
+            directive_paths = self.aug.match(f'/files/etc/apache2/sites-available/{filename}/VirtualHost/directive')
+            for directive_path in directive_paths:
+                directive = self.aug.get(directive_path)
+                directive_value = self.aug.get(directive_path + '/arg')
+                if directive == 'DocumentRoot':
+                    match = APACHE_USERDIR_RE.match(directive_value)
+                    if match is not None:
+                        club_name = match.group('club_name')
+                elif directive == 'ServerAdmin':
+                    club_email = directive_value
+            if club_name is not None:
+                self.clubs[club_name]['email'] = club_email
+
+    def _get_disabled_sites(self):
+        config_file_paths = glob.glob(self.conf_available_dir + '/disable-*.conf')
+        for config_file_path in config_file_paths:
+            filename = os.path.basename(config_file_path)
+            directory_paths = self.aug.match(f'/files/etc/apache2/conf-available/{filename}/Directory')
+            for directory_path in directory_paths:
+                directory = self.aug.get(directory_path + '/arg')
+                match = APACHE_USERDIR_RE.match(directory)
+                if match is None:
+                    continue
+                club_name = match.group('club_name')
+                directive_paths = self.aug.match(directory_path + '/directive')
+                for directive_path in directive_paths:
+                    if self.aug.get(directive_path) != 'Include':
+                        continue
+                    included_file = self.aug.get(directive_path + '/arg')
+                    if is_a_disabling_snippet(included_file):
+                        self.clubs[club_name]['disabled'] = True
+
+    def disable_club_site(self, club_name: str):
+        logger.info(f'Disabling website for {club_name}')
+        directory_path = f'/files/etc/apache2/{APACHE_DISABLED_CLUBS_FILE}/Directory'
+        num_directories = len(self.aug.match(directory_path))
+        # Create a new <Directory> section
+        directory_path += '[%d]' % (num_directories + 1)
+        # FIXME: use self.clubs_home here instead (need to update unit tests)
+        self.aug.set(directory_path + '/arg', f'/users/{club_name}/www')
+        self.aug.set(directory_path + '/directive', 'Include')
+        self.aug.set(directory_path + '/directive/arg', APACHE_DISABLING_SNIPPET_FILE)
+
+        self.clubs[club_name]['disabled'] = True
+        self.made_at_least_one_change = True
+
+    def _site_uses_php(self, club_name: str) -> bool:
+        www = f'{self.clubs_home}/{club_name}/www'
+        if os.path.isdir(www):
+            # We're just going to look one level deep; that should be good enough.
+            for filename in os.listdir(www):
+                filepath = os.path.join(www, filename)
+                if os.path.isfile(filepath) and filename.endswith('.php'):
+                    return True
+        return False
+
+    # This method needs to be called from within a transaction (uses self.clubs)
+    def _need_to_disable_inactive_club_site(self, club_name: str) -> bool:
+        if self.clubs[club_name]['disabled']:
+            # already disabled - nothing to do
+            return False
+        if not self._site_uses_php(club_name):
+            # These are the only sites which are actually a security concern
+            # to us - it's OK for a purely static website to be unmaintained.
+            return False
+        return True
+
+    def disable_sites_for_inactive_clubs(
+        self,
+        dry_run: bool = False,
+        remove_inactive_club_reps: bool = False,
+    ) -> List[str]:
+        ldap_srv = component.getUtility(ILDAPService)
+        mail_srv = component.getUtility(IMailService)
+        all_clubs = ldap_srv.get_clubs()
+        club_rep_uids = list({
+            member
+            for club in all_clubs
+            for member in club.members
+        })
+        club_reps = ldap_srv.get_club_reps_non_member_terms(club_rep_uids)
+        # If a club rep's last non-member term is before cutoff_term, then
+        # they are considered inactive
+        cutoff_term = Term.current() - MAX_TERMS_WITH_NO_ACTIVE_CLUB_REPS
+        active_club_reps = {
+            club_rep
+            for club_rep, non_member_terms in club_reps.items()
+            if len(non_member_terms) > 0 and max(non_member_terms) >= cutoff_term
+        }
+        # a club is inactive if it does not have at least one active club rep
+        inactive_clubs = [
+            club
+            for club in all_clubs
+            if not any(map(lambda member: member in active_club_reps, club.members))
+        ]
+
+        # STEP 1: update the Apache configs
+        with self.begin_transaction():
+            clubs_to_disable = [
+                club.cn
+                for club in inactive_clubs
+                if self._need_to_disable_inactive_club_site(club.cn)
+            ]
+            if dry_run:
+                return clubs_to_disable
+
+            for club_name in clubs_to_disable:
+                self.disable_club_site(club_name)
+            self.commit()
+            # self.clubs is set to None once the transaction closes, so make a copy now
+            clubs_info = self.clubs.copy()
+
+        # STEP 2: send emails to clubs whose websites were disabled
+        clubs_who_were_not_notified = set()
+        for club_name in clubs_to_disable:
+            address = clubs_info['email']
+            if address is None:
+                clubs_who_were_not_notified.add(club_name)
+                continue
+            try:
+                mail_srv.send_club_website_has_been_disabled_message(club_name, address)
+            except Exception:
+                trace = traceback.format_exc()
+                logger.error(f'Failed to send email to {address}:\n{trace}')
+                clubs_who_were_not_notified.add(club_name)
+
+        # STEP 3: remove inactive club reps from Unix groups
+        if remove_inactive_club_reps:
+            for club in all_clubs:
+                if club.cn in clubs_who_were_not_notified:
+                    continue
+                # club.members gets modified after calling club.remove_member(),
+                # so we need to make a copy
+                members = club.members.copy()
+                for member in members:
+                    if member not in active_club_reps:
+                        logger.info(f'Removing {member} from {club.cn}')
+                        club.remove_member(member)
+        return clubs_to_disable

ceod/model/LDAPService.py

@@ -344,3 +344,37 @@ class LDAPService:
             conn.modify(self.uid_to_dn(uid), changes)
 
         return users_to_change
+
+    def _get_club_uids(self, conn: ldap3.Connection) -> List[str]:
+        conn.search(self.ldap_users_base, '(objectClass=club)', attributes=['uid'])
+        return [entry.uid.value for entry in conn.entries]
+
+    def get_clubs(self) -> List[IGroup]:
+        batch_size = 100
+        conn = self._get_ldap_conn()
+        club_uids = self._get_club_uids(conn)
+        clubs = []
+        for i in range(0, len(club_uids), batch_size):
+            club_uids_slice = club_uids[i:i + batch_size]
+            filter = '(|' + ''.join([f'(cn={uid})' for uid in club_uids_slice]) + ')'
+            conn.search(self.ldap_groups_base, filter, attributes=ldap3.ALL_ATTRIBUTES)
+            for entry in conn.entries:
+                clubs.append(Group.deserialize_from_ldap(entry))
+        return clubs
+
+    def get_club_reps_non_member_terms(self, club_reps: List[str]) -> Dict[str, List[Term]]:
+        batch_size = 100
+        conn = self._get_ldap_conn()
+        club_reps_terms = {}
+        for i in range(0, len(club_reps), batch_size):
+            club_reps_slice = club_reps[i:i + batch_size]
+            filter = '(|' + ''.join([f'(uid={uid})' for uid in club_reps_slice]) + ')'
+            conn.search(self.ldap_users_base, filter, attributes=['uid', 'nonMemberTerm'])
+            for entry in conn.entries:
+                uid = entry.uid.value
+                if 'nonMemberTerm' in entry.entry_attributes:
+                    non_member_terms = list(map(Term, entry.nonMemberTerm.values))
+                else:
+                    non_member_terms = []
+                club_reps_terms[uid] = non_member_terms
+        return club_reps_terms

ceod/model/MailService.py

@@ -137,3 +137,17 @@ class MailService:
             },
             body,
         )
+
+    def send_club_website_has_been_disabled_message(self, club: str, address: str):
+        template = self.jinja_env.get_template('club_website_has_been_disabled.j2')
+        body = template.render(club=club)
+        self.send(
+            f'Computer Science Club <ceo+webhosting@{self.base_domain}>',
+            f'{club} <{address}>',
+            {
+                'Subject': 'Your club website has been disabled',
+                'Cc': f'root@{self.base_domain}',
+                'Reply-To': f'syscom@{self.base_domain}',
+            },
+            body,
+        )

ceod/model/__init__.py

@@ -13,3 +13,4 @@ from .MailmanService import MailmanService
 from .VHostManager import VHostManager
 from .KubernetesService import KubernetesService
 from .ContainerRegistryService import ContainerRegistryService
+from .ClubWebHostingService import ClubWebHostingService

ceod/model/templates/club_website_has_been_disabled.j2

@@ -0,0 +1,20 @@
+Hello {{ club }} representatives,
+
+This is an automated message from ceo, the CSC Electronic Office.
+
+Since your club has not had any active CSC club reps for 3 or more consecutive
+terms, your club's website has been disabled. Websites require active
+maintenance and software upgrades, and an unmaintained website poses a
+security risk to CSC and to the University of Waterloo.
+
+If you wish for your site to be re-enabled, please register at least
+one club rep with CSC and, if necessary, upgrade the software running
+your website. A club rep membership is free. You can find instructions
+on our website for how to obtain a membership:
+
+https://csclub.uwaterloo.ca/get-involved/
+
+If you have any questions or concerns, please contact syscom@csclub.uwaterloo.ca.
+
+Sincerely,
+ceo

debian/control

@@ -11,6 +11,7 @@ Build-Depends: debhelper (>= 12.1.1),
     python3-venv (>= 3.7),
     libkrb5-dev (>= 1.17),
     libpq-dev (>= 11.13),
+    libaugeas0 (>= 1.11),
     scdoc (>= 1.9)
 
 Package: ceo-common
@@ -19,6 +20,7 @@ Depends: python3 (>= 3.7),
     krb5-user (>= 1.17),
     libkrb5-3 (>= 1.17),
     libpq5 (>= 11.13),
+    libaugeas0 (>= 1.11),
     ${python3:Depends},
     ${misc:Depends}
 Description: CSC Electronic Office common files

etc/ceo.ini

@@ -7,6 +7,8 @@ uw_domain = uwaterloo.ca
 admin_host = phosphoric-acid
 # this is the host with root access to the databases
 database_host = caffeine
+# this is the host where clubs' websites are hosted
+webhosting_host = caffeine
 # this is the host which can make API requests to Mailman
 mailman_host = mailman
 # this is the host running a CloudStack management server

etc/ceod.ini

@@ -8,6 +8,8 @@ admin_host = phosphoric-acid
 fs_root_host = phosphoric-acid
 # this is the host with root access to the databases
 database_host = caffeine
+# this is the host where clubs' websites are hosted
+webhosting_host = caffeine
 # this is the host which can make API requests to Mailman
 mailman_host = mailman
 # this is the host which is running a CloudStack management server

requirements.txt

@@ -7,9 +7,10 @@ Jinja2==3.1.2
 ldap3==2.9.1
 mysql-connector-python==8.0.26
 psycopg2==2.9.1
+python-augeas==1.1.0
 requests==2.26.0
 requests-gssapi==1.2.3
 urwid==2.1.2
-werkzeug==2.1.2
+Werkzeug==2.1.2
 zope.component==5.0.1
 zope.interface==5.4.0

tests/ceo/cli/test_webhosting.py

@@ -0,0 +1,62 @@
+from click.testing import CliRunner
+
+from ceo.cli import cli
+from ceo_common.model import Term
+from tests.utils import (
+    create_php_file_for_club,
+    reset_disable_club_conf,
+    set_datetime_in_app_process,
+    restore_datetime_in_app_process,
+)
+
+
+def test_disable_club_sites(
+    cfg, cli_setup, app_process, webhosting_srv, webhosting_srv_resources,
+    new_club_gen, new_user_gen, g_admin_ctx, ldap_srv_session,
+):
+    runner = CliRunner()
+    term = Term.current()
+    clubs_home = cfg.get('clubs_home')
+    with new_club_gen() as group, new_user_gen() as user:
+        create_php_file_for_club(clubs_home, group.cn)
+        user.add_non_member_terms([str(Term.current())])
+        group.add_member(user.uid)
+
+        set_datetime_in_app_process(app_process, (term + 4).to_datetime())
+
+        result = runner.invoke(cli, ['webhosting', 'disableclubsites', '--dry-run'])
+        assert result.exit_code == 0
+        assert result.output.endswith(
+            'The following club websites would have been disabled:\n'
+            f'{group.cn}\n'
+        )
+
+        result = runner.invoke(cli, ['webhosting', 'disableclubsites'], input='y\n')
+        assert result.exit_code == 0
+        assert result.output.endswith(
+            'The following club websites were disabled:\n'
+            f'{group.cn}\n'
+        )
+
+        with g_admin_ctx():
+            group = ldap_srv_session.get_group(group.cn)
+        assert group.members == [user.uid]
+
+        reset_disable_club_conf(webhosting_srv)
+
+        result = runner.invoke(
+            cli,
+            ['webhosting', 'disableclubsites', '--remove-inactive-club-reps'],
+            input='y\n'
+        )
+        assert result.exit_code == 0
+        assert result.output.endswith(
+            'The following club websites were disabled:\n'
+            f'{group.cn}\n'
+        )
+
+        with g_admin_ctx():
+            group = ldap_srv_session.get_group(group.cn)
+        assert group.members == []
+
+    restore_datetime_in_app_process(app_process)

tests/ceo_dev.ini

@@ -6,6 +6,7 @@ uw_domain = uwaterloo.internal
 # this is the host with the ceod/admin Kerberos key
 admin_host = phosphoric-acid
 database_host = coffee
+webhosting_host = coffee
 mailman_host = mail
 cloud_host = mail
 use_https = false

tests/ceod/api/test_webhosting.py

@@ -0,0 +1,43 @@
+import datetime
+from unittest.mock import patch
+
+from ceo_common.model import Term
+import ceo_common.utils
+from tests.utils import create_php_file_for_club, reset_disable_club_conf
+
+
+def test_disable_club_sites(
+    cfg, client, webhosting_srv, webhosting_srv_resources, new_club_gen,
+    new_user_gen, g_admin_ctx, ldap_srv_session,
+):
+    term = Term.current()
+    clubs_home = cfg.get('clubs_home')
+    with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
+        now_mock.return_value = datetime.datetime.now()
+        with new_club_gen() as group, new_user_gen() as user:
+            create_php_file_for_club(clubs_home, group.cn)
+            user.add_non_member_terms([str(Term.current())])
+            group.add_member(user.uid)
+
+            now_mock.return_value = (term + 4).to_datetime()
+            status, data = client.post('/api/webhosting/disableclubsites?dry_run=true')
+            assert status == 200
+            assert data == [group.cn]
+
+            status, data = client.post('/api/webhosting/disableclubsites')
+            assert status == 200
+            assert data == [group.cn]
+
+            with g_admin_ctx():
+                group = ldap_srv_session.get_group(group.cn)
+            assert group.members == [user.uid]
+
+            reset_disable_club_conf(webhosting_srv)
+
+            status, data = client.post('/api/webhosting/disableclubsites?remove_inactive_club_reps=true')
+            assert status == 200
+            assert data == [group.cn]
+
+            with g_admin_ctx():
+                group = ldap_srv_session.get_group(group.cn)
+            assert group.members == []

tests/ceod/model/test_webhosting.py

@@ -0,0 +1,163 @@
+import datetime
+import os
+import re
+import subprocess
+from unittest.mock import patch
+
+from ceo_common.model import Term
+import ceo_common.utils
+from tests.utils import create_php_file_for_club
+
+
+def create_website_config_for_club(sites_available_dir, cn, filename=None):
+    if filename is None:
+        filename = f'club-{cn}.conf'
+    filepath = os.path.join(sites_available_dir, filename)
+    with open(filepath, 'w') as fo:
+        fo.write(f"""
+        <VirtualHost *:80>
+            ServerName {cn}.uwaterloo.internal
+            ServerAdmin {cn}@{cn}.uwaterloo.internal
+            DocumentRoot /users/{cn}/www/
+        </VirtualHost>
+        """)
+
+
+def get_enabled_sites(webhosting_srv):
+    return [
+        club_name
+        for club_name, club_info in webhosting_srv.clubs.items()
+        if not club_info['disabled']
+    ]
+
+
+def test_disable_club_sites(webhosting_srv, webhosting_srv_resources, new_club_gen):
+    sites_available_dir = webhosting_srv.sites_available_dir
+    with new_club_gen() as group1, new_club_gen() as group2:
+        create_website_config_for_club(sites_available_dir, group1.cn)
+        create_website_config_for_club(sites_available_dir, group2.cn, 'club-somerandomname.conf')
+        with webhosting_srv.begin_transaction():
+            enabled_clubs = get_enabled_sites(webhosting_srv)
+            assert sorted(enabled_clubs) == [group1.cn, group2.cn]
+            webhosting_srv.disable_club_site(group1.cn)
+        # Make sure that if we don't call commit(), nothing gets saved
+        with webhosting_srv.begin_transaction():
+            enabled_clubs = get_enabled_sites(webhosting_srv)
+            assert sorted(enabled_clubs) == [group1.cn, group2.cn]
+            webhosting_srv.disable_club_site(group1.cn)
+            webhosting_srv.commit()
+        # Now that we committed the changes, they should be persistent
+        with webhosting_srv.begin_transaction():
+            enabled_clubs = get_enabled_sites(webhosting_srv)
+            assert enabled_clubs == [group2.cn]
+
+
+def test_disable_inactive_club_sites(
+    cfg, webhosting_srv, webhosting_srv_resources, g_admin_ctx, new_club_gen,
+    new_user_gen, mock_mail_server,
+):
+    sites_available_dir = webhosting_srv.sites_available_dir
+    term = Term.current()
+    clubs_home = cfg.get('clubs_home')
+    with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
+        now_mock.return_value = datetime.datetime.now()
+        with new_club_gen() as group1, \
+                new_club_gen() as group2, \
+                new_user_gen() as user1, \
+                new_user_gen() as user2:
+            create_website_config_for_club(sites_available_dir, group1.cn)
+            create_website_config_for_club(sites_available_dir, group2.cn)
+            create_php_file_for_club(clubs_home, group1.cn)
+            with g_admin_ctx():
+                # group1 has no club reps so it should be disabled
+                # group2 has no club reps but it doesn't use PHP
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [group1.cn]
+
+            user1.add_non_member_terms([str(Term.current())])
+            group1.add_member(user1.uid)
+            with g_admin_ctx():
+                # group1 has an active club rep, so it shouldn't be disabled anymore
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
+
+            now_mock.return_value = (term + 3).to_datetime()
+            with g_admin_ctx():
+                # club reps are allowed to be inactive for up to 3 terms
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
+
+            now_mock.return_value = (term + 4).to_datetime()
+            with g_admin_ctx():
+                # club site should be disabled now that club rep is inactive
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [group1.cn]
+
+            user2.add_non_member_terms([str(Term.current())])
+            group1.add_member(user2.uid)
+            with g_admin_ctx():
+                # group1 has a new club rep, so it shouldn't be disabled anymore
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
+
+            create_php_file_for_club(clubs_home, group2.cn)
+            group2.add_member(user2.uid)
+            with g_admin_ctx():
+                # user2 is an active club rep for both group1 and group2
+                assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
+
+            now_mock.return_value = (term + 8).to_datetime()
+            mock_mail_server.messages.clear()
+            subprocess.run(['git', 'init'], cwd=webhosting_srv.apache_dir, check=True)
+            with g_admin_ctx():
+                disabled_sites = webhosting_srv.disable_sites_for_inactive_clubs()
+            # user2 expired and both sites use PHP, so they should both be disabled
+            assert sorted(disabled_sites) == [group1.cn, group2.cn]
+            # since each club had a ServerAdmin directive, they should both have received
+            # notification emails
+            assert len(mock_mail_server.messages) == 2
+            with open(webhosting_srv.conf_available_dir + '/disable-club.conf') as fi:
+                disable_club_conf_content = fi.read()
+            print(disable_club_conf_content)
+            for group in [group1, group2]:
+                pat = re.compile(
+                    (
+                        rf'^<Directory "?/users/{group.cn}/www"?>\n'
+                        r'\s*Include snippets/disable-club\.conf\n'
+                        r'</Directory>$'
+                    ),
+                    re.MULTILINE
+                )
+                assert pat.search(disable_club_conf_content) is not None
+
+            with g_admin_ctx():
+                # Club sites should only be disabled once
+                assert webhosting_srv.disable_sites_for_inactive_clubs() == []
+
+    mock_mail_server.messages.clear()
+
+
+def test_remove_inactive_club_reps(
+    cfg, webhosting_srv, webhosting_srv_resources, g_admin_ctx, new_club_gen,
+    new_user_gen, ldap_srv_session,
+):
+    term = Term.current()
+    clubs_home = cfg.get('clubs_home')
+    with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
+        now_mock.return_value = datetime.datetime.now()
+        with new_club_gen() as group, \
+                new_user_gen() as user1, \
+                new_user_gen() as user2:
+            create_php_file_for_club(clubs_home, group.cn)
+
+            for user in [user1, user2]:
+                user.add_non_member_terms([str(Term.current())])
+                group.add_member(user.uid)
+            now_mock.return_value = (term + 4).to_datetime()
+            with g_admin_ctx():
+                webhosting_srv.disable_sites_for_inactive_clubs(remove_inactive_club_reps=True)
+                group = ldap_srv_session.get_group(group.cn)
+                assert group.members == []
+
+            # Make sure that inactive club reps get removed even if the site
+            # has already been disabled
+            group.add_member(user1.uid)
+            with g_admin_ctx():
+                webhosting_srv.disable_sites_for_inactive_clubs(remove_inactive_club_reps=True)
+                group = ldap_srv_session.get_group(group.cn)
+                assert group.members == []

tests/ceod_dev.ini

@@ -8,6 +8,8 @@ admin_host = phosphoric-acid
 fs_root_host = phosphoric-acid
 mailman_host = mail
 database_host = coffee
+# this is the host where clubs' websites are hosted
+webhosting_host = coffee
 cloud_host = mail
 use_https = false
 port = 9987

tests/ceod_test_local.ini

@@ -9,6 +9,7 @@ fs_root_host = phosphoric-acid
 mailman_host = phosphoric-acid
 database_host = phosphoric-acid
 cloud_host = phosphoric-acid
+webhosting_host = phosphoric-acid
 use_https = false
 port = 9988
 

tests/conftest.py

@@ -7,6 +7,7 @@ import shutil
 import subprocess
 from subprocess import DEVNULL
 import sys
+import tempfile
 import threading
 import time
 from unittest.mock import Mock
@@ -24,11 +25,12 @@ from .utils import (  # noqa: F401
     gssapi_token_ctx,
     ccache_cleanup,
     mocks_for_create_user_ctx,
+    reset_disable_club_conf,
 )
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
     IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
     IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \
-    ICloudResourceManager, IContainerRegistryService
+    ICloudResourceManager, IContainerRegistryService, IClubWebHostingService
 from ceo_common.model import Config, HTTPClient, Term
 import ceo_common.utils
 from ceod.api import create_app
@@ -36,7 +38,7 @@ from ceod.db import MySQLService, PostgreSQLService
 from ceod.model import KerberosService, LDAPService, FileService, User, \
     MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
     CloudStackService, KubernetesService, VHostManager, CloudResourceManager, \
-    ContainerRegistryService
+    ContainerRegistryService, ClubWebHostingService
 from .MockSMTPServer import MockSMTPServer
 from .MockMailmanServer import MockMailmanServer
 from .MockCloudStackServer import MockCloudStackServer
@@ -132,12 +134,12 @@ def g_syscom(app):
     """
     with gssapi_token_ctx('ctdalek') as token, app.app_context():
         try:
-            flask.g.sasl_user = 'ctdalek'
+            flask.g.auth_user = 'ctdalek'
             flask.g.client_token = token
             yield
         finally:
             flask.g.pop('client_token')
-            flask.g.pop('sasl_user')
+            flask.g.pop('auth_user')
 
 
 @pytest.fixture(scope='session')
@@ -237,6 +239,14 @@ def uwldap_srv(cfg, ldap_conn):
     delete_subtree(conn, base_dn)
 
 
+@pytest.fixture(scope='session')
+def webhosting_srv():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        srv = ClubWebHostingService(tmpdir)
+        component.getGlobalSiteManager().registerUtility(srv, IClubWebHostingService)
+        yield srv
+
+
 @pytest.fixture(scope='session')
 def mock_mail_server():
     mock_server = MockSMTPServer()
@@ -343,6 +353,7 @@ def app(
     k8s_srv,
     registry_srv,
     cloud_mgr,
+    webhosting_srv,
 ):
     app = create_app({'TESTING': True})
     return app
@@ -442,6 +453,31 @@ def new_user(new_user_gen):
         yield user
 
 
+@pytest.fixture
+def new_club_gen(client, g_admin_ctx, ldap_srv_session):  # noqa: F811
+    new_club_id_counter = 31001
+
+    @contextlib.contextmanager
+    def wrapper():
+        nonlocal new_club_id_counter
+        cn = 'test' + str(new_club_id_counter)
+        new_club_id_counter += 1
+        status, data = client.post('/api/groups', json={
+            'cn': cn,
+            'description': 'Test ' + str(new_club_id_counter),
+        })
+        assert status == 200
+        assert data[-1]['status'] == 'completed'
+        with g_admin_ctx():
+            group = ldap_srv_session.get_group(cn)
+        yield group
+        status, data = client.delete(f'/api/groups/{cn}')
+        assert status == 200
+        assert data[-1]['status'] == 'completed'
+
+    return wrapper
+
+
 @pytest.fixture
 def simple_group():
     return Group(
@@ -507,6 +543,17 @@ def uwldap_user(cfg, uwldap_srv, ldap_conn):
     conn.delete(dn)
 
 
+@pytest.fixture
+def webhosting_srv_resources(webhosting_srv):
+    os.makedirs(webhosting_srv.conf_available_dir, exist_ok=True)
+    os.makedirs(webhosting_srv.sites_available_dir, exist_ok=True)
+    reset_disable_club_conf(webhosting_srv)
+    for file in os.listdir(webhosting_srv.sites_available_dir):
+        os.unlink(os.path.join(webhosting_srv.sites_available_dir, file))
+    if os.path.isdir(webhosting_srv.apache_dir + '/.git'):
+        shutil.rmtree(webhosting_srv.apache_dir + '/.git')
+
+
 @pytest.fixture(scope='module')
 def app_process(cfg, app, http_client):
     port = cfg.get('ceod_port')

tests/resources/apache2/disable-club.conf

@@ -0,0 +1,8 @@
+# This file would normally be in /etc/apache2/conf-available/disable-club.conf
+# on caffeine.
+
+Alias /~sysadmin/disabled /users/sysadmin/www/disabled
+
+<Directory "/users/somerandomclub/www">
+    Include snippets/disable-club.conf
+</Directory>

tests/utils.py

@@ -1,6 +1,7 @@
 import ceod.utils as ceod_utils
 import contextlib
 import os
+import shutil
 import subprocess
 from subprocess import DEVNULL
 import tempfile
@@ -70,3 +71,13 @@ def set_datetime_in_app_process(app_process, value):
 
 def restore_datetime_in_app_process(app_process):
     set_datetime_in_app_process(app_process, None)
+
+
+def reset_disable_club_conf(webhosting_srv):
+    shutil.copy('tests/resources/apache2/disable-club.conf', webhosting_srv.conf_available_dir)
+
+
+def create_php_file_for_club(clubs_home, club_name):
+    www = f'{clubs_home}/{club_name}/www'
+    os.makedirs(www, exist_ok=True)
+    open(www + '/index.php', 'w').close()